github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jul 4th update

Browse Source
This commit is contained in:
Ned Wright 2024-07-04 14:10:34 +00:00
parent 01770475ec
commit 962bd31d46
17 changed files with 152 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
components/security_apps/orchestration/details_resolver/details_resolver.cc
View File

@ -152,6 +152,7 @@ DetailsResolver::Impl::isCloudStorageEnabled()
bool
DetailsResolver::Impl::isKernelVersion3OrHigher()
{
#if defined(gaia) || defined(smb)
static const string cmd =
"clish -c 'show version os kernel' | awk '{print $4}' "
"| cut -d '.' -f 1 | awk -F: '{ if ( $1 >= 3 ) {print 1} else {print 0}}'";
@ -160,12 +161,14 @@ DetailsResolver::Impl::isKernelVersion3OrHigher()
if (is_gogo.ok() && !is_gogo.unpack().empty()) {
return is_gogo.unpack().front() == '1';
}
#endif
return false;
}
bool
DetailsResolver::Impl::isGwNotVsx()
{
#if defined(gaia) || defined(smb)
static const string is_gw_cmd = "cpprod_util FwIsFirewallModule";
static const string is_vsx_cmd = "cpprod_util FWisVSX";
auto is_gw = DetailsResolvingHanlder::getCommandOutput(is_gw_cmd);
@ -173,6 +176,7 @@ DetailsResolver::Impl::isGwNotVsx()
if (is_gw.ok() && is_vsx.ok() && !is_gw.unpack().empty() && !is_vsx.unpack().empty()) {
return is_gw.unpack().front() == '1' && is_vsx.unpack().front() == '0';
}
#endif
return false;
}

14
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -24,14 +24,16 @@
Maybe<string>
checkSAMLSupportedBlade(const string &command_output)
{
string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
// uncomment when vpn will support SAML authentication
// string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
string supportedBlades[1] = {"identityServer"};
for(const string &blade : supportedBlades) {
if (command_output.find(blade) != string::npos) {
return string("true");
}
}
return genError("Current host does not have SAML capability");
return string("false");
}
Maybe<string>
@ -42,7 +44,7 @@ checkIDABlade(const string &command_output)
return string("true");
}
return genError("Current host does not have IDA installed");
return string("false");
}
Maybe<string>
@ -52,7 +54,7 @@ checkSAMLPortal(const string &command_output)
return string("true");
}
return genError("Current host does not have SAML Portal configured");
return string("false");
}
Maybe<string>
@ -61,7 +63,7 @@ checkPepIdaIdnStatus(const string &command_output)
if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) {
return string("true");
}
return genError("Current host does not have PEP control scaled_sharing enabled");
return string("false");
}
Maybe<string>
@ -87,7 +89,7 @@ checkIDP(shared_ptr<istream> file_stream)
}
}
return genError("Identity Provider was not found");
return string("false");
}
#endif // gaia

1
components/security_apps/orchestration/include/fog_communication.h
View File

@ -51,6 +51,7 @@ public:
private:
I_DeclarativePolicy *i_declarative_policy = nullptr;
std::string profile_mode;
};
#endif // __FOG_COMMUNICATION_H__

2
components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
View File

@ -141,7 +141,7 @@ DeclarativePolicyUtils::sendUpdatesToFog(
auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<DeclarativePolicyUtils>();
string exec_command =
getFilesystemPathConfig()
+ "/scripts/open-appsec-cloud-mgmt --upload_policy_only"
+ "/scripts/open-appsec-cloud-mgmt --config-upload-only"
+ " --access_token " + access_token
+ " --tenant_id " + tenant_id
+ " --profile_id " + profile_id;

11
components/security_apps/orchestration/update_communication/fog_communication.cc
View File

@ -32,6 +32,7 @@ FogCommunication::init()
{
FogAuthenticator::init();
i_declarative_policy = Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>();
profile_mode = getSettingWithDefault<string>("management", "profileManagedMode");
}
Maybe<void>
@ -66,6 +67,16 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
Maybe<string> maybe_new_data = request.getData();
string data_checksum = maybe_new_data.ok() ? maybe_new_data.unpack() : "";
if (profile_mode != policy_mgmt_mode) {
dbgTrace(D_ORCHESTRATOR)
<< "The profile managed mode was changed from: "
<< profile_mode
<< " to: "
<< policy_mgmt_mode;
profile_mode = policy_mgmt_mode;
i_declarative_policy->turnOnApplyPolicyFlag();
}
if (i_declarative_policy->shouldApplyPolicy()) {
string policy_response = i_declarative_policy->getUpdate(request);
if (!policy_response.empty()) {

2
components/security_apps/waap/waap_clib/ScanResult.cc
View File

@ -23,6 +23,7 @@ unescaped_line(),
param_name(),
location(),
score(0.0f),
scoreNoFilter(0.0f),
scoreArray(),
keywordCombinations(),
attack_types(),
@ -40,6 +41,7 @@ void Waf2ScanResult::clear()
param_name.clear();
location.clear();
score = 0;
scoreNoFilter = 0;
scoreArray.clear();
keywordCombinations.clear();
attack_types.clear();

1
components/security_apps/waap/waap_clib/ScanResult.h
View File

@ -29,6 +29,7 @@ struct Waf2ScanResult {
std::string param_name;
std::string location;
double score;
double scoreNoFilter;
std::vector<double> scoreArray;
std::vector<std::string> keywordCombinations;
std::set<std::string> attack_types;

2
components/security_apps/waap/waap_clib/WaapConversions.cc
View File

@ -39,7 +39,7 @@ namespace Conversions {
return HIGH_THREAT;
}
bool shouldDoWafBlocking(const IWaapConfig* pWaapConfig, ThreatLevel threatLevel)
bool shouldDoWafBlocking(const IWaapConfig* const pWaapConfig, ThreatLevel threatLevel)
{
if (pWaapConfig == NULL)
{

2
components/security_apps/waap/waap_clib/WaapConversions.h
View File

@ -20,7 +20,7 @@
namespace Waap {
namespace Conversions {
ThreatLevel convertFinalScoreToThreatLevel(double finalScore);
bool shouldDoWafBlocking(const IWaapConfig* pSitePolicy, ThreatLevel threatLevel);
bool shouldDoWafBlocking(const IWaapConfig* const pSitePolicy, ThreatLevel threatLevel);
}
}

19
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@ -25,7 +25,7 @@ USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
// id generated by xml parser for an entity attribute
const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f";
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName)
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning)
{
std::string source = m_transaction->getSourceIdentifier();
@ -33,8 +33,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
Waap::Keywords::KeywordsSet keywordsSet;
Waap::Keywords::computeKeywordsSet(keywordsSet, res.keyword_matches, res.found_patterns);
if (applyLearning) {
std::string param_name = IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction);
dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name;
dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name <<
", indicators count: " << keywordsSet.size();
m_transaction->getAssetState()->logIndicatorsInFilters(param_name, keywordsSet, m_transaction);
m_transaction->getAssetState()->filterKeywords(param_name, keywordsSet, res.filtered_keywords);
@ -47,7 +49,8 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
}
m_transaction->getAssetState()->filterKeywordsByParameters(res.param_name, keywordsSet);
dbgTrace(D_WAAP_SCANNER) << "post filtering indicators count: " << keywordsSet.size();
}
// The keywords are only removed in production, they are still used while building scores
if (!m_transaction->get_ignoreScore()) {
m_transaction->getAssetState()->removeKeywords(keywordsSet);
@ -148,9 +151,16 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
// Select scores pool by location
std::string poolName = Waap::Scores::getScorePoolNameByLocation(location);
Waf2ScanResult nonFilterRes = res;
res.scoreNoFilter = getScoreData(nonFilterRes, poolName, false);
double score = getScoreData(res, poolName);
dbgTrace(D_WAAP_SCANNER) << "score: " << score;
// call shouldIgnoreOverride post score calculation and filtering to evaluate ignore override effectivness
res.score = score;
m_transaction->shouldIgnoreOverride(res);
dbgTrace(D_WAAP_SCANNER) << "score: " << score << " should ignore: " << ignoreOverride;
// Add record about scores to the notes[] log (also reported in logs)
if (score > 1.0f) {
DetectionEvent(location, res.keyword_matches).notify();
@ -166,6 +176,7 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
if (isKeyCspReport(key, res, dp) || ignoreOverride) {
dbgTrace(D_WAAP_SCANNER) << "Ignoring parameter key/value " << res.param_name <<
" due to ignore action in override";
res.score = 0;
m_bIgnoreOverride = true;
return false;
}

2
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@ -43,7 +43,7 @@ namespace Waap {
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);

85
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -329,6 +329,7 @@ Waf2Transaction::Waf2Transaction() :
is_schema_validation(false),
m_waf2TransactionFlags()
{
m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = 0;
I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
@ -1729,6 +1730,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin());
waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds));
}
if (!m_exceptionLearned.empty()) {
std::vector<std::string> vLearningAffected(m_exceptionLearned.size());
std::copy(m_exceptionLearned.begin(), m_exceptionLearned.end(), vLearningAffected.begin());
waapLog.addToOrigin(LogField("redundantExceptionIdList", vLearningAffected));
}
}
}
@ -1809,12 +1815,6 @@ Waf2Transaction::sendLog()
return;
}
dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size();
bool shouldBlock = false;
if (m_overrideState.bForceBlock) {
// If override forces "reject" decision, mention it in the "override" log field.
@ -2091,7 +2091,30 @@ Waf2Transaction::decideAutonomousSecurity(
transactionResult.threatLevel = threat;
}
dbgTrace(D_WAAP_OVERRIDE) << "override ids count: " << m_matchedOverrideIds.size();
// Apply overrides
for (auto it = m_overridePostFilterMaxScore.begin(); it != m_overridePostFilterMaxScore.end(); it++) {
const string id = it->first;
if (m_overrideState.forceBlockIds.find(id) != m_overrideState.forceBlockIds.end()) {
// blocked effectivness is calculates later from the force block exception ids list
continue;
}
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(it->second);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threat);
dbgTrace(D_WAAP_OVERRIDE) << "checking effectivness of override: " << id << ", should have blocked: " << shouldBlock
<< ", scores: " << m_overridePostFilterMaxScore[id] << ", " << m_overrideOriginalMaxScore[id];
if (shouldBlock) {
m_effectiveOverrideIds.insert(id);
} else {
ThreatLevel threatNoFilter = Waap::Conversions::convertFinalScoreToThreatLevel(
m_overrideOriginalMaxScore[id]
);
if (Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threatNoFilter)) {
m_exceptionLearned.insert(id);
}
}
}
if (m_overrideState.bForceBlock) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces REJECT ...";
@ -2105,25 +2128,25 @@ Waf2Transaction::decideAutonomousSecurity(
}
}
else if (m_overrideState.bForceException) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
dbgTrace(D_WAAP) << "de cideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces ALLOW ...";
if (m_scanResult) {
// on accept exception the decision is not set and needs to be calculated to determine effectivness
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat);
if (shouldBlock) {
m_effectiveOverrideIds.insert(
m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end()
);
}
}
decision->setBlock(false);
if (!m_overrideState.bIgnoreLog)
{
decision->setOverridesLog(true);
}
} else if (!m_matchedOverrideIds.empty()) {
if (!m_overrideState.bIgnoreLog)
{
decision->setOverridesLog(true);
}
}
dbgTrace(D_WAAP_OVERRIDE) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size() <<
" learned overrides count: " << m_exceptionLearned.size();
bool log_all = false;
@ -2262,7 +2285,7 @@ bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
return false;
}
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
@ -2305,6 +2328,24 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (const auto &behavior : behaviors) {
if (!res.filtered_keywords.empty() || res.score > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
std::string overrideId = behavior.getId();
if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
m_overridePostFilterMaxScore[overrideId] = res.score;
} else {
if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) {
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
}
if (res.score > m_overridePostFilterMaxScore[overrideId]) {
m_overridePostFilterMaxScore[overrideId] = res.score;
}
}
if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) {
m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter;
}
}
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
@ -2312,12 +2353,6 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);
}
if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId)
{
if (!overrideId.empty()) {
m_effectiveOverrideIds.insert(overrideId);
}
}
return true;
}
}

3
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -293,6 +293,9 @@ private:
// Matched override IDs
std::set<std::string> m_matchedOverrideIds;
std::set<std::string> m_effectiveOverrideIds;
std::set<std::string> m_exceptionLearned;
std::map<std::string, double> m_overrideOriginalMaxScore;
std::map<std::string, double> m_overridePostFilterMaxScore;
//csrf state
Waap::CSRF::State m_csrfState;

6
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@ -459,9 +459,15 @@ Waf2Transaction::getUserLimitVerdict()
}
else if (mode == AttackMitigationMode::PREVENT) {
decision->setLog(true);
if (!m_overrideState.bForceException) {
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
} else {
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
}
return verdict;

6
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
View File

@ -109,7 +109,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
Intelligence::IntelligenceRequest query(requests, true, true, false, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"queriesTypes\": {\n"
" \"proxyToCloud\": false\n"
" },\n"
" \"queries\": [\n"
@ -150,7 +150,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequestsProxied) {
Intelligence::IntelligenceRequest query(requests, true, true, true, MessageMetadata("", 0));
std::string expected = "{\n"
" \"queryTypes\": {\n"
" \"queriesTypes\": {\n"
" \"proxyToCloud\": true\n"
" },\n"
" \"queries\": [\n"
@ -191,7 +191,7 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
Intelligence::IntelligenceRequest query(requests, false, true, false, MessageMetadata("", 0));
std::string expected = "{"
"\"queryTypes\":{"
"\"queriesTypes\":{"
"\"proxyToCloud\":false"
"},"
"\"queries\":[{"

2
core/intelligence_is_v2/intelligence_request.cc
View File

@ -90,7 +90,7 @@ IntelligenceRequest::genJson() const
{
cereal::JSONOutputArchive out_ar(json_stream);
out_ar.setNextName("queryTypes");
out_ar.setNextName(isBulk() ? "queriesTypes" : "queryTypes");
out_ar.startNode();
out_ar(cereal::make_nvp("proxyToCloud", is_proxy));
out_ar.finishNode();

26
nodes/orchestration/package/open-appsec-cloud-mgmt
View File

@ -3,6 +3,7 @@
POLICY_TEMP_PATH="/tmp/policy_temp.json"
DECLARATIVE_CONFIG_PATH="/etc/cp/conf/declarative_config.cfg"
CHANGE_AGENT_MODE=true
UPLOAD_AGENT_POLICY=false
ra_token=
tenant_id=
agent_id=
@ -10,15 +11,16 @@ profile_id=
load_agent_details()
{
tenant_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Tenant ID|TenantID|g" | /etc/cp/bin/yq -P '.TenantID')
profile_id=$(cat /etc/cp/conf/agent_details.json | sed "s|Profile ID|ProfileID|g" | /etc/cp/bin/yq -P '.ProfileID')
tenant_id=$(awk -F\" '/Tenant ID/{print $4}' /etc/cp/conf/agent_details.json)
profile_id=$(awk -F\" '/Profile ID/{print $4}' /etc/cp/conf/agent_details.json)
agent_id=$(awk -F\" '/Agent ID/{print $4}' /etc/cp/conf/agent_details.json)
}
generate_policy()
{
cp -f /etc/cp/conf/local_policy.yaml /tmp/tmp_local_policy.yaml
sed -i "s|\"\*\"|\"Any\"|g" /tmp/tmp_local_policy.yaml
POLICY=$(/etc/cp/bin/yq /tmp/tmp_local_policy.yaml -o json)
POLICY=$(/etc/cp/bin/yq eval /tmp/tmp_local_policy.yaml -o json)
echo $POLICY > $POLICY_TEMP_PATH
rm -f /tmp/tmp_local_policy.yaml
}
@ -193,6 +195,7 @@ upload_policy_to_the_cloud()
fi
fi
upload_the_policy_to_s3
if [ "$?" = "1" ]; then
echo "Failed uploading the policy to S3"
@ -228,7 +231,8 @@ usage()
echo "Options:"
echo " --namespace <namespace> : Namespace with the relevant Helm Chart"
echo " --fog <fog address> : Namespace with the relevant Helm Chart"
echo " --upload_policy_only : Upload policy to the fog, withput changing agent mode"
echo " --config-upload : Upload policy to the fog"
echo " --config-upload-only : Upload policy to the fog, withput changing agent mode"
exit 255
}
@ -254,7 +258,10 @@ while true; do
validate_arg_value_exists "$1" "$#"
shift
var_fog="$1"
elif [ "$1" = "--upload_policy_only" ]; then
elif [ "$1" = "--config-upload" ]; then
UPLOAD_AGENT_POLICY=true
elif [ "$1" = "--config-upload-only" ]; then
UPLOAD_AGENT_POLICY=true
CHANGE_AGENT_MODE=false
source $DECLARATIVE_CONFIG_PATH
elif [ "$1" = "--access_token" ] || [ "$1" = "-at" ]; then
@ -271,6 +278,9 @@ while true; do
profile_id="$1"
elif [ -z "$1" ]; then
break
else
usage
exit 1
fi
shift
done
@ -279,7 +289,11 @@ if [ -z "$var_fog" ]; then
var_fog="https://inext-agents.cloud.ngen.checkpoint.com"
fi
upload_policy_to_the_cloud
if [ $UPLOAD_AGENT_POLICY = true ]; then
upload_policy_to_the_cloud
else
open-appsec-ctl --set-mode --online_mode --token $var_token --fog $var_fog
fi
if [ "$?" = "0" ]; then
echo "SUCCESS"
fi