github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-10-14 14:51:28 +00:00
parent b58f7781e6
commit c2ea2cda6d
89 changed files with 2545 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
components/security_apps/local_policy_mgmt_gen/access_control_practice.cc
View File

@@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in)
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<string>("name", practice_name, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode;
throw PolicyGenException("AppSec Access control practice mode invalid: " + mode);
}
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
parseMandatoryAppsecJSONKey<AccessControlRateLimit>("rateLimit", rate_limit, archive_in);
}

95
components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc
View File

@@ -438,19 +438,30 @@ WebAppSection::WebAppSection(
csrf_protection_mode("Disabled"),
open_redirect_mode("Disabled"),
error_disclosure_mode("Disabled"),
schema_validation_mode("Disabled"),
schema_validation_enforce_level("fullSchema"),
practice_advanced_config(parsed_appsec_spec),
anti_bots(parsed_appsec_spec.getAntiBot()),
trusted_sources({ parsed_trusted_sources })
{
auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) {
dbgWarning(D_LOCAL_POLICY)
<< "web attack mitigation severity invalid: "
<< mitigation_sevirity;
throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity);
} else {
web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity);
}
web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
web_attack_mitigation_severity =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
web_attack_mitigation_severity;
web_attack_mitigation_action =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
web_attack_mitigation_severity == "critical" ? "low" :
web_attack_mitigation_severity == "high" ? "balanced" :
web_attack_mitigation_severity == "medium" ? "high" :
web_attack_mitigation_severity == "Critical" ? "Low" :
web_attack_mitigation_severity == "High" ? "Balanced" :
web_attack_mitigation_severity == "Medium" ? "High" :
"Error";
triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
@@ -479,6 +490,9 @@ WebAppSection::WebAppSection(
const string &_web_attack_mitigation_severity,
const string &_web_attack_mitigation_mode,
const string &_bot_protection,
const string &_schema_validation_mode,
const string &_schema_validation_enforce_level,
const vector<string> &_schema_validation_oas,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
@@ -493,19 +507,29 @@ WebAppSection::WebAppSection(
practice_id(_practice_id),
practice_name(_practice_name),
context(_context),
web_attack_mitigation_severity(_web_attack_mitigation_severity),
web_attack_mitigation_mode(_web_attack_mitigation_mode),
bot_protection(_bot_protection),
schema_validation_mode(_schema_validation_mode),
schema_validation_enforce_level(_schema_validation_enforce_level),
schema_validation_oas(_schema_validation_oas),
practice_advanced_config(_practice_advanced_config),
anti_bots(_anti_bots),
trusted_sources({ parsed_trusted_sources })
{
if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) {
dbgWarning(D_LOCAL_POLICY)
<< "web attack mitigation severity invalid: "
<< _web_attack_mitigation_severity;
throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity);
} else {
web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity);
}
web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
web_attack_mitigation_action =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
web_attack_mitigation_severity == "critical" ? "low" :
web_attack_mitigation_severity == "high" ? "balanced" :
web_attack_mitigation_severity == "medium" ? "high" :
web_attack_mitigation_severity == "Critical" ? "Low" :
web_attack_mitigation_severity == "High" ? "Balanced" :
web_attack_mitigation_severity == "Medium" ? "High" :
"Error";
csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
@@ -516,6 +540,7 @@ WebAppSection::WebAppSection(
for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
overrides.push_back(AppSecOverride(source_ident));
}
}
// LCOV_EXCL_STOP
@@ -523,35 +548,35 @@ WebAppSection::WebAppSection(
void
WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
{
string disabled_str = "Disabled";
vector<string> empty_list;
out_ar(
cereal::make_nvp("context", context),
cereal::make_nvp("webAttackMitigation", web_attack_mitigation),
cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action),
cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode),
cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config),
cereal::make_nvp("csrfProtection", csrf_protection_mode),
cereal::make_nvp("openRedirect", open_redirect_mode),
cereal::make_nvp("errorDisclosure", error_disclosure_mode),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("assetName", asset_name),
cereal::make_nvp("ruleId", rule_id),
cereal::make_nvp("ruleName", rule_name),
cereal::make_nvp("schemaValidation", false),
cereal::make_nvp("schemaValidation_v2", disabled_str),
cereal::make_nvp("oas", empty_list),
cereal::make_nvp("triggers", triggers),
cereal::make_nvp("applicationUrls", application_urls),
cereal::make_nvp("overrides", overrides),
cereal::make_nvp("trustedSources", trusted_sources),
cereal::make_nvp("waapParameters", empty_list),
cereal::make_nvp("botProtection", false),
cereal::make_nvp("antiBot", anti_bots),
cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect"))
cereal::make_nvp("context", context),
cereal::make_nvp("webAttackMitigation", web_attack_mitigation),
cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action),
cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode),
cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config),
cereal::make_nvp("csrfProtection", csrf_protection_mode),
cereal::make_nvp("openRedirect", open_redirect_mode),
cereal::make_nvp("errorDisclosure", error_disclosure_mode),
cereal::make_nvp("practiceId", practice_id),
cereal::make_nvp("practiceName", practice_name),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("assetName", asset_name),
cereal::make_nvp("ruleId", rule_id),
cereal::make_nvp("ruleName", rule_name),
cereal::make_nvp("schemaValidation", schema_validation_mode == "Prevent"),
cereal::make_nvp("schemaValidation_v2", schema_validation_mode),
cereal::make_nvp("oas", schema_validation_oas),
cereal::make_nvp("schemaValidationEnforceLevel", schema_validation_enforce_level),
cereal::make_nvp("triggers", triggers),
cereal::make_nvp("applicationUrls", application_urls),
cereal::make_nvp("overrides", overrides),
cereal::make_nvp("trustedSources", trusted_sources),
cereal::make_nvp("waapParameters", empty_list),
cereal::make_nvp("botProtection", false),
cereal::make_nvp("antiBot", anti_bots),
cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect"))
);
}

50
components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@@ -291,6 +291,9 @@ public:
const std::string &_web_attack_mitigation_severity,
const std::string &_web_attack_mitigation_mode,
const std::string &_bot_protection,
const std::string &schema_validation_mode,
const std::string &schema_validation_enforce_level,
const std::vector<std::string> &schema_validation_oas,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
@@ -302,27 +305,30 @@ public:
bool operator< (const WebAppSection &other) const;
private:
std::string application_urls;
std::string asset_id;
std::string asset_name;
std::string rule_id;
std::string rule_name;
std::string practice_id;
std::string practice_name;
std::string context;
std::string web_attack_mitigation_action;
std::string web_attack_mitigation_severity;
std::string web_attack_mitigation_mode;
std::string csrf_protection_mode;
std::string open_redirect_mode;
std::string error_disclosure_mode;
std::string bot_protection;
bool web_attack_mitigation;
std::vector<TriggersInWaapSection> triggers;
PracticeAdvancedConfig practice_advanced_config;
AppsecPracticeAntiBotSection anti_bots;
std::vector<AppSecTrustedSources> trusted_sources;
std::vector<AppSecOverride> overrides;
bool web_attack_mitigation;
std::string application_urls;
std::string asset_id;
std::string asset_name;
std::string rule_id;
std::string rule_name;
std::string practice_id;
std::string practice_name;
std::string context;
std::string web_attack_mitigation_action;
std::string web_attack_mitigation_severity;
std::string web_attack_mitigation_mode;
std::string csrf_protection_mode;
std::string open_redirect_mode;
std::string error_disclosure_mode;
std::string bot_protection;
std::string schema_validation_mode;
std::string schema_validation_enforce_level;
std::vector<std::string> schema_validation_oas;
PracticeAdvancedConfig practice_advanced_config;
AppsecPracticeAntiBotSection anti_bots;
std::vector<AppSecOverride> overrides;
std::vector<AppSecTrustedSources> trusted_sources;
std::vector<TriggersInWaapSection> triggers;
};
class WebAPISection
@@ -410,7 +416,7 @@ class ParsedRule
{
public:
ParsedRule() {}
ParsedRule(const std::string &_host) : host(_host) {}
ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
void load(cereal::JSONInputArchive &archive_in);
const std::vector<std::string> & getExceptions() const;

5
components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h
View File

@@ -24,6 +24,7 @@
#include "maybe_res.h"
#include "i_orchestration_tools.h"
#include "i_shell_cmd.h"
#include "i_encryptor.h"
#include "i_messaging.h"
#include "i_env_details.h"
#include "i_agent_details.h"
@@ -40,6 +41,7 @@ class K8sPolicyUtils
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_EnvDetails>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_AgentDetails>
{
public:
@@ -80,6 +82,8 @@ private:
void createSnortFile(std::vector<NewAppSecPracticeSpec> &practices) const;
void createSchemaValidationOas(std::vector<NewAppSecPracticeSpec> &practices) const;
template<class T>
std::vector<T> extractV1Beta2ElementsFromCluster(
const std::string &crd_plural,
@@ -112,6 +116,7 @@ private:
I_Messaging* messaging = nullptr;
EnvType env_type;
std::string token;
std::string agent_ns;
};
#endif // __K8S_POLICY_UTILS_H__

17
components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h
View File

@@ -49,6 +49,13 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
{ "WebUserResponse", TriggerType::WebUserResponse }
};
static const std::unordered_map<std::string, std::string> key_to_mitigation_severity = {
{ "high", "High"},
{ "medium", "Medium"},
{ "critical", "Critical"},
{ "Transparent", "Transparent"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Learn"},
@@ -57,6 +64,14 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val =
{ "inactive", "Inactive"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Disabled"}
};
static const std::unordered_map<std::string, std::string> key_to_practices_val2 = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Learn"},
@@ -66,6 +81,8 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val2
};
static const std::string default_appsec_url = "http://*:*";
static const std::string default_appsec_name = "Any";
class PolicyGenException : public std::exception
{

2
components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h
View File

@@ -31,7 +31,7 @@ class NewParsedRule
{
public:
NewParsedRule() {}
NewParsedRule(const std::string &_host) : host(_host) {}
NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
void load(cereal::JSONInputArchive &archive_in);

50
components/security_apps/local_policy_mgmt_gen/include/new_practice.h
View File

@@ -23,6 +23,8 @@
#include "config.h"
#include "debug.h"
#include "local_policy_common.h"
#include "i_orchestration_tools.h"
#include "i_encryptor.h"
bool isModeInherited(const std::string &mode);
@@ -88,6 +90,8 @@ public:
void save(cereal::JSONOutputArchive &out_ar) const;
bool operator<(const IpsProtectionsSection &other) const;
private:
std::string context;
std::string name;
@@ -105,7 +109,7 @@ public:
// LCOV_EXCL_START Reason: no test exist
IPSSection() {};
IPSSection(const std::vector<IpsProtectionsSection> &_ips) : ips(_ips) {};
IPSSection(const std::vector<IpsProtectionsSection> &_ips);
// LCOV_EXCL_STOP
void save(cereal::JSONOutputArchive &out_ar) const;
@@ -138,6 +142,12 @@ public:
const std::string & getMode(const std::string &default_mode = "inactive") const;
private:
const std::string & getRulesMode(
const std::string &mode,
const std::string &default_mode = "inactive"
) const;
std::string override_mode;
std::string max_performance_impact;
std::string min_severity_level;
@@ -487,15 +497,16 @@ private:
SnortSection snort;
};
class NewSnortSignaturesAndOpenSchemaAPI
class NewSnortSignatures
{
public:
NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {};
NewSnortSignatures() : is_temporary(false) {};
void load(cereal::JSONInputArchive &archive_in);
void addFile(const std::string &file_name);
const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
const std::string & getEnforceLevel() const;
const std::vector<std::string> & getConfigMap() const;
const std::vector<std::string> & getFiles() const;
bool isTemporary() const;
@@ -503,17 +514,40 @@ public:
private:
std::string override_mode;
std::string enforcement_level;
std::vector<std::string> config_map;
std::vector<std::string> files;
bool is_temporary;
};
class NewOpenApiSchema : Singleton::Consume<I_OrchestrationTools>, Singleton::Consume<I_Encryptor>
{
public:
NewOpenApiSchema() {};
void load(cereal::JSONInputArchive &archive_in);
void addOas(const std::string &file);
const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
const std::string & getEnforceLevel() const;
const std::vector<std::string> & getConfigMap() const;
const std::vector<std::string> & getFiles() const;
const std::vector<std::string> & getOas() const;
private:
std::string override_mode;
std::string enforcement_level;
std::vector<std::string> config_map;
std::vector<std::string> files;
std::vector<std::string> oas;
};
class NewAppSecPracticeAntiBot
{
public:
const std::vector<std::string> & getIjectedUris() const;
const std::vector<std::string> & getValidatedUris() const;
const std::string & getMode() const;
const std::string & getMode(const std::string &default_mode = "inactive") const;
void load(cereal::JSONInputArchive &archive_in);
void save(cereal::JSONOutputArchive &out_ar) const;
@@ -569,8 +603,8 @@ class NewAppSecPracticeSpec
public:
void load(cereal::JSONInputArchive &archive_in);
NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures();
const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const;
NewSnortSignatures & getSnortSignatures();
NewOpenApiSchema & getOpenSchemaValidation();
const NewAppSecPracticeWebAttacks & getWebAttacks() const;
const NewAppSecPracticeAntiBot & getAntiBot() const;
const NewIntrusionPrevention & getIntrusionPrevention() const;
@@ -583,8 +617,8 @@ public:
private:
NewFileSecurity file_security;
NewIntrusionPrevention intrusion_prevention;
NewSnortSignaturesAndOpenSchemaAPI openapi_schema_validation;
NewSnortSignaturesAndOpenSchemaAPI snort_signatures;
NewOpenApiSchema openapi_schema_validation;
NewSnortSignatures snort_signatures;
NewAppSecPracticeWebAttacks web_attacks;
NewAppSecPracticeAntiBot anti_bot;
std::string appsec_class_name;

2
components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h
View File

@@ -123,6 +123,7 @@ public:
);
const std::string & getIdentifier() const;
const std::string & getIdentifierValue() const;
void save(cereal::JSONOutputArchive &out_ar) const;
@@ -145,6 +146,7 @@ public:
);
const std::string & getIdentifier() const;
const std::string & getIdentifierValue() const;
void save(cereal::JSONOutputArchive &out_ar) const;

87
components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc
View File

@@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key)
}
}
string
getAppSecScopeType()
{
auto env_res = getenv("CRDS_SCOPE");
if (env_res != nullptr) return env_res;
return "cluster";
}
void
K8sPolicyUtils::init()
{
@@ -42,6 +50,7 @@ K8sPolicyUtils::init()
env_type = env_details->getEnvType();
if (env_type == EnvType::K8S) {
token = env_details->getToken();
agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : "";
messaging = Singleton::Consume<I_Messaging>::by<K8sPolicyUtils>();
}
}
@@ -140,10 +149,12 @@ extractElementsFromNewRule(
const NewParsedRule &rule,
map<AnnotationTypes, unordered_set<string>> &policy_elements_names)
{
policy_elements_names[AnnotationTypes::EXCEPTION].insert(
rule.getExceptions().begin(),
rule.getExceptions().end()
);
if (rule.getExceptions().size() > 0) {
policy_elements_names[AnnotationTypes::EXCEPTION].insert(
rule.getExceptions().begin(),
rule.getExceptions().end()
);
}
policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert(
rule.getPractices().begin(),
rule.getPractices().end()
@@ -152,14 +163,24 @@ extractElementsFromNewRule(
rule.getAccessControlPractices().begin(),
rule.getAccessControlPractices().end()
);
policy_elements_names[AnnotationTypes::TRIGGER].insert(
rule.getLogTriggers().begin(),
rule.getLogTriggers().end()
);
policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
if (rule.getLogTriggers().size() > 0) {
policy_elements_names[AnnotationTypes::TRIGGER].insert(
rule.getLogTriggers().begin(),
rule.getLogTriggers().end()
);
}
if (rule.getCustomResponse() != "" ) {
policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
}
if (rule.getSourceIdentifiers() != "" ) {
policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
}
if (rule.getTrustedSources() != "" ) {
policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
}
if (rule.getUpgradeSettings() != "" ) {
policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
}
}
map<AnnotationTypes, unordered_set<string>>
@@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster(
dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural;
vector<T> elements;
for (const string &element_name : elements_names) {
string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name;
auto maybe_appsec_element = getObjectFromCluster<AppsecSpecParser<T>>(
"/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name
"/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name
);
if (!maybe_appsec_element.ok()) {
@@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
practice.getSnortSignatures().setTemporary(true);
for (const string &config_map : practice.getSnortSignatures().getConfigMap())
{
string ns = agent_ns == "" ? "default/" : agent_ns;
auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
"/api/v1/namespaces/default/configmaps/" + config_map
"/api/v1/namespaces/" + ns + "configmaps/" + config_map
);
if (!maybe_configmap.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
@@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
}
}
void
K8sPolicyUtils::createSchemaValidationOas(vector<NewAppSecPracticeSpec> &practices) const
{
for (NewAppSecPracticeSpec &practice : practices) {
vector<string> res;
for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap())
{
string ns = agent_ns == "" ? "default/" : agent_ns;
auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
"/api/v1/namespaces/" + ns + "configmaps/" + config_map
);
if (!maybe_configmap.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
continue;
}
string file_content = maybe_configmap.unpack().getFileContent();
string res = Singleton::Consume<I_Encryptor>::by<K8sPolicyUtils>()->base64Encode(file_content);
practice.getOpenSchemaValidation().addOas(res);
}
}
}
Maybe<V1beta2AppsecLinuxPolicy>
K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
const AppsecSpecParser<NewAppsecPolicySpec> &appsec_policy_spec,
@@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
}
if (default_rule.getMode().empty() && !ingress_mode.empty()) {
dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode;
default_rule.setMode(ingress_mode);
}
@@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
);
createSnortFile(threat_prevention_practices);
createSchemaValidationOas(threat_prevention_practices);
vector<AccessControlPracticeSpec> access_control_practices =
extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
@@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
dbgWarning(D_LOCAL_POLICY
) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
"/apis/openappsec.io/v1beta2/policies/" + policy_name
"/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name
);
if (!maybe_v1beta2_appsec_policy_spec.ok()) {
dbgWarning(D_LOCAL_POLICY)
<< "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
@@ -535,10 +586,11 @@ K8sPolicyUtils::createPolicy(
if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) {
policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
}
auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
if (item.getSpec().doesDefaultBackendExist()) {
dbgTrace(D_LOCAL_POLICY)
<< "Inserting Any host rule to the specific asset set";
K ingress_rule = K("*");
K ingress_rule = K("*", default_mode);
policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
}
@@ -556,12 +608,11 @@ K8sPolicyUtils::createPolicy(
<< "' uri: '"
<< uri.getPath()
<< "'";
K ingress_rule = K(host);
K ingress_rule = K(host, default_mode);
policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
}
}
}
}
std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>

259
components/security_apps/local_policy_mgmt_gen/new_practice.cc
View File

@@ -22,6 +22,7 @@ static const set<string> performance_impacts = {"low", "medium", "high"};
static const set<string> severity_levels = {"low", "medium", "high", "critical"};
static const set<string> size_unit = {"bytes", "KB", "MB", "GB"};
static const set<string> confidences_actions = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
static const set<string> valied_enforcement_level = {"fullSchema", "endpointOnly"};
static const set<string> valid_modes = {
"prevent",
"detect",
@@ -32,38 +33,38 @@ static const set<string> valid_modes = {
"inherited"
};
static const set<string> valid_confidences = {"medium", "high", "critical"};
static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
static const unordered_map<string, string> key_to_performance_impact_val = {
{ "low", "Low or lower"},
{ "medium", "Medium or lower"},
{ "high", "High or lower"}
};
static const std::unordered_map<std::string, std::string> key_to_severity_level_val = {
static const unordered_map<string, string> key_to_severity_level_val = {
{ "low", "Low or above"},
{ "medium", "Medium or above"},
{ "high", "High or above"},
{ "critical", "Critical"}
};
static const std::unordered_map<std::string, std::string> key_to_mode_val = {
static const unordered_map<string, string> key_to_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Inactive"}
};
static const std::unordered_map<std::string, std::string> anti_bot_key_to_mode_val = {
static const unordered_map<string, string> anti_bot_key_to_mode_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Disabled"}
};
static const std::unordered_map<std::string, uint64_t> unit_to_int = {
static const unordered_map<string, uint64_t> unit_to_int = {
{ "bytes", 1},
{ "KB", 1024},
{ "MB", 1048576},
{ "GB", 1073741824}
};
static const std::string TRANSPARENT_MODE = "Transparent";
static const string TRANSPARENT_MODE = "Transparent";
bool
isModeInherited(const string &mode)
@@ -71,11 +72,11 @@ isModeInherited(const string &mode)
return mode == "as-top-level" || mode == "inherited";
}
const std::string &
const string &
getModeWithDefault(
const std::string &mode,
const std::string &default_mode,
const std::unordered_map<std::string, std::string> &key_to_val)
const string &mode,
const string &default_mode,
const unordered_map<string, string> &key_to_val)
{
if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
@@ -88,36 +89,35 @@ getModeWithDefault(
return key_to_val.at(mode);
}
const std::vector<std::string> &
const vector<string> &
NewAppSecPracticeAntiBot::getIjectedUris() const
{
return injected_uris;
}
const std::vector<std::string> &
const vector<string> &
NewAppSecPracticeAntiBot::getValidatedUris() const
{
return validated_uris;
}
const std::string &
NewAppSecPracticeAntiBot::getMode() const
const string &
NewAppSecPracticeAntiBot::getMode(const string &default_mode) const
{
return override_mode;
return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val);
}
void
NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
string mode;
parseAppsecJSONKey<vector<string>>("injectedUris", injected_uris, archive_in);
parseAppsecJSONKey<vector<string>>("validatedUris", validated_uris, archive_in);
parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode;
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode);
}
override_mode = anti_bot_key_to_mode_val.at(mode);
}
void
@@ -242,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const
}
SnortProtectionsSection::SnortProtectionsSection(
const std::string &_context,
const std::string &_asset_name,
const std::string &_asset_id,
const std::string &_practice_name,
const std::string &_practice_id,
const std::string &_source_identifier,
const std::string &_mode,
const std::vector<std::string> &_files)
const string &_context,
const string &_asset_name,
const string &_asset_id,
const string &_practice_name,
const string &_practice_id,
const string &_source_identifier,
const string &_mode,
const vector<string> &_files)
:
context(_context),
asset_name(_asset_name),
@@ -278,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
}
DetectionRules::DetectionRules(
const std::string &_type,
const std::string &_SSM,
const std::string &_keywords,
const std::vector<std::string> &_context)
const string &_type,
const string &_SSM,
const string &_keywords,
const vector<string> &_context)
:
type(_type),
SSM(_SSM),
@@ -314,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const
ProtectionMetadata::ProtectionMetadata(
bool _silent,
const std::string &_protection_name,
const std::string &_severity,
const std::string &_confidence_level,
const std::string &_performance_impact,
const std::string &_last_update,
const std::string &_maintrain_id,
const std::vector<std::string> &_tags,
const std::vector<std::string> &_cve_list)
const string &_protection_name,
const string &_severity,
const string &_confidence_level,
const string &_performance_impact,
const string &_last_update,
const string &_maintrain_id,
const vector<string> &_tags,
const vector<string> &_cve_list)
:
silent(_silent),
protection_name(_protection_name),
@@ -394,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
}
ProtectionsSection::ProtectionsSection(
const std::vector<ProtectionsProtectionsSection> &_protections,
const std::string &_name,
const std::string &_modification_time)
const vector<ProtectionsProtectionsSection> &_protections,
const string &_name,
const string &_modification_time)
:
protections(_protections),
name(_name),
@@ -460,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const
}
void
NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
NewSnortSignatures::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseAppsecJSONKey<vector<string>>("files", files, archive_in);
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode);
}
is_temporary = false;
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
@@ -474,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
}
void
NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
NewSnortSignatures::addFile(const string &file_name)
{
files.push_back(file_name);
}
const string &
NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
NewSnortSignatures::getOverrideMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
return res;
}
const vector<string> &
NewSnortSignaturesAndOpenSchemaAPI::getFiles() const
NewSnortSignatures::getFiles() const
{
return files;
}
const vector<string> &
NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const
NewSnortSignatures::getConfigMap() const
{
return config_map;
}
bool
NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const
NewSnortSignatures::isTemporary() const
{
return is_temporary;
}
void
NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val)
NewSnortSignatures::setTemporary(bool val)
{
is_temporary = val;
}
void
NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice";
parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
parseAppsecJSONKey<vector<string>>("files", files, archive_in);
parseAppsecJSONKey<string>("enforcementLevel", enforcement_level, archive_in, "fullSchema");
if (valied_enforcement_level.count(enforcement_level) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level;
throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level);
}
if (valid_modes.count(override_mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode;
throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode);
}
for (const string &file : files)
{
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<NewOpenApiSchema>();
auto file_content = i_orchestration_tools->readFile(file);
if (!file_content.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file";
continue;
}
oas.push_back(Singleton::Consume<I_Encryptor>::by<NewOpenApiSchema>()->base64Encode(file_content.unpack()));
}
}
void
NewOpenApiSchema::addOas(const string &file)
{
oas.push_back(file);
}
const string &
NewOpenApiSchema::getOverrideMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2);
return res;
}
const string &
NewOpenApiSchema::getEnforceLevel() const
{
return enforcement_level;
}
const vector<string> &
NewOpenApiSchema::getFiles() const
{
return files;
}
const vector<string> &
NewOpenApiSchema::getConfigMap() const
{
return config_map;
}
const vector<string> &
NewOpenApiSchema::getOas() const
{
return oas;
}
void
IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
{
@@ -548,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection(
{
}
std::string &
string &
IpsProtectionsSection::getMode()
{
return mode;
@@ -570,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
);
}
bool
IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const
{
// for sorting from the most specific to the least specific rule
if (name == default_appsec_name) return false;
if (other.name == default_appsec_name) return true;
return name.size() > other.name.size();
}
IPSSection::IPSSection(const vector<IpsProtectionsSection> &_ips) : ips(_ips)
{
sort(ips.begin(), ips.end());
}
void
IPSSection::save(cereal::JSONOutputArchive &out_ar) const
{
@@ -648,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
vector<IpsProtectionsRulesSection> ips_rules;
IpsProtectionsRulesSection high_rule(
min_cve_Year,
getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(high_confidence_event_action, default_mode),
string("High"),
max_performance_impact,
string(""),
@@ -658,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
IpsProtectionsRulesSection med_rule(
min_cve_Year,
getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(medium_confidence_event_action, default_mode),
string("Medium"),
max_performance_impact,
string(""),
@@ -668,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
IpsProtectionsRulesSection low_rule(
min_cve_Year,
getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
getRulesMode(low_confidence_event_action, default_mode),
string("Low"),
max_performance_impact,
string(""),
@@ -679,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
return ips_rules;
}
const std::string &
NewIntrusionPrevention::getMode(const std::string &default_mode) const
const string &
NewIntrusionPrevention::getMode(const string &default_mode) const
{
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
return res;
}
const string &
NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const
{
if (isModeInherited(mode)) return default_mode;
if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) {
dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
return key_to_practices_mode_val.at("inactive");
}
return key_to_practices_mode_val.at(mode);
}
FileSecurityProtectionsSection::FileSecurityProtectionsSection(
uint64_t _file_size_limit,
uint64_t _archive_file_size_limit,
bool _allow_files_without_name,
bool _required_file_size_limit,
bool _required_archive_extraction,
const std::string &_context,
const std::string &_name,
const std::string &_asset_id,
const std::string &_practice_name,
const std::string &_practice_id,
const std::string &_action,
const std::string &_files_without_name_action,
const std::string &_high_confidence_action,
const std::string &_medium_confidence_action,
const std::string &_low_confidence_action,
const std::string &_severity_level,
const std::string &_file_size_limit_action,
const std::string &_multi_level_archive_action,
const std::string &_unopened_archive_action)
const string &_context,
const string &_name,
const string &_asset_id,
const string &_practice_name,
const string &_practice_id,
const string &_action,
const string &_files_without_name_action,
const string &_high_confidence_action,
const string &_medium_confidence_action,
const string &_low_confidence_action,
const string &_severity_level,
const string &_file_size_limit_action,
const string &_multi_level_archive_action,
const string &_unopened_archive_action)
:
file_size_limit(_file_size_limit),
archive_file_size_limit(_archive_file_size_limit),
@@ -831,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const
return extract_archive_files;
}
const std::string &
const string &
NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const
{
return archived_files_within_archived_files;
}
const std::string &
const string &
NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const
{
return archived_files_where_content_extraction_failed;
@@ -886,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const
return (file_size_limit * unit_to_int.at(file_size_limit_unit));
}
const std::string &
const string &
NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const
{
return files_exceeding_size_limit_action;
@@ -1007,7 +1102,7 @@ void
NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
parseAppsecJSONKey<NewOpenApiSchema>(
"schemaValidation",
openapi_schema_validation,
archive_in
@@ -1015,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewSnortSignatures>("snortSignatures", snort_signatures, archive_in);
parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
parseAppsecJSONKey<string>("name", practice_name, archive_in);
parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
if (valid_modes.count(mode) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode;
throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode);
}
}
void
@@ -1028,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name)
practice_name = _name;
}
const NewSnortSignaturesAndOpenSchemaAPI &
NewAppSecPracticeSpec::getOpenSchemaValidation() const
NewOpenApiSchema &
NewAppSecPracticeSpec::getOpenSchemaValidation()
{
return openapi_schema_validation;
}
NewSnortSignaturesAndOpenSchemaAPI &
NewSnortSignatures &
NewAppSecPracticeSpec::getSnortSignatures()
{
return snort_signatures;

47
components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc
View File

@@ -23,6 +23,14 @@ using namespace std;
USE_DEBUG_FLAG(D_NGINX_POLICY);
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const std::unordered_map<std::string, std::string> key_to_source_identefier_val = {
{ "sourceip", "Source IP"},
{ "cookie", "Cookie:"},
{ "headerkey", "Header:"},
{ "JWTKey", ""},
{ "x-forwarded-for", "X-Forwarded-For"}
};
void
SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const
{
@@ -1038,7 +1046,7 @@ PolicyMakerUtils::createIpsSections(
practice_name,
practice_id,
source_identifier,
override_mode,
"Inactive",
apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
);
@@ -1048,8 +1056,7 @@ PolicyMakerUtils::createIpsSections(
void
PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary)
{
auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name;
string in_file = is_temporary ? path + ".rule" : path;
auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name;
if (snort_protections.find(path) != snort_protections.end()) {
dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists";
@@ -1060,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
<< (is_temporary ? " temporary" : "") << " file " << path;
auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err";
auto tmp_out = "/tmp/" + file_name + ".out";
auto tmp_err = "/tmp/" + file_name + ".err";
auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err;
auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
@@ -1069,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
return;
}
Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(path + ".out");
Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(tmp_out);
if (!maybe_protections.ok()){
dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr();
return;
}
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
if (is_temporary) i_orchestration_tools->removeFile(in_file);
i_orchestration_tools->removeFile(path + ".out");
i_orchestration_tools->removeFile(path + ".err");
if (is_temporary) i_orchestration_tools->removeFile(path);
i_orchestration_tools->removeFile(tmp_out);
i_orchestration_tools->removeFile(tmp_err);
snort_protections[path] = ProtectionsSection(
maybe_protections.unpack().getProtections(),
@@ -1208,7 +1217,8 @@ void
PolicyMakerUtils::createWebAppSection(
const V1beta2AppsecLinuxPolicy &policy,
const RulesConfigRulebase& rule_config,
const string &practice_id, const string &full_url,
const string &practice_id,
const string &full_url,
const string &default_mode,
map<AnnotationTypes, string> &rule_annotations)
{
@@ -1225,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection(
apssec_practice.getWebAttacks().getMaxObjectDepth(),
apssec_practice.getWebAttacks().getMaxUrlSizeBytes()
);
WebAppSection web_app = WebAppSection(
full_url == "Any" ? default_appsec_url : full_url,
rule_config.getAssetId(),
@@ -1236,7 +1247,10 @@ PolicyMakerUtils::createWebAppSection(
rule_config.getContext(),
apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode),
apssec_practice.getWebAttacks().getMode(practice_mode),
apssec_practice.getAntiBot().getMode(),
apssec_practice.getAntiBot().getMode(practice_mode),
apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode),
apssec_practice.getOpenSchemaValidation().getEnforceLevel(),
apssec_practice.getOpenSchemaValidation().getOas(),
practice_advance_config,
apssec_practice.getAntiBot(),
log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
@@ -1290,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
);
rules_config[rule_config.getAssetName()] = rule_config;
string current_identifier;
string current_identifier, current_identifier_value;
if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) {
UsersIdentifiersRulebase user_identifiers = createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS],
@@ -1299,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
);
users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers;
current_identifier = user_identifiers.getIdentifier();
current_identifier_value = user_identifiers.getIdentifierValue();
}
string ips_identifier, ips_identifier_value;
if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) {
ips_identifier = key_to_source_identefier_val.at(current_identifier);
}
if (current_identifier == "cookie" || current_identifier == "headerkey") {
ips_identifier_value = current_identifier_value;
}
createIpsSections(
@@ -1306,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
rule_config.getAssetName(),
practice_id,
rule_annotations[AnnotationTypes::PRACTICE],
current_identifier,
ips_identifier + ips_identifier_value,
rule_config.getContext(),
policy,
rule_annotations,

16
components/security_apps/local_policy_mgmt_gen/rules_config_section.cc
View File

@@ -17,6 +17,8 @@ using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static const string empty_string="";
AssetUrlParser
AssetUrlParser::parse(const string &uri)
{
@@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const
{
return source_identifier;
}
const string &
UsersIdentifier::getIdentifierValue() const
{
if (identifier_values.empty()) return empty_string;
return identifier_values[0];
}
// LCOV_EXCL_STOP
void
@@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const
if (source_identifiers.empty()) return source_identifier;
return source_identifiers[0].getIdentifier();
}
const string &
UsersIdentifiersRulebase::getIdentifierValue() const
{
if (source_identifiers.empty()) return empty_string;
return source_identifiers[0].getIdentifierValue();
}
// LCOV_EXCL_STOP
void

7
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@@ -66,7 +66,12 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
getQUID)
SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''",
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(

13
components/security_apps/orchestration/env_details/env_details.cc
View File

@@ -28,6 +28,7 @@ EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
token = retrieveToken();
agent_namespace = retrieveNamespace();
if (!token.empty()) {
auto env_res = getenv("deployment_type");
env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
@@ -46,12 +47,24 @@ EnvDetails::getToken()
return token;
}
string
EnvDetails::getNameSpace()
{
return agent_namespace;
}
string
EnvDetails::retrieveToken()
{
return readFileContent(k8s_service_account + "/token");
}
string
EnvDetails::retrieveNamespace()
{
return readFileContent(k8s_service_account + "/namespace");
}
string
EnvDetails::readFileContent(const string &file_path)
{

9
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -1485,11 +1485,10 @@ private:
}
void
setUpgradeTime()
setDelayedUpgradeTime()
{
if (getConfigurationFlag("service_startup") != "true") return;
if (i_service_controller->getServiceToPortMap().empty()) return;
if (!i_agent_details->isOpenAppsecAgent() && i_service_controller->getServiceToPortMap().empty()) return;
try {
string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN");
int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30;
@@ -1506,6 +1505,7 @@ private:
void
run()
{
loadExistingPolicy();
sleep_interval = policy.getErrorSleepInterval();
Maybe<void> registration_status(genError("Not running yet."));
while (!(registration_status = registerToTheFog()).ok()) {
@@ -1530,7 +1530,6 @@ private:
<< " seconds";
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(seconds(sleep_interval));
}
loadExistingPolicy();
failure_count = 0;
Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(chrono::seconds(1));
@@ -1587,7 +1586,7 @@ private:
).notify();
}
setUpgradeTime();
setDelayedUpgradeTime();
while (true) {
Singleton::Consume<I_Environment>::by<OrchestrationComp>()->startNewTrace(false);
if (shouldReportAgentDetailsMetadata()) {

2
components/security_apps/waap/include/i_transaction.h
View File

@@ -70,6 +70,7 @@ public:
virtual const std::string getParam() const = 0;
virtual const std::vector<std::string> getKeywordMatches() const = 0;
virtual const std::vector<std::string> getKeywordsCombinations() const = 0;
virtual const std::vector<std::string> getKeywordsAfterFilter() const = 0;
virtual const std::string getContentTypeStr() const = 0;
virtual Waap::Util::ContentType getContentType() const = 0;
virtual const std::string getKeywordMatchesStr() const = 0;
@@ -84,6 +85,7 @@ public:
virtual const std::string getUriStr() const = 0;
virtual const std::string& getSourceIdentifier() const = 0;
virtual double getScore() const = 0;
virtual double getOtherModelScore() const = 0;
virtual const std::vector<double> getScoreArray() const = 0;
virtual Waap::CSRF::State& getCsrfState() = 0;
virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;

96
components/security_apps/waap/waap_clib/CidrMatch.cc
View File

@@ -25,6 +25,51 @@
#include "log_generator.h"
#include <stdexcept>
static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) {
in_addr maskedAddr;
if (prefixLength == 0) {
maskedAddr.s_addr = 0;
} else {
uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1)); // Create mask
maskedAddr.s_addr = addr.s_addr & mask; // Apply mask
}
return maskedAddr;
}
// Function to apply a network mask to an IPv6 address
static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) {
in6_addr maskedAddr = addr;
int fullBytes = prefixLength / 8;
int remainingBits = prefixLength % 8;
// Mask full bytes
for (int i = fullBytes; i < 16; ++i) {
maskedAddr.s6_addr[i] = 0;
}
// Mask remaining bits
if (remainingBits > 0) {
uint8_t mask = ~((1 << (8 - remainingBits)) - 1);
maskedAddr.s6_addr[fullBytes] &= mask;
}
return maskedAddr;
}
// Helper function to convert an IPv4 address to string
static std::string ipv4ToString(const in_addr& ipv4) {
char str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN);
return std::string(str);
}
// Helper function to convert an IPv6 address to string
static std::string ipv6ToString(const in6_addr& ipv6) {
char str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN);
return std::string(str);
}
USE_DEBUG_FLAG(D_WAAP);
namespace Waap {
namespace Util {
@@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const {
isIPV6 == other.isIPV6;
}
bool CIDRData::operator<(const CIDRData &other) const {
if (isIPV6) {
if (!other.isIPV6) return false;
return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0;
}
if (other.isIPV6) return true;
return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr);
}
bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) {
if (bits == 0) {
// C99 6.5.7 (3): u32 << 32 is undefined behaviour
@@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr));
if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) {
cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits);
cidr.isIPV6 = false;
}
else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) {
cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits);
cidr.isIPV6 = true;
}
else
@@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
return true;
}
bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
CIDRData cidrData;
@@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
return cidrMatch(sourceip, cidrData);
}
bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
struct in_addr source_inaddr;
struct in6_addr source_inaddr6;
@@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions.";
return false;
}
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) {
if (first.isIPV6 != second.isIPV6) return false; // IPv4 and IPv6 cannot overlap
if (first.networkBits >= second.networkBits) return false;
if (!first.isIPV6) {
// IPv4 containment check
in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits);
return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr);
}
// IPv6 containment check
in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits);
for (int i = 0; i < 16; ++i) {
if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) {
return false;
}
}
return true;
}
std::string cidrsToString(const std::vector<CIDRData>& cidrs) {
std::stringstream ss;
bool is_first = true;
ss << "[";
for (const auto& cidr : cidrs) {
if (!is_first) ss << ", ";
if (cidr.isIPV6) {
ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast<int>(cidr.networkBits);
} else {
ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast<int>(cidr.networkBits);
}
is_first = false;
}
ss << "]";
return ss.str();
}
}
}

4
components/security_apps/waap/waap_clib/CidrMatch.h
View File

@@ -18,6 +18,7 @@
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <vector>
namespace Waap {
namespace Util {
@@ -29,11 +30,14 @@ struct CIDRData {
uint8_t networkBits;
bool isIPV6;
bool operator==(const CIDRData &other) const;
bool operator<(const CIDRData &other) const;
};
bool isCIDR(const std::string& strCIDR, CIDRData& cidr);
bool cidrMatch(const std::string& sourceip, const CIDRData& cidr);
bool cidrMatch(const std::string &sourceip, const std::string &target);
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second);
std::string cidrsToString(const std::vector<CIDRData>& cidrs);
}
}

7
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -1308,7 +1308,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
} else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)";
@@ -1323,7 +1323,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
offset = 0;
return offset;
@@ -1545,5 +1545,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const
static const std::string PDF_header("%PDF-");
if (cur_val.size() < 10)
return false;
return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size())
.find(PDF_header) != std::string::npos;
}

3
components/security_apps/waap/waap_clib/ParserBinaryFile.cc
View File

@@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len)
}
} else {
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
c = strstr(buf + len - tail.size(), tail.c_str());
size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, tail.c_str());
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
if (c) {
m_state = s_end;

2
components/security_apps/waap/waap_clib/ParserBinaryFile.h
View File

@@ -20,7 +20,7 @@
#define MIN_HEADER_LOOKUP 16
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_TAIL_LOOKUP 20
class ParserBinaryFile : public ParserBase {
public:

26
components/security_apps/waap/waap_clib/ParserPDF.cc
View File

@@ -38,9 +38,10 @@ size_t
ParserPDF::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PDF)
<< "buf="
<< buf
<< "len="
<< "buf='"
<< std::string(buf, std::min((size_t)200, len))
<< (len > 200 ? "..." : "")
<< "' len="
<< len;
const char *c;
@@ -65,13 +66,18 @@ ParserPDF::push(const char *buf, size_t len)
m_state = s_body;
CP_FALL_THROUGH;
case s_body:
c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
{
size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF)
<< "string to search: " << std::string(buf + tail_lookup_offset)
<< " c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
}
}
case s_end:
if (m_receiver.onKey("PDF", 3) != 0) {

4
components/security_apps/waap/waap_clib/ParserPDF.h
View File

@@ -17,8 +17,8 @@
#include "ParserBase.h"
#include <string.h>
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_PDF_HEADER_LOOKUP 64
#define MAX_PDF_TAIL_LOOKUP 20
class ParserPDF : public ParserBase {
public:

4
components/security_apps/waap/waap_clib/ScanResult.cc
View File

@@ -23,9 +23,11 @@ unescaped_line(),
param_name(),
location(),
score(0.0f),
other_model_score(0.0f),
scoreNoFilter(0.0f),
scoreArray(),
keywordCombinations(),
keywordsAfterFilter(),
attack_types(),
m_isAttackInParam(false)
{
@@ -41,9 +43,11 @@ void Waf2ScanResult::clear()
param_name.clear();
location.clear();
score = 0;
other_model_score = 0;
scoreNoFilter = 0;
scoreArray.clear();
keywordCombinations.clear();
keywordsAfterFilter.clear();
attack_types.clear();
}

3
components/security_apps/waap/waap_clib/ScanResult.h
View File

@@ -29,9 +29,12 @@ struct Waf2ScanResult {
std::string param_name;
std::string location;
double score;
double other_model_score;
double scoreNoFilter;
std::vector<double> scoreArray;
std::vector<double> coefArray;
std::vector<std::string> keywordCombinations;
std::vector<std::string> keywordsAfterFilter;
std::set<std::string> attack_types;
bool m_isAttackInParam;
void clear(); // clear Waf2ScanResult

75
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
m_keywordsDataMap[it->first] = it->second;
}
}
m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept;
m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef;
m_stats.isLinModel = baseScores.m_stats.isLinModel;
}
@@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
m_pWaapAssetState(pWaapAssetState)
{
restore();
// merge
mergeScores(baseScores);
}
@@ -352,16 +355,42 @@ void ScoreBuilder::snap()
const std::string &poolName = pool.first;
const KeywordsScorePool& keywordScorePool = pool.second;
m_snapshotKwScoreMap[poolName];
m_snapshotKwCoefMap[poolName];
if (keywordScorePool.m_keywordsDataMap.empty()) {
m_snapshotStatsMap[poolName] = KeywordsStats();
} else {
m_snapshotStatsMap[poolName] = keywordScorePool.m_stats;
}
for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
{
const std::string &kwName = kwData.first;
double kwScore = kwData.second.score;
m_snapshotKwScoreMap[poolName][kwName] = kwScore;
if (keywordScorePool.m_stats.isLinModel) {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score;
} else {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score;
}
m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef;
}
}
}
KeywordsStats
ScoreBuilder::getSnapshotStats(const std::string &poolName) const
{
std::map<std::string, KeywordsStats>::const_iterator poolIt = m_snapshotStatsMap.find(poolName);
if (poolIt == m_snapshotStatsMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool";
poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE);
if (poolIt == m_snapshotStatsMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning empty stats";
return KeywordsStats();
}
}
return poolIt->second;
}
double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore,
const std::string &poolName) const
{
@@ -369,12 +398,11 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool";
poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
}
}
const KeywordScoreMap &kwScoreMap = poolIt->second;
@@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
return kwScoreFound->second;
}
double
ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const
{
std::map<std::string, KeywordScoreMap>::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName);
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool";
poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef;
return defaultCoef;
}
const KeywordScoreMap &kwCoefMap = poolIt->second;
KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword);
if (kwCoefFound == kwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef <<
" (default, keyword not found in pool '" << poolName << "')";
return defaultCoef;
}
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" <<
poolName << "')";
return kwCoefFound->second;
}
keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip)
{
return m_fpStore.ipItems[ip];

40
components/security_apps/waap/waap_clib/ScoreBuilder.h
View File

@@ -26,6 +26,8 @@
#include <cereal/types/string.hpp>
#include "WaapDefines.h"
USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER);
struct ScoreBuilderData {
std::string m_sourceIdentifier;
std::string m_userAgent;
@@ -52,11 +54,15 @@ enum KeywordType {
};
struct KeywordData {
KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {}
KeywordData() :
truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN)
{}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double score;
double coef;
double new_score;
KeywordType type;
template <class Archive>
@@ -65,20 +71,42 @@ struct KeywordData {
cereal::make_nvp("true_positives", truePositiveCtr),
cereal::make_nvp("score", score),
cereal::make_nvp("type", type));
try {
ar(cereal::make_nvp("coef", coef),
cereal::make_nvp("new_score", new_score));
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
coef = 0;
new_score = 0;
}
}
};
struct KeywordsStats {
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {}
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0)
{}
template <class Archive>
void serialize(Archive& ar) {
ar(cereal::make_nvp("false_positives", falsePositiveCtr),
cereal::make_nvp("true_positives", truePositiveCtr));
cereal::make_nvp("true_positives", truePositiveCtr));
try {
ar(cereal::make_nvp("intercept", linModelIntercept),
cereal::make_nvp("log_nnz_coef", linModelNNZCoef));
isLinModel = true;
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
linModelIntercept = 0;
linModelNNZCoef = 0;
isLinModel = false;
}
}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double linModelIntercept;
double linModelNNZCoef;
bool isLinModel;
};
typedef std::unordered_set<std::string> keywords_set;
@@ -106,6 +134,7 @@ struct KeywordsScorePool {
KeywordsScorePool();
// LCOV_EXCL_START Reason: no test exist
template <typename _A>
KeywordsScorePool(_A &iarchive)
{
@@ -120,6 +149,7 @@ struct KeywordsScorePool {
m_keywordsDataMap[key] = item.second;
}
}
// LCOV_EXCL_STOP
template <class Archive>
void serialize(Archive& ar) {
@@ -148,6 +178,8 @@ public:
void snap();
double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const;
double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const;
KeywordsStats getSnapshotStats(const std::string &poolName) const;
keywords_set getIpItemKeywordsSet(std::string ip);
keywords_set getUaItemKeywordsSet(std::string userAgent);
@@ -200,6 +232,8 @@ protected:
SerializedData m_serializedData;
std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordScoreMap> m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordsStats> m_snapshotStatsMap; // the snapshot is updated only by a call to snap()
std::list<std::string> m_falsePositivesSetsIntersection;
I_WaapAssetState* m_pWaapAssetState;
};

3
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -195,6 +195,9 @@ void SerializeToFileBase::saveData()
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
}
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
filestream << ss.str();

5
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@@ -12,7 +12,6 @@
// limitations under the License.
#include "WaapOverride.h"
#include "Waf2Util.h"
USE_DEBUG_FLAG(D_WAAP);
@@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const
(m_operand1 == other.m_operand1) &&
(m_operand2 == other.m_operand2) &&
(m_tag == other.m_tag) &&
Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) &&
m_cidr == other.m_cidr &&
(m_valuesRegex == other.m_valuesRegex) &&
m_ip_addr_values == other.m_ip_addr_values &&
m_isCidr == other.m_isCidr;
}

105
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@@ -21,6 +21,7 @@
#include <memory>
#include "debug.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
@@ -52,23 +53,52 @@ public:
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
try {
ar(cereal::make_nvp("values", m_values));
dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead.";
} catch (const cereal::Exception &e) {
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
m_values.insert(m_value);
}
if (m_tag == "sourceip" || m_tag == "sourceidentifier") {
m_isCidr = Waap::Util::isCIDR(m_value, m_cidr);
m_isCidr = true;
m_ip_addr_values.resize(m_values.size());
int val_idx = 0;
for (const auto &cur_val : m_values) {
if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) {
dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val;
m_isValid = false;
break;
}
val_idx++;
}
sortAndMergeCIDRs();
dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values);
}
m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody");
if (!m_isCidr) {
// regex build may throw boost::regex_error
m_valueRegex = nullptr;
try {
m_valueRegex = std::make_shared<boost::regex>(m_value);
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" <<
m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'";
for (const auto &cur_val : m_values) {
try {
m_valuesRegex.emplace(std::make_shared<boost::regex>(cur_val));
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE)
<< "Waap::Override::Match(): Failed to compile regex pattern '"
<< cur_val
<< "' on position "
<< err.position()
<< ". Reason: '"
<< err.what()
<< "'";
m_isValid = false;
m_valuesRegex.clear();
break;
}
}
}
}
@@ -95,13 +125,21 @@ public:
template<typename TestFunctor>
bool match(TestFunctor testFunctor) const {
if (m_op == "basic" && m_isCidr) {
bool result = testFunctor(m_tag, m_cidr);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result;
bool result = testFunctor(m_tag, m_ip_addr_values);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching CIDR list: "
<< cidrsToString(m_ip_addr_values)
<< " result: "
<< result;
return result;
}
else if (m_op == "basic" && m_valueRegex) {
bool result = testFunctor(m_tag, *m_valueRegex);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result;
else if (m_op == "basic" && !m_valuesRegex.empty()) {
bool result = testFunctor(m_tag, m_valuesRegex);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching regex list: "
<< regexSetToString(m_valuesRegex)
<< " result: "
<< result;
return result;
}
if (m_op == "and") {
@@ -125,24 +163,39 @@ public:
return false;
}
bool isOverrideResponse() const {
return m_isOverrideResponse;
}
bool isOverrideResponse() const { return m_isOverrideResponse; }
bool isValidMatch() const{
return m_isValid;
}
bool isValidMatch() const { return m_isValid; }
private:
void sortAndMergeCIDRs() {
if (m_ip_addr_values.empty()) return;
std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end());
size_t mergedIndex = 0;
for (size_t i = 1; i < m_ip_addr_values.size(); ++i) {
Waap::Util::CIDRData &current = m_ip_addr_values[mergedIndex];
Waap::Util::CIDRData &next = m_ip_addr_values[i];
if (!doesFirstCidrContainSecond(current, next)) {
++mergedIndex;
if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next;
}
}
m_ip_addr_values.resize(mergedIndex + 1);
}
std::string m_op;
std::shared_ptr<Match> m_operand1;
std::shared_ptr<Match> m_operand2;
std::string m_tag;
std::string m_value;
std::shared_ptr<boost::regex> m_valueRegex;
Waap::Util::CIDRData m_cidr;
bool m_isCidr;
bool m_isOverrideResponse;
std::set<std::string> m_values;
std::vector<Waap::Util::CIDRData> m_ip_addr_values;
std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> m_valuesRegex;
bool m_isCidr;
bool m_isOverrideResponse;
bool m_isValid;
};

165
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -17,100 +17,150 @@
#include "WaapOverrideFunctor.h"
#include "Waf2Engine.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
#include "agent_core_utilities.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
#define REGX_MATCH(FIELD_FETCH) \
NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx)
#define W2T_REGX_MATCH(FIELD_GETTER) \
REGX_MATCH(waf2Transaction.FIELD_GETTER())
WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction)
{
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) {
bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector<Waap::Util::CIDRData> &values) {
std::string sourceIp;
if (tag == "sourceip") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getRemoteAddr();
// match sourceIp against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
dbgDebug(D_WAAP_OVERRIDE)
<< "Remote IP Address : "
<< waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getRemoteAddr();
} else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getSourceIdentifier();
} else {
dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag;
return false;
}
else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getSourceIdentifier();
// match source against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
Waap::Util::CIDRData source_cidr;
if (!Waap::Util::isCIDR(sourceIp, source_cidr)) {
dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp;
return false;
}
int left = 0;
int right = values.size() - 1;
while (left <= right) {
int mid = left + (right - left) / 2;
if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true;
if (values[mid] < source_cidr) {
left = mid + 1;
} else {
right = mid - 1;
}
}
return false;
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
bool WaapOverrideFunctor::operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes)
{
boost::cmatch what;
std::string tagLower = tag;
std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
try {
if (tagLower == "method") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getMethod)) return true;
}
return false;
}
else if (tagLower == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getUriStr)) return true;
}
return false;
}
else if (tagLower == "hostname") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getHost)) return true;
}
return false;
}
else if (tagLower == "sourceidentifier") {
return NGEN::Regex::regexMatch(
__FILE__,
__LINE__,
waf2Transaction.getSourceIdentifier().c_str(),
what,
rx
);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getSourceIdentifier)) return true;
}
return false;
}
else if (tagLower == "keyword") {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {
return true;
}
}
}
return false;
}
else if (tagLower == "paramname") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getName())) {
return true;
}
}
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) {
return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) {
return true;
if (W2T_REGX_MATCH(getParamKey)) return true;
if (W2T_REGX_MATCH(getParam)) return true;
}
return false;
}
else if (tagLower == "paramvalue") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getValue())) {
return true;
}
}
if (W2T_REGX_MATCH(getSample)) return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) {
return true;
}
return false;
}
else if (tagLower == "paramlocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getLocation)) return true;
}
return false;
}
else if (tagLower == "responsebody") {
waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
if (!waf2Transaction.getResponseBody().empty()) {
boost::smatch matcher;
return NGEN::Regex::regexSearch(__FILE__, __LINE__,
waf2Transaction.getResponseBody().c_str(), matcher, rx);
for (const auto &rx : rxes) {
boost::smatch matcher;
if (NGEN::Regex::regexSearch(
__FILE__,
__LINE__,
waf2Transaction.getResponseBody().c_str(),
matcher,
*rx
)) {
return true;
}
}
return false;
} else {
return false;
}
@@ -119,11 +169,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@@ -132,11 +184,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();
return false;
}
// Unknown tag: should not occur
dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
return false;

12
components/security_apps/waap/waap_clib/WaapOverrideFunctor.h
View File

@@ -16,6 +16,7 @@
namespace Waap {
namespace Util {
struct CIDRData; // forward decleration
struct RegexComparator;
}
}
@@ -24,12 +25,15 @@ class Waf2Transaction;
// Functor used to match Override rules against request data
class WaapOverrideFunctor {
public:
WaapOverrideFunctor(Waf2Transaction& waf2Transaction);
WaapOverrideFunctor(Waf2Transaction &waf2Transaction);
bool operator()(const std::string& tag, const Waap::Util::CIDRData& value);
bool operator()(const std::string &tag, const std::vector<Waap::Util::CIDRData> &values);
bool operator()(const std::string& tag, const boost::regex& rx);
bool operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes
);
private:
Waf2Transaction& waf2Transaction;
Waf2Transaction &waf2Transaction;
};

49
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -13,7 +13,9 @@
#include "WaapScanner.h"
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@@ -105,20 +107,59 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
}
std::sort(newKeywords.begin(), newKeywords.end());
res.keywordsAfterFilter.clear();
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
string other_pool_name = Waap::Scores::getOtherScorePoolName();
Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
}
return res_score;
}
double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {
// Collect scores of individual keywords
Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray);
res.scoreArray, res.coefArray);
// Collect keyword combinations and their scores. Append scores to scoresArray,
// and also populate m_scanResultKeywordCombinations list
Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray, res.keywordCombinations);
res.scoreArray, res.coefArray, res.keywordCombinations);
}
if (stats.isLinModel) {
return Waap::Scores::calcLogisticRegressionScore(
res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef
);
}
// use base_scores calculation
return Waap::Scores::calcArrayScore(res.scoreArray);
}
@@ -127,7 +168,7 @@ bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res,
{
if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
return true;
}

3
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -44,6 +44,9 @@ namespace Waap {
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
double getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);

79
components/security_apps/waap/waap_clib/WaapScores.cc
View File

@@ -14,6 +14,8 @@
#include "WaapScores.h"
#include <vector>
#include <string>
#include <cmath>
#include "ScoreBuilder.h"
#include "WaapDefines.h"
#include "debug.h"
@@ -24,11 +26,44 @@ namespace Waap {
namespace Scores {
std::string getScorePoolNameByLocation(const std::string &location) {
std::string poolName = KEYWORDS_SCORE_POOL_BASE;
if (location == "header") {
poolName = KEYWORDS_SCORE_POOL_HEADERS;
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.scorePoolName");
std::string res = KEYWORDS_SCORE_POOL_BASE;
if (maybePoolName.ok()) {
res = maybePoolName.unpack();
}
return poolName;
else if (location == "header") {
res = KEYWORDS_SCORE_POOL_HEADERS;
}
return res;
}
std::string getOtherScorePoolName()
{
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.otherScorePoolName");
if (maybePoolName.ok()) {
return maybePoolName.unpack();
}
return KEYWORDS_SCORE_POOL_BASE;
}
ModelLoggingSettings getModelLoggingSettings()
{
ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF,
.logToS3 = false,
.logToStream = true};
auto maybeLogS3 = getProfileAgentSetting<bool>("agent.waap.modelLogToS3");
if (maybeLogS3.ok()) {
settings.logToS3 = maybeLogS3.unpack();
}
auto maybeLogKusto = getProfileAgentSetting<bool>("agent.waap.modelLogToStream");
if (maybeLogKusto.ok()) {
settings.logToStream = maybeLogKusto.unpack();
}
auto maybeLogLevel = getProfileAgentSetting<uint>("agent.waap.modelLogLevel");
if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) {
settings.logLevel = static_cast<ModelLogLevel>(maybeLogLevel.unpack());
}
return settings;
}
void
@@ -37,9 +72,16 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray)
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName));
double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName);
double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName);
dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef
<< " keyword: '" << keyword << "' pool: " << poolName;
scoresArray.push_back(score);
coefArray.push_back(coef);
}
// Calculate score of individual keywords
@@ -48,13 +90,14 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray)
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
std::vector<std::string> keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!)
std::sort(keywords.begin(), keywords.end());
for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) {
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray);
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray);
}
}
@@ -65,10 +108,12 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations)
{
keyword_combinations.clear();
static const double max_combi_score = 1.0f;
double default_coef = 0.8f;
for (size_t i = 0; i < keyword_matches.size(); ++i) {
std::vector<std::string> combinations;
@@ -93,7 +138,7 @@ calcCombinations(
}
// set default combination score to be the sum of its keywords, bounded by 1
default_score = std::min(default_score, max_combi_score);
addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray);
keyword_combinations.push_back(combination);
}
}
@@ -105,7 +150,6 @@ calcArrayScore(std::vector<double>& scoreArray)
// Calculate cumulative score from array of individual scores
double score = 1.0f;
for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore;
double left = 10.0f - score;
double divisor = (*pScore / 3.0f + 10.0f); // note: divisor can't be empty because
// *pScore is always positive and there's a +10 offset
@@ -115,5 +159,20 @@ calcArrayScore(std::vector<double>& scoreArray)
return score;
}
double
calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef)
{
// Sparse logistic regression model, with boolean feature values
// Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features
// An additional feature was added for the log of the number of non-zero features, as a regularization term
double log_odds = intercept + nnzCoef * log(static_cast<double>(coefArray.size()) + 1);
for (double &pCoef : coefArray) {
log_odds += pCoef;
}
// Apply the expit function to the log-odds to obtain the probability,
// and multiply by 10 to obtain a 'score' in the range [0, 10]
return 1.0f / (1.0f + exp(-log_odds)) * 10.0f;
}
}
}

23
components/security_apps/waap/waap_clib/WaapScores.h
View File

@@ -20,7 +20,21 @@
namespace Waap {
namespace Scores {
enum class ModelLogLevel {
OFF = 0,
DIFF = 1,
ALL = 2
};
struct ModelLoggingSettings {
ModelLogLevel logLevel;
bool logToS3;
bool logToStream;
};
std::string getScorePoolNameByLocation(const std::string &location);
std::string getOtherScorePoolName();
ModelLoggingSettings getModelLoggingSettings();
void
addKeywordScore(
@@ -28,7 +42,9 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray);
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate score of individual keywords
void
@@ -36,7 +52,8 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray);
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate keyword combinations and their scores
void
@@ -45,9 +62,11 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations);
double calcArrayScore(std::vector<double>& scoreArray);
double calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef=0.0);
}
}

2
components/security_apps/waap/waap_clib/WaapTrigger.h
View File

@@ -125,6 +125,7 @@ struct Log {
struct Trigger {
template <typename _A>
void serialize(_A &ar) {
ar(cereal::make_nvp("id", triggerId));
ar(cereal::make_nvp("$triggerType", triggerType));
triggerType = to_lower_copy(triggerType);
@@ -137,6 +138,7 @@ struct Trigger {
Trigger();
bool operator==(const Trigger &other) const;
std::string triggerId;
std::string triggerType;
std::shared_ptr<Log> log;
};

28
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = uriEnd - p;
dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
bool should_decode = checkUrlEncoded(p, buff_len);
if (should_decode) {
std::string url_as_string(p, buff_len);
should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true);
}
dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) {
m_ignoreScore = ignoreScore;
}
bool Waf2Transaction::get_ignoreScore() const
{
auto maybe_override_ignore_score = getProfileAgentSetting<std::string>("agent.waap.alwaysReportScore");
if (maybe_override_ignore_score.ok()) {
return maybe_override_ignore_score.unpack() == "true";
}
return m_ignoreScore;
}
void
Waf2Transaction::decide(
bool& bForceBlock,
@@ -1889,6 +1905,14 @@ Waf2Transaction::sendLog()
return;
}
std::set<std::string> triggers_set;
for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
triggers_set.insert(trigger.triggerId);
dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
}
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
switch (decision_type)
{

9
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -20,6 +20,8 @@
#include "DeepParser.h"
#include "WaapAssetState.h"
#include "PatternMatcher.h"
#include "generic_rulebase/rulebase_config.h"
#include "generic_rulebase/evaluators/trigger_eval.h"
#include "Waf2Util.h"
#include "WaapConfigApplication.h"
#include "WaapConfigApi.h"
@@ -39,6 +41,7 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"
@@ -91,6 +94,7 @@ public:
const std::vector<std::string> getFilteredKeywords() const;
const std::map<std::string, std::vector<std::string>> getFilteredVerbose() const;
virtual const std::vector<std::string> getKeywordsCombinations() const;
virtual const std::vector<std::string> getKeywordsAfterFilter() const;
const std::vector<DeepParser::KeywordInfo>& getKeywordInfo() const;
const std::vector<std::pair<std::string, std::string> >& getKvPairs() const;
const std::string getKeywordMatchesStr() const;
@@ -99,6 +103,9 @@ public:
const std::string getLastScanSample() const;
virtual const std::string& getLastScanParamName() const;
double getScore() const;
// LCOV_EXCL_START Reason: model testing
double getOtherModelScore() const;
// LCOV_EXCL_STOP
const std::vector<double> getScoreArray() const;
Waap::CSRF::State& getCsrfState();
const std::set<std::string> getFoundPatterns() const;
@@ -161,7 +168,7 @@ public:
// decision functions
void set_ignoreScore(bool ignoreScore);
bool get_ignoreScore() const { return m_ignoreScore; }
bool get_ignoreScore() const;
void decide(
bool& bForceBlock,
bool& bForceException,

19
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -189,6 +189,15 @@ const std::vector<std::string> Waf2Transaction::getKeywordsCombinations() const
}
return std::vector<std::string>();
}
const std::vector<std::string> Waf2Transaction::getKeywordsAfterFilter() const
{
if (m_scanResult)
{
return m_scanResult->keywordsAfterFilter;
}
return std::vector<std::string>();
}
const std::vector<DeepParser::KeywordInfo>& Waf2Transaction::getKeywordInfo() const
{
return m_deepParser.m_keywordInfo;
@@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const
}
return 0;
}
// LCOV_EXCL_START Reason: model testing
double Waf2Transaction::getOtherModelScore() const
{
if (m_scanResult) {
return m_scanResult->other_model_score;
}
return 0;
}
// LCOV_EXCL_STOP
const std::vector<double> Waf2Transaction::getScoreArray() const
{
if (m_scanResult) {
@@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
waap_log << LogField("waapUriFalsePositiveScore", (int)(
autonomousSecurityDecision->getFpMitigationScore() * 100));
waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100));
waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100));
waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100));
waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel());
}

39
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -508,6 +508,8 @@ const char* g_htmlTags[] = {
};
static const string b64_prefix("base64,");
// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded
static const double base64_entropy_threshold = 4.01;
const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]);
@@ -966,7 +968,8 @@ base64_decode_status decodeBase64Chunk(
string::const_iterator it,
string::const_iterator end,
string& decoded,
bool clear_on_error)
bool clear_on_error,
bool called_with_prefix)
{
decoded.clear();
uint32_t acc = 0;
@@ -974,6 +977,7 @@ base64_decode_status decodeBase64Chunk(
int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them.
uint32_t nonPrintableCharsCount = 0;
uint32_t spacer_count = 0;
uint32_t length = end - it;
dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
string::const_iterator begin = it;
@@ -986,6 +990,8 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID;
}
std::unordered_map<char, double> frequency;
while (it != end) {
unsigned char c = *it;
@@ -1008,6 +1014,7 @@ base64_decode_status decodeBase64Chunk(
// allow for more terminator characters
it++;
frequency[c]++;
continue;
}
@@ -1032,6 +1039,7 @@ base64_decode_status decodeBase64Chunk(
// Start tracking terminator characters
terminatorCharsSeen++;
it++;
frequency[c]++;
continue;
}
else {
@@ -1062,6 +1070,7 @@ base64_decode_status decodeBase64Chunk(
}
it++;
frequency[c]++;
}
// end of encoded sequence decoded.
@@ -1078,6 +1087,27 @@ base64_decode_status decodeBase64Chunk(
<< "; decoded='"
<< decoded << "'";
// Check if entropy is correlates with b64 threshold (initially > 4.5)
if (!called_with_prefix) {
double entropy = 0;
double p = 0;
for (const auto& pair : frequency) {
p = pair.second / length;
entropy -= p * std::log2(p);
}
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: base entropy = " << entropy << "length = " << length;
// Add short payload factor
if (length < 16)
entropy = entropy * 16 / length;
// Enforce tailoring '=' characters
entropy+=terminatorCharsSeen;
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: corrected entropy = " << entropy << "length = " << length;
if (entropy <= base64_entropy_threshold) {
return B64_DECODE_INVALID;
}
}
// Return success only if decoded.size>=5 and there are less than 10% of non-printable
// characters in output.
if (decoded.size() >= 5) {
@@ -1323,10 +1353,11 @@ processDecodedChunk(
string::const_iterator start,
string::const_iterator end,
string &value,
BinaryFileType &binaryFileType
BinaryFileType &binaryFileType,
bool called_with_prefix = false
)
{
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix);
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
<< " value.size()=" << value.size();
if (retVal != B64_DECODE_INVALID && !value.empty()) {
@@ -1349,7 +1380,7 @@ bool isBase64PrefixProcessingOK (
if (detectBase64Chunk(s, start, end)) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
if ((start != s.end()) && (end == s.end())) {
retVal = processDecodedChunk(s, start, end, value, binaryFileType);
retVal = processDecodedChunk(s, start, end, value, binaryFileType, true);
}
} else if (start != s.end()) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."

3
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -871,7 +871,8 @@ decodeBase64Chunk(
std::string::const_iterator it,
std::string::const_iterator end,
std::string &decoded,
bool clear_on_error = true);
bool clear_on_error = true,
bool called_with_prefix = false);
bool
b64DecodeChunk(

3
components/security_apps/waap/waap_component_impl.cc
View File

@@ -50,7 +50,8 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer()
deepAnalyzer(),
waapModelResultLogger()
{
}

2
components/security_apps/waap/waap_component_impl.h
View File

@@ -19,6 +19,7 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@@ -80,6 +81,7 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};