Merge pull request #207 from openappsec/Nov_28_2024-Dev

Nov 28 2024 dev

components/attachment-intakers/nginx_attachment/nginx_attachment.cc

@@ -1135,7 +1135,11 @@ private:
             "webUserResponse"
         );
 
+        bool remove_event_id_param =
+            getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
+
         string uuid;
+        string redirectUrl;
         if (i_transaction_table->hasState<NginxAttachmentOpaque>()) {
             NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
             uuid = opaque.getSessionUUID();
@@ -1145,7 +1149,12 @@ private:
         if (web_trigger_conf.getDetailsLevel() == "Redirect") {
             web_response_data.response_data.redirect_data.redirect_location_size =
                 web_trigger_conf.getRedirectURL().size();
-            web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0;
+            bool add_event = web_trigger_conf.getAddEventId();
+            if (add_event && !remove_event_id_param) {
+                web_response_data.response_data.redirect_data.redirect_location_size +=
+                    strlen("?event_id=") + uuid.size();
+            }
+            web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
             web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
         } else {
             web_response_data.response_data.custom_response_data.title_size =
@@ -1159,8 +1168,13 @@ private:
         verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));
 
         if (web_trigger_conf.getDetailsLevel() == "Redirect") {
-            verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getRedirectURL().data()));
-            verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size());
+            redirectUrl = web_trigger_conf.getRedirectURL();
+            if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
+                redirectUrl += "?event-id=" + uuid;
+            }
+
+            verdict_data.push_back(reinterpret_cast<const char *>(redirectUrl.data()));
+            verdict_data_sizes.push_back(redirectUrl.size());
         } else {
             verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getResponseTitle().data()));
             verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size());

components/attachment-intakers/nginx_attachment/user_identifiers_config.cc

@@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector<CIDRSData> &cidr_values)
 }
 
 Maybe<string>
-UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
+UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const
 {
     vector<string> header_values = split(str);
 
@@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
     vector<string> xff_values = getHeaderValuesFromConfig("x-forwarded-for");
     vector<CIDRSData> cidr_values(xff_values.begin(), xff_values.end());
 
-    for (const string &value : header_values) {
-        if (!IPAddr::createIPAddr(value).ok()) {
-            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value;
+    for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) {
+        if (!IPAddr::createIPAddr(*it).ok()) {
+            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it;
             return genError("Invalid IP address");
         }
-        if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found");
+        if (type == ExtractType::PROXYIP) continue;
+        if (!isIpTrusted(*it, cidr_values)) {
+            dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it;
+            return *it;
+        }
+    }
+
+    if (!IPAddr::createIPAddr(header_values[0]).ok()) {
+        dbgWarning(D_NGINX_ATTACHMENT_PARSER)
+            << "Invalid IP address found in the xff header IPs list: "
+            << header_values[0];
+        return genError("Invalid IP address");
     }
 
     return header_values[0];
@@ -312,7 +323,7 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
         return;
     }
     NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
-    auto value = parseXForwardedFor(header.getValue());
+    auto value = parseXForwardedFor(header.getValue(), type);
     if (!value.ok()) {
         dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
         return;
@@ -321,12 +332,13 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
     if (type == ExtractType::SOURCEIDENTIFIER) {
         opaque.setSourceIdentifier(header.getKey(), value.unpack());
         dbgDebug(D_NGINX_ATTACHMENT_PARSER)
-            << "Added source identifir to XFF "
+            << "Added source identifier from XFF header"
             <<  value.unpack();
         opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
+        opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack());
         dbgTrace(D_NGINX_ATTACHMENT_PARSER)
-        << "XFF found, set ctx with value from header: "
-        << static_cast<string>(header.getValue());
+            << "XFF found, set ctx with value from header: "
+            << static_cast<string>(header.getValue());
     } else {
         opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack());
     }

components/include/env_details.h

@@ -29,12 +29,15 @@ public:
 
     virtual EnvType getEnvType() override;
     virtual std::string getToken() override;
+    virtual std::string getNameSpace() override;
 
 private:
     std::string retrieveToken();
+    std::string retrieveNamespace();
     std::string readFileContent(const std::string &file_path);
 
     std::string token;
+    std::string agent_namespace;
     EnvType env_type;
 };
 

components/include/generic_rulebase/match_query.h

@@ -91,7 +91,7 @@ private:
     bool matchAttributesString(const std::set<std::string> &values) const;
     bool matchAttributesIp(const std::set<std::string> &values) const;
     bool isRegEx() const;
-    bool isIP() const;
+    void sortAndMergeIpRangesValues();
 
     MatchType type;
     Operators operator_type;

components/include/ip_utilities.h

@@ -28,8 +28,9 @@
 
 // LCOV_EXCL_START Reason: temporary until we add relevant UT until 07/10
 bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
-
 bool operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<(const IPRange &range1, const IPRange &range2);
 // LCOV_EXCL_STOP
 
 Maybe<std::pair<std::string, int>> extractAddressAndMaskSize(const std::string &cidr);

components/include/reverse_proxy_defaults.h

@@ -28,7 +28,7 @@ static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/ngi
 static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf";
 static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
 static const std::string default_nginx_config_include_file =
-    "/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf";
+    "/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf";
 static const std::string default_global_conf_include_template =
     "/etc/cp/conf/rpmanager/nginx-conf-include-template";
 static const std::string default_global_conf_include_template_no_responses =

components/include/user_identifiers_config.h

@@ -58,7 +58,7 @@ private:
         const std::string::const_iterator &end,
         const std::string &key) const;
     Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const;
-    Maybe<std::string> parseXForwardedFor(const std::string &str) const;
+    Maybe<std::string> parseXForwardedFor(const std::string &str, ExtractType type) const;
 
     std::vector<UsersIdentifiersConfig> user_identifiers;
 };

components/security_apps/local_policy_mgmt_gen/access_control_practice.cc

@@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
 
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
-    parseAppsecJSONKey<string>("practiceMode", mode, archive_in);
+    parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Access control practice mode invalid: " + mode);
+    }
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<AccessControlRateLimit>("rateLimit", rate_limit, archive_in);
 }

components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc

@@ -438,19 +438,30 @@ WebAppSection::WebAppSection(
     csrf_protection_mode("Disabled"),
     open_redirect_mode("Disabled"),
     error_disclosure_mode("Disabled"),
+    schema_validation_mode("Disabled"),
+    schema_validation_enforce_level("fullSchema"),
     practice_advanced_config(parsed_appsec_spec),
     anti_bots(parsed_appsec_spec.getAntiBot()),
     trusted_sources({ parsed_trusted_sources })
 {
+    auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
+    if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << mitigation_sevirity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
     web_attack_mitigation_severity =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
+        web_attack_mitigation_severity;
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
@@ -479,6 +490,9 @@ WebAppSection::WebAppSection(
     const string &_web_attack_mitigation_severity,
     const string &_web_attack_mitigation_mode,
     const string &_bot_protection,
+    const string &_schema_validation_mode,
+    const string &_schema_validation_enforce_level,
+    const vector<string> &_schema_validation_oas,
     const PracticeAdvancedConfig &_practice_advanced_config,
     const AppsecPracticeAntiBotSection &_anti_bots,
     const LogTriggerSection &parsed_log_trigger,
@@ -493,19 +507,29 @@ WebAppSection::WebAppSection(
     practice_id(_practice_id),
     practice_name(_practice_name),
     context(_context),
-    web_attack_mitigation_severity(_web_attack_mitigation_severity),
     web_attack_mitigation_mode(_web_attack_mitigation_mode),
     bot_protection(_bot_protection),
+    schema_validation_mode(_schema_validation_mode),
+    schema_validation_enforce_level(_schema_validation_enforce_level),
+    schema_validation_oas(_schema_validation_oas),
     practice_advanced_config(_practice_advanced_config),
     anti_bots(_anti_bots),
     trusted_sources({ parsed_trusted_sources })
 {
+    if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << _web_attack_mitigation_severity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
@@ -516,6 +540,7 @@ WebAppSection::WebAppSection(
     for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
         overrides.push_back(AppSecOverride(source_ident));
     }
+
 }
 
 // LCOV_EXCL_STOP
@@ -523,35 +548,35 @@ WebAppSection::WebAppSection(
 void
 WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
 {
-    string disabled_str = "Disabled";
     vector<string> empty_list;
     out_ar(
-        cereal::make_nvp("context",                     context),
-        cereal::make_nvp("webAttackMitigation",         web_attack_mitigation),
-        cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
-        cereal::make_nvp("webAttackMitigationAction",   web_attack_mitigation_action),
-        cereal::make_nvp("webAttackMitigationMode",     web_attack_mitigation_mode),
-        cereal::make_nvp("practiceAdvancedConfig",      practice_advanced_config),
-        cereal::make_nvp("csrfProtection",              csrf_protection_mode),
-        cereal::make_nvp("openRedirect",                open_redirect_mode),
-        cereal::make_nvp("errorDisclosure",             error_disclosure_mode),
-        cereal::make_nvp("practiceId",                  practice_id),
-        cereal::make_nvp("practiceName",                practice_name),
-        cereal::make_nvp("assetId",                     asset_id),
-        cereal::make_nvp("assetName",                   asset_name),
-        cereal::make_nvp("ruleId",                      rule_id),
-        cereal::make_nvp("ruleName",                    rule_name),
-        cereal::make_nvp("schemaValidation",               false),
-        cereal::make_nvp("schemaValidation_v2",            disabled_str),
-        cereal::make_nvp("oas",                            empty_list),
-        cereal::make_nvp("triggers",                    triggers),
-        cereal::make_nvp("applicationUrls",             application_urls),
-        cereal::make_nvp("overrides",                   overrides),
-        cereal::make_nvp("trustedSources",              trusted_sources),
-        cereal::make_nvp("waapParameters",              empty_list),
-        cereal::make_nvp("botProtection",               false),
-        cereal::make_nvp("antiBot",                     anti_bots),
-        cereal::make_nvp("botProtection_v2",            bot_protection != "" ? bot_protection : string("Detect"))
+        cereal::make_nvp("context",                         context),
+        cereal::make_nvp("webAttackMitigation",             web_attack_mitigation),
+        cereal::make_nvp("webAttackMitigationSeverity",     web_attack_mitigation_severity),
+        cereal::make_nvp("webAttackMitigationAction",       web_attack_mitigation_action),
+        cereal::make_nvp("webAttackMitigationMode",         web_attack_mitigation_mode),
+        cereal::make_nvp("practiceAdvancedConfig",          practice_advanced_config),
+        cereal::make_nvp("csrfProtection",                  csrf_protection_mode),
+        cereal::make_nvp("openRedirect",                    open_redirect_mode),
+        cereal::make_nvp("errorDisclosure",                 error_disclosure_mode),
+        cereal::make_nvp("practiceId",                      practice_id),
+        cereal::make_nvp("practiceName",                    practice_name),
+        cereal::make_nvp("assetId",                         asset_id),
+        cereal::make_nvp("assetName",                       asset_name),
+        cereal::make_nvp("ruleId",                          rule_id),
+        cereal::make_nvp("ruleName",                        rule_name),
+        cereal::make_nvp("schemaValidation",                schema_validation_mode == "Prevent"),
+        cereal::make_nvp("schemaValidation_v2",             schema_validation_mode),
+        cereal::make_nvp("oas",                             schema_validation_oas),
+        cereal::make_nvp("schemaValidationEnforceLevel",    schema_validation_enforce_level),
+        cereal::make_nvp("triggers",                        triggers),
+        cereal::make_nvp("applicationUrls",                 application_urls),
+        cereal::make_nvp("overrides",                       overrides),
+        cereal::make_nvp("trustedSources",                  trusted_sources),
+        cereal::make_nvp("waapParameters",                  empty_list),
+        cereal::make_nvp("botProtection",                   false),
+        cereal::make_nvp("antiBot",                         anti_bots),
+        cereal::make_nvp("botProtection_v2",                bot_protection != "" ? bot_protection : string("Detect"))
     );
 }
 

components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h

@@ -291,6 +291,9 @@ public:
         const std::string &_web_attack_mitigation_severity,
         const std::string &_web_attack_mitigation_mode,
         const std::string &_bot_protection,
+        const std::string &schema_validation_mode,
+        const std::string &schema_validation_enforce_level,
+        const std::vector<std::string> &schema_validation_oas,
         const PracticeAdvancedConfig &_practice_advanced_config,
         const AppsecPracticeAntiBotSection &_anti_bots,
         const LogTriggerSection &parsed_log_trigger,
@@ -302,27 +305,30 @@ public:
     bool operator< (const WebAppSection &other) const;
 
 private:
-    std::string application_urls;
-    std::string asset_id;
-    std::string asset_name;
-    std::string rule_id;
-    std::string rule_name;
-    std::string practice_id;
-    std::string practice_name;
-    std::string context;
-    std::string web_attack_mitigation_action;
-    std::string web_attack_mitigation_severity;
-    std::string web_attack_mitigation_mode;
-    std::string csrf_protection_mode;
-    std::string open_redirect_mode;
-    std::string error_disclosure_mode;
-    std::string bot_protection;
-    bool web_attack_mitigation;
-    std::vector<TriggersInWaapSection> triggers;
-    PracticeAdvancedConfig practice_advanced_config;
-    AppsecPracticeAntiBotSection anti_bots;
-    std::vector<AppSecTrustedSources> trusted_sources;
-    std::vector<AppSecOverride> overrides;
+    bool                                    web_attack_mitigation;
+    std::string                             application_urls;
+    std::string                             asset_id;
+    std::string                             asset_name;
+    std::string                             rule_id;
+    std::string                             rule_name;
+    std::string                             practice_id;
+    std::string                             practice_name;
+    std::string                             context;
+    std::string                             web_attack_mitigation_action;
+    std::string                             web_attack_mitigation_severity;
+    std::string                             web_attack_mitigation_mode;
+    std::string                             csrf_protection_mode;
+    std::string                             open_redirect_mode;
+    std::string                             error_disclosure_mode;
+    std::string                             bot_protection;
+    std::string                             schema_validation_mode;
+    std::string                             schema_validation_enforce_level;
+    std::vector<std::string>                schema_validation_oas;
+    PracticeAdvancedConfig                  practice_advanced_config;
+    AppsecPracticeAntiBotSection            anti_bots;
+    std::vector<AppSecOverride>             overrides;
+    std::vector<AppSecTrustedSources>       trusted_sources;
+    std::vector<TriggersInWaapSection>      triggers;
 };
 
 class WebAPISection
@@ -410,7 +416,7 @@ class ParsedRule
 {
 public:
     ParsedRule() {}
-    ParsedRule(const std::string &_host) : host(_host) {}
+    ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
     const std::vector<std::string> & getExceptions() const;

components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h

@@ -24,6 +24,7 @@
 #include "maybe_res.h"
 #include "i_orchestration_tools.h"
 #include "i_shell_cmd.h"
+#include "i_encryptor.h"
 #include "i_messaging.h"
 #include "i_env_details.h"
 #include "i_agent_details.h"
@@ -40,6 +41,7 @@ class K8sPolicyUtils
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_ShellCmd>,
     Singleton::Consume<I_EnvDetails>,
+    Singleton::Consume<I_Encryptor>,
     Singleton::Consume<I_AgentDetails>
 {
 public:
@@ -80,6 +82,8 @@ private:
 
     void createSnortFile(std::vector<NewAppSecPracticeSpec> &practices) const;
 
+    void createSchemaValidationOas(std::vector<NewAppSecPracticeSpec> &practices) const;
+
     template<class T>
     std::vector<T> extractV1Beta2ElementsFromCluster(
         const std::string &crd_plural,
@@ -112,6 +116,7 @@ private:
     I_Messaging* messaging = nullptr;
     EnvType env_type;
     std::string token;
+    std::string agent_ns;
 };
 
 #endif // __K8S_POLICY_UTILS_H__

components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h

@@ -49,6 +49,13 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
     { "WebUserResponse", TriggerType::WebUserResponse }
 };
 
+static const std::unordered_map<std::string, std::string> key_to_mitigation_severity = {
+    { "high", "High"},
+    { "medium", "Medium"},
+    { "critical", "Critical"},
+    { "Transparent", "Transparent"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -57,6 +64,14 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val =
     { "inactive", "Inactive"}
 };
 
+static const std::unordered_map<std::string, std::string> key_to_practices_mode_val = {
+    { "prevent-learn", "Prevent"},
+    { "detect-learn", "Detect"},
+    { "prevent", "Prevent"},
+    { "detect", "Detect"},
+    { "inactive", "Disabled"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val2 = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -66,6 +81,8 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val2
 };
 
 static const std::string default_appsec_url = "http://*:*";
+static const std::string default_appsec_name = "Any";
+
 
 class PolicyGenException : public std::exception
 {

components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h

@@ -31,7 +31,7 @@ class NewParsedRule
 {
 public:
     NewParsedRule() {}
-    NewParsedRule(const std::string &_host) : host(_host) {}
+    NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
 

components/security_apps/local_policy_mgmt_gen/include/new_practice.h

@@ -23,6 +23,8 @@
 #include "config.h"
 #include "debug.h"
 #include "local_policy_common.h"
+#include "i_orchestration_tools.h"
+#include "i_encryptor.h"
 
 bool isModeInherited(const std::string &mode);
 
@@ -88,6 +90,8 @@ public:
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
+    bool operator<(const IpsProtectionsSection &other) const;
+
 private:
     std::string                                 context;
     std::string                                 name;
@@ -105,7 +109,7 @@ public:
     // LCOV_EXCL_START Reason: no test exist
     IPSSection() {};
 
-    IPSSection(const std::vector<IpsProtectionsSection> &_ips) : ips(_ips) {};
+    IPSSection(const std::vector<IpsProtectionsSection> &_ips);
     // LCOV_EXCL_STOP
 
     void save(cereal::JSONOutputArchive &out_ar) const;
@@ -138,6 +142,12 @@ public:
     const std::string & getMode(const std::string &default_mode = "inactive") const;
 
 private:
+
+    const std::string & getRulesMode(
+        const std::string &mode,
+        const std::string &default_mode = "inactive"
+    ) const;
+
     std::string override_mode;
     std::string max_performance_impact;
     std::string min_severity_level;
@@ -487,15 +497,16 @@ private:
     SnortSection snort;
 };
 
-class NewSnortSignaturesAndOpenSchemaAPI
+class NewSnortSignatures
 {
 public:
-    NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {};
+    NewSnortSignatures() : is_temporary(false) {};
 
     void load(cereal::JSONInputArchive &archive_in);
 
     void addFile(const std::string &file_name);
     const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
     const std::vector<std::string> & getConfigMap() const;
     const std::vector<std::string> & getFiles() const;
     bool isTemporary() const;
@@ -503,17 +514,40 @@ public:
 
 private:
     std::string override_mode;
+    std::string enforcement_level;
     std::vector<std::string> config_map;
     std::vector<std::string> files;
     bool is_temporary;
 };
 
+class NewOpenApiSchema : Singleton::Consume<I_OrchestrationTools>, Singleton::Consume<I_Encryptor>
+{
+public:
+    NewOpenApiSchema() {};
+
+    void load(cereal::JSONInputArchive &archive_in);
+
+    void addOas(const std::string &file);
+    const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
+    const std::vector<std::string> & getConfigMap() const;
+    const std::vector<std::string> & getFiles() const;
+    const std::vector<std::string> & getOas() const;
+
+private:
+    std::string override_mode;
+    std::string enforcement_level;
+    std::vector<std::string> config_map;
+    std::vector<std::string> files;
+    std::vector<std::string> oas;
+};
+
 class NewAppSecPracticeAntiBot
 {
 public:
     const std::vector<std::string> & getIjectedUris() const;
     const std::vector<std::string> & getValidatedUris() const;
-    const std::string & getMode() const;
+    const std::string & getMode(const std::string &default_mode = "inactive") const;
 
     void load(cereal::JSONInputArchive &archive_in);
     void save(cereal::JSONOutputArchive &out_ar) const;
@@ -569,8 +603,8 @@ class NewAppSecPracticeSpec
 public:
     void load(cereal::JSONInputArchive &archive_in);
 
-    NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures();
-    const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const;
+    NewSnortSignatures & getSnortSignatures();
+    NewOpenApiSchema & getOpenSchemaValidation();
     const NewAppSecPracticeWebAttacks & getWebAttacks() const;
     const NewAppSecPracticeAntiBot & getAntiBot() const;
     const NewIntrusionPrevention & getIntrusionPrevention() const;
@@ -583,8 +617,8 @@ public:
 private:
     NewFileSecurity                             file_security;
     NewIntrusionPrevention                      intrusion_prevention;
-    NewSnortSignaturesAndOpenSchemaAPI          openapi_schema_validation;
-    NewSnortSignaturesAndOpenSchemaAPI          snort_signatures;
+    NewOpenApiSchema                            openapi_schema_validation;
+    NewSnortSignatures                          snort_signatures;
     NewAppSecPracticeWebAttacks                 web_attacks;
     NewAppSecPracticeAntiBot                    anti_bot;
     std::string                                 appsec_class_name;

components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h

@@ -123,6 +123,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
@@ -145,6 +146,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 

components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc

@@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key)
     }
 }
 
+string
+getAppSecScopeType()
+{
+    auto env_res = getenv("CRDS_SCOPE");
+    if (env_res != nullptr) return env_res;
+    return "cluster";
+}
+
 void
 K8sPolicyUtils::init()
 {
@@ -42,6 +50,7 @@ K8sPolicyUtils::init()
     env_type = env_details->getEnvType();
     if (env_type == EnvType::K8S) {
         token = env_details->getToken();
+        agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : "";
         messaging = Singleton::Consume<I_Messaging>::by<K8sPolicyUtils>();
     }
 }
@@ -140,10 +149,12 @@ extractElementsFromNewRule(
     const NewParsedRule &rule,
     map<AnnotationTypes, unordered_set<string>> &policy_elements_names)
 {
-    policy_elements_names[AnnotationTypes::EXCEPTION].insert(
-        rule.getExceptions().begin(),
-        rule.getExceptions().end()
-    );
+    if (rule.getExceptions().size() > 0) {
+        policy_elements_names[AnnotationTypes::EXCEPTION].insert(
+            rule.getExceptions().begin(),
+            rule.getExceptions().end()
+        );
+    }
     policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert(
         rule.getPractices().begin(),
         rule.getPractices().end()
@@ -152,14 +163,24 @@ extractElementsFromNewRule(
         rule.getAccessControlPractices().begin(),
         rule.getAccessControlPractices().end()
     );
-    policy_elements_names[AnnotationTypes::TRIGGER].insert(
-        rule.getLogTriggers().begin(),
-        rule.getLogTriggers().end()
-    );
-    policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
-    policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
-    policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
-    policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
+    if (rule.getLogTriggers().size() > 0) {
+        policy_elements_names[AnnotationTypes::TRIGGER].insert(
+            rule.getLogTriggers().begin(),
+            rule.getLogTriggers().end()
+        );
+    }
+    if (rule.getCustomResponse() != "" ) {
+        policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
+    }
+    if (rule.getSourceIdentifiers() != "" ) {
+        policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
+    }
+    if (rule.getTrustedSources() != "" ) {
+        policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
+    }
+    if (rule.getUpgradeSettings() != "" ) {
+        policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
+    }
 }
 
 map<AnnotationTypes, unordered_set<string>>
@@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster(
     dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural;
     vector<T> elements;
     for (const string &element_name : elements_names) {
+        string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+        string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
         dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name;
         auto maybe_appsec_element = getObjectFromCluster<AppsecSpecParser<T>>(
-            "/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name
+            "/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name
         );
 
         if (!maybe_appsec_element.ok()) {
@@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
         practice.getSnortSignatures().setTemporary(true);
         for (const string &config_map : practice.getSnortSignatures().getConfigMap())
         {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
             auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
-                "/api/v1/namespaces/default/configmaps/" + config_map
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
             );
             if (!maybe_configmap.ok())  {
                 dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
@@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
     }
 }
 
+void
+K8sPolicyUtils::createSchemaValidationOas(vector<NewAppSecPracticeSpec> &practices) const
+{
+    for (NewAppSecPracticeSpec &practice : practices) {
+        vector<string> res;
+        for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap())
+        {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
+            auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
+            );
+            if (!maybe_configmap.ok())  {
+                dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
+                continue;
+            }
+            string file_content = maybe_configmap.unpack().getFileContent();
+            string res = Singleton::Consume<I_Encryptor>::by<K8sPolicyUtils>()->base64Encode(file_content);
+            practice.getOpenSchemaValidation().addOas(res);
+        }
+    }
+}
+
 Maybe<V1beta2AppsecLinuxPolicy>
 K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     const AppsecSpecParser<NewAppsecPolicySpec> &appsec_policy_spec,
@@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     }
 
     if (default_rule.getMode().empty() && !ingress_mode.empty()) {
+        dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode;
         default_rule.setMode(ingress_mode);
     }
 
@@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
         );
 
     createSnortFile(threat_prevention_practices);
+    createSchemaValidationOas(threat_prevention_practices);
 
     vector<AccessControlPracticeSpec> access_control_practices =
         extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
@@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
                 maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
             dbgWarning(D_LOCAL_POLICY
             ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
+            string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+            string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
             auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
-                "/apis/openappsec.io/v1beta2/policies/" + policy_name
+                "/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name
             );
+
             if (!maybe_v1beta2_appsec_policy_spec.ok()) {
                 dbgWarning(D_LOCAL_POLICY)
                     << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
@@ -535,10 +586,11 @@ K8sPolicyUtils::createPolicy(
     if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) {
         policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
     }
+    auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
     if (item.getSpec().doesDefaultBackendExist()) {
         dbgTrace(D_LOCAL_POLICY)
             << "Inserting Any host rule to the specific asset set";
-        K ingress_rule = K("*");
+        K ingress_rule = K("*", default_mode);
         policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
     }
 
@@ -556,12 +608,11 @@ K8sPolicyUtils::createPolicy(
                     << "' uri: '"
                     << uri.getPath()
                     << "'";
-                K ingress_rule = K(host);
+                K ingress_rule = K(host, default_mode);
                 policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
             }
         }
     }
-
 }
 
 std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>

components/security_apps/local_policy_mgmt_gen/new_practice.cc

@@ -22,6 +22,7 @@ static const set<string> performance_impacts    = {"low", "medium", "high"};
 static const set<string> severity_levels        = {"low", "medium", "high", "critical"};
 static const set<string> size_unit              = {"bytes", "KB", "MB", "GB"};
 static const set<string> confidences_actions    = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
+static const set<string> valied_enforcement_level = {"fullSchema", "endpointOnly"};
 static const set<string> valid_modes = {
     "prevent",
     "detect",
@@ -32,38 +33,38 @@ static const set<string> valid_modes = {
     "inherited"
 };
 static const set<string> valid_confidences      = {"medium", "high", "critical"};
-static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
+static const unordered_map<string, string> key_to_performance_impact_val = {
     { "low", "Low or lower"},
     { "medium", "Medium or lower"},
     { "high", "High or lower"}
 };
-static const std::unordered_map<std::string, std::string> key_to_severity_level_val = {
+static const unordered_map<string, string> key_to_severity_level_val = {
     { "low", "Low or above"},
     { "medium", "Medium or above"},
     { "high", "High or above"},
     { "critical", "Critical"}
 };
-static const std::unordered_map<std::string, std::string> key_to_mode_val = {
+static const unordered_map<string, string> key_to_mode_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Detect"},
     { "prevent", "Prevent"},
     { "detect", "Detect"},
     { "inactive", "Inactive"}
 };
-static const std::unordered_map<std::string, std::string> anti_bot_key_to_mode_val = {
+static const unordered_map<string, string> anti_bot_key_to_mode_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Detect"},
     { "prevent", "Prevent"},
     { "detect", "Detect"},
     { "inactive", "Disabled"}
 };
-static const std::unordered_map<std::string, uint64_t> unit_to_int = {
+static const unordered_map<string, uint64_t> unit_to_int = {
     { "bytes", 1},
     { "KB", 1024},
     { "MB", 1048576},
     { "GB", 1073741824}
 };
-static const std::string TRANSPARENT_MODE = "Transparent";
+static const string TRANSPARENT_MODE = "Transparent";
 
 bool
 isModeInherited(const string &mode)
@@ -71,11 +72,11 @@ isModeInherited(const string &mode)
     return mode == "as-top-level" || mode == "inherited";
 }
 
-const std::string &
+const string &
 getModeWithDefault(
-    const std::string &mode,
-    const std::string &default_mode,
-    const std::unordered_map<std::string, std::string> &key_to_val)
+    const string &mode,
+    const string &default_mode,
+    const unordered_map<string, string> &key_to_val)
 {
     if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
         dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
@@ -88,36 +89,35 @@ getModeWithDefault(
     return key_to_val.at(mode);
 }
 
-const std::vector<std::string> &
+const vector<string> &
 NewAppSecPracticeAntiBot::getIjectedUris() const
 {
     return injected_uris;
 }
 
-const std::vector<std::string> &
+const vector<string> &
 NewAppSecPracticeAntiBot::getValidatedUris() const
 {
     return validated_uris;
 }
 
-const std::string &
-NewAppSecPracticeAntiBot::getMode() const
+const string &
+NewAppSecPracticeAntiBot::getMode(const string &default_mode) const
 {
-    return override_mode;
+    return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val);
 }
 
 void
 NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
-    string mode;
     parseAppsecJSONKey<vector<string>>("injectedUris", injected_uris, archive_in);
     parseAppsecJSONKey<vector<string>>("validatedUris", validated_uris, archive_in);
-    parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
-    if (valid_modes.count(mode) == 0) {
-        dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode;
+    parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode);
     }
-    override_mode = anti_bot_key_to_mode_val.at(mode);
 }
 
 void
@@ -242,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const
 }
 
 SnortProtectionsSection::SnortProtectionsSection(
-    const std::string               &_context,
-    const std::string               &_asset_name,
-    const std::string               &_asset_id,
-    const std::string               &_practice_name,
-    const std::string               &_practice_id,
-    const std::string               &_source_identifier,
-    const std::string               &_mode,
-    const std::vector<std::string>  &_files)
+    const string               &_context,
+    const string               &_asset_name,
+    const string               &_asset_id,
+    const string               &_practice_name,
+    const string               &_practice_id,
+    const string               &_source_identifier,
+    const string               &_mode,
+    const vector<string>  &_files)
         :
     context(_context),
     asset_name(_asset_name),
@@ -278,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 DetectionRules::DetectionRules(
-    const std::string                   &_type,
-    const std::string                   &_SSM,
-    const std::string                   &_keywords,
-    const std::vector<std::string>      &_context)
+    const string                   &_type,
+    const string                   &_SSM,
+    const string                   &_keywords,
+    const vector<string>      &_context)
         :
     type(_type),
     SSM(_SSM),
@@ -314,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const
 
 ProtectionMetadata::ProtectionMetadata(
     bool                                _silent,
-    const std::string                   &_protection_name,
-    const std::string                   &_severity,
-    const std::string                   &_confidence_level,
-    const std::string                   &_performance_impact,
-    const std::string                   &_last_update,
-    const std::string                   &_maintrain_id,
-    const std::vector<std::string>      &_tags,
-    const std::vector<std::string>      &_cve_list)
+    const string                   &_protection_name,
+    const string                   &_severity,
+    const string                   &_confidence_level,
+    const string                   &_performance_impact,
+    const string                   &_last_update,
+    const string                   &_maintrain_id,
+    const vector<string>      &_tags,
+    const vector<string>      &_cve_list)
         :
     silent(_silent),
     protection_name(_protection_name),
@@ -394,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 ProtectionsSection::ProtectionsSection(
-    const std::vector<ProtectionsProtectionsSection>    &_protections,
-    const std::string                                   &_name,
-    const std::string                                   &_modification_time)
+    const vector<ProtectionsProtectionsSection>    &_protections,
+    const string                                   &_name,
+    const string                                   &_modification_time)
         :
     protections(_protections),
     name(_name),
@@ -460,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
+NewSnortSignatures::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
     parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
     parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
     parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode);
+    }
     is_temporary = false;
     if (valid_modes.count(override_mode) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
@@ -474,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
+NewSnortSignatures::addFile(const string &file_name)
 {
     files.push_back(file_name);
 }
 
 const string &
-NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
+NewSnortSignatures::getOverrideMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getFiles() const
+NewSnortSignatures::getFiles() const
 {
     return files;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const
+NewSnortSignatures::getConfigMap() const
 {
     return config_map;
 }
 
 bool
-NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const
+NewSnortSignatures::isTemporary() const
 {
     return is_temporary;
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val)
+NewSnortSignatures::setTemporary(bool val)
 {
     is_temporary = val;
 }
 
+void
+NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in)
+{
+    dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice";
+    parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
+    parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
+    parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    parseAppsecJSONKey<string>("enforcementLevel", enforcement_level, archive_in, "fullSchema");
+    if (valied_enforcement_level.count(enforcement_level) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level;
+        throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level);
+    }
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode);
+    }
+    for (const string &file : files)
+    {
+        auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<NewOpenApiSchema>();
+        auto file_content = i_orchestration_tools->readFile(file);
+        if (!file_content.ok()) {
+            dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file";
+            continue;
+        }
+        oas.push_back(Singleton::Consume<I_Encryptor>::by<NewOpenApiSchema>()->base64Encode(file_content.unpack()));
+    }
+}
+
+void
+NewOpenApiSchema::addOas(const string &file)
+{
+    oas.push_back(file);
+}
+
+const string &
+NewOpenApiSchema::getOverrideMode(const string &default_mode) const
+{
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2);
+    return res;
+}
+
+const string &
+NewOpenApiSchema::getEnforceLevel() const
+{
+    return enforcement_level;
+}
+
+const vector<string> &
+NewOpenApiSchema::getFiles() const
+{
+    return files;
+}
+
+const vector<string> &
+NewOpenApiSchema::getConfigMap() const
+{
+    return config_map;
+}
+
+const vector<string> &
+NewOpenApiSchema::getOas() const
+{
+    return oas;
+}
+
 void
 IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -548,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection(
 {
 }
 
-std::string &
+string &
 IpsProtectionsSection::getMode()
 {
     return mode;
@@ -570,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
     );
 }
 
+bool
+IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const
+{
+    // for sorting from the most specific to the least specific rule
+    if (name == default_appsec_name) return false;
+    if (other.name == default_appsec_name) return true;
+    return name.size() > other.name.size();
+}
+
+IPSSection::IPSSection(const vector<IpsProtectionsSection> &_ips) : ips(_ips)
+{
+    sort(ips.begin(), ips.end());
+}
+
 void
 IPSSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -648,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     vector<IpsProtectionsRulesSection> ips_rules;
     IpsProtectionsRulesSection high_rule(
         min_cve_Year,
-        getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(high_confidence_event_action, default_mode),
         string("High"),
         max_performance_impact,
         string(""),
@@ -658,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection med_rule(
         min_cve_Year,
-        getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(medium_confidence_event_action, default_mode),
         string("Medium"),
         max_performance_impact,
         string(""),
@@ -668,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection low_rule(
         min_cve_Year,
-        getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(low_confidence_event_action, default_mode),
         string("Low"),
         max_performance_impact,
         string(""),
@@ -679,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     return ips_rules;
 }
 
-const std::string &
-NewIntrusionPrevention::getMode(const std::string &default_mode) const
+const string &
+NewIntrusionPrevention::getMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
+const string &
+NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const
+{
+    if (isModeInherited(mode)) return default_mode;
+
+    if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) {
+        dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
+        return key_to_practices_mode_val.at("inactive");
+    }
+    return key_to_practices_mode_val.at(mode);
+}
+
 FileSecurityProtectionsSection::FileSecurityProtectionsSection(
     uint64_t                _file_size_limit,
     uint64_t                _archive_file_size_limit,
     bool                    _allow_files_without_name,
     bool                    _required_file_size_limit,
     bool                    _required_archive_extraction,
-    const std::string       &_context,
-    const std::string       &_name,
-    const std::string       &_asset_id,
-    const std::string       &_practice_name,
-    const std::string       &_practice_id,
-    const std::string       &_action,
-    const std::string       &_files_without_name_action,
-    const std::string       &_high_confidence_action,
-    const std::string       &_medium_confidence_action,
-    const std::string       &_low_confidence_action,
-    const std::string       &_severity_level,
-    const std::string       &_file_size_limit_action,
-    const std::string       &_multi_level_archive_action,
-    const std::string       &_unopened_archive_action)
+    const string       &_context,
+    const string       &_name,
+    const string       &_asset_id,
+    const string       &_practice_name,
+    const string       &_practice_id,
+    const string       &_action,
+    const string       &_files_without_name_action,
+    const string       &_high_confidence_action,
+    const string       &_medium_confidence_action,
+    const string       &_low_confidence_action,
+    const string       &_severity_level,
+    const string       &_file_size_limit_action,
+    const string       &_multi_level_archive_action,
+    const string       &_unopened_archive_action)
         :
     file_size_limit(_file_size_limit),
     archive_file_size_limit(_archive_file_size_limit),
@@ -831,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const
     return extract_archive_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const
 {
     return archived_files_within_archived_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const
 {
     return archived_files_where_content_extraction_failed;
@@ -886,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const
     return (file_size_limit * unit_to_int.at(file_size_limit_unit));
 }
 
-const std::string &
+const string &
 NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const
 {
     return files_exceeding_size_limit_action;
@@ -1007,7 +1102,7 @@ void
 NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
-    parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
+    parseAppsecJSONKey<NewOpenApiSchema>(
         "schemaValidation",
         openapi_schema_validation,
         archive_in
@@ -1015,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
     parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
-    parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
+    parseMandatoryAppsecJSONKey<NewSnortSignatures>("snortSignatures", snort_signatures, archive_in);
     parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
     parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
     parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode);
+    }
 }
 
 void
@@ -1028,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name)
     practice_name = _name;
 }
 
-const NewSnortSignaturesAndOpenSchemaAPI &
-NewAppSecPracticeSpec::getOpenSchemaValidation() const
+NewOpenApiSchema &
+NewAppSecPracticeSpec::getOpenSchemaValidation()
 {
     return openapi_schema_validation;
 }
 
-NewSnortSignaturesAndOpenSchemaAPI &
+NewSnortSignatures &
 NewAppSecPracticeSpec::getSnortSignatures()
 {
     return snort_signatures;

components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc

@@ -23,6 +23,14 @@ using namespace std;
 USE_DEBUG_FLAG(D_NGINX_POLICY);
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
+static const std::unordered_map<std::string, std::string> key_to_source_identefier_val = {
+    { "sourceip", "Source IP"},
+    { "cookie", "Cookie:"},
+    { "headerkey", "Header:"},
+    { "JWTKey", ""},
+    { "x-forwarded-for", "X-Forwarded-For"}
+};
+
 void
 SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -1038,7 +1046,7 @@ PolicyMakerUtils::createIpsSections(
         practice_name,
         practice_id,
         source_identifier,
-        override_mode,
+        "Inactive",
         apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
     );
 
@@ -1048,8 +1056,7 @@ PolicyMakerUtils::createIpsSections(
 void
 PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary)
 {
-    auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name;
-    string in_file = is_temporary ? path + ".rule" : path;
+    auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name;
 
     if (snort_protections.find(path) != snort_protections.end()) {
         dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists";
@@ -1060,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         << (is_temporary ? " temporary" : "") << " file " << path;
 
     auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
-    auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err";
+    auto tmp_out = "/tmp/" + file_name + ".out";
+    auto tmp_err = "/tmp/" + file_name + ".err";
+    auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err;
 
     auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
 
@@ -1069,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         return;
     }
 
-    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(path + ".out");
+    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(tmp_out);
     if (!maybe_protections.ok()){
         dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr();
         return;
     }
 
     auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
-    if (is_temporary) i_orchestration_tools->removeFile(in_file);
-    i_orchestration_tools->removeFile(path + ".out");
-    i_orchestration_tools->removeFile(path + ".err");
+    if (is_temporary) i_orchestration_tools->removeFile(path);
+    i_orchestration_tools->removeFile(tmp_out);
+    i_orchestration_tools->removeFile(tmp_err);
 
     snort_protections[path] = ProtectionsSection(
         maybe_protections.unpack().getProtections(),
@@ -1208,7 +1217,8 @@ void
 PolicyMakerUtils::createWebAppSection(
     const V1beta2AppsecLinuxPolicy &policy,
     const RulesConfigRulebase& rule_config,
-    const string &practice_id, const string &full_url,
+    const string &practice_id,
+    const string &full_url,
     const string &default_mode,
     map<AnnotationTypes, string> &rule_annotations)
 {
@@ -1225,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection(
         apssec_practice.getWebAttacks().getMaxObjectDepth(),
         apssec_practice.getWebAttacks().getMaxUrlSizeBytes()
     );
+
     WebAppSection web_app = WebAppSection(
         full_url == "Any" ? default_appsec_url : full_url,
         rule_config.getAssetId(),
@@ -1236,7 +1247,10 @@ PolicyMakerUtils::createWebAppSection(
         rule_config.getContext(),
         apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode),
         apssec_practice.getWebAttacks().getMode(practice_mode),
-        apssec_practice.getAntiBot().getMode(),
+        apssec_practice.getAntiBot().getMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getEnforceLevel(),
+        apssec_practice.getOpenSchemaValidation().getOas(),
         practice_advance_config,
         apssec_practice.getAntiBot(),
         log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
@@ -1290,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
     );
     rules_config[rule_config.getAssetName()] = rule_config;
 
-    string current_identifier;
+    string current_identifier, current_identifier_value;
     if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) {
         UsersIdentifiersRulebase user_identifiers = createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
             rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS],
@@ -1299,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         );
         users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers;
         current_identifier = user_identifiers.getIdentifier();
+        current_identifier_value = user_identifiers.getIdentifierValue();
+    }
+
+    string ips_identifier, ips_identifier_value;
+    if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) {
+        ips_identifier = key_to_source_identefier_val.at(current_identifier);
+    }
+    if (current_identifier == "cookie" || current_identifier == "headerkey") {
+        ips_identifier_value = current_identifier_value;
     }
 
     createIpsSections(
@@ -1306,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         rule_config.getAssetName(),
         practice_id,
         rule_annotations[AnnotationTypes::PRACTICE],
-        current_identifier,
+        ips_identifier + ips_identifier_value,
         rule_config.getContext(),
         policy,
         rule_annotations,

components/security_apps/local_policy_mgmt_gen/rules_config_section.cc

@@ -17,6 +17,8 @@ using namespace std;
 
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
+static const string empty_string="";
+
 AssetUrlParser
 AssetUrlParser::parse(const string &uri)
 {
@@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const
 {
     return source_identifier;
 }
+
+const string &
+UsersIdentifier::getIdentifierValue() const
+{
+    if (identifier_values.empty()) return empty_string;
+    return identifier_values[0];
+}
 // LCOV_EXCL_STOP
 
 void
@@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const
     if (source_identifiers.empty()) return source_identifier;
     return source_identifiers[0].getIdentifier();
 }
+
+const string &
+UsersIdentifiersRulebase::getIdentifierValue() const
+{
+    if (source_identifiers.empty()) return empty_string;
+    return source_identifiers[0].getIdentifierValue();
+}
 // LCOV_EXCL_STOP
 
 void

components/security_apps/orchestration/details_resolver/details_resolver.cc

@@ -350,7 +350,7 @@ DetailsResolver::Impl::readCloudMetadata()
     }
 
     if (!cloud_metadata.ok()) {
-        dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr();
+        dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
         return genError("Failed to fetch cloud metadata");
     }
 

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -18,6 +18,8 @@
 #include <regex>
 #include <boost/regex.hpp>
 #include <boost/algorithm/string.hpp>
+#include <cereal/external/rapidjson/document.h>
+#include <cereal/external/rapidjson/filereadstream.h>
 
 #if defined(gaia)
 
@@ -100,6 +102,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
     return command_output;
 }
 
+Maybe<string>
+getOtlpAgentGaiaOsRole(const string &command_output)
+{
+    if (command_output == "" ) return string("-1");
+
+    return command_output;
+}
+
 Maybe<string>
 getQUID(const string &command_output)
 {
@@ -111,6 +121,13 @@ getQUID(const string &command_output)
     return command_output;
 }
 
+Maybe<string>
+getIsAiopsRunning(const string &command_output)
+{
+    if (command_output == "" ) return string("false");
+    
+    return command_output;
+}
 
 Maybe<string>
 checkHasSDWan(const string &command_output)
@@ -186,6 +203,24 @@ getMgmtObjAttr(shared_ptr<istream> file_stream, const string &attr)
     return genError("Object attribute was not found. Attr: " + attr);
 }
 
+Maybe<string>
+getAttrFromCpsdwanGetDataJson(const string &attr)
+{
+    static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json";
+    std::ifstream ifs(get_data_json_path);
+    if (ifs.is_open()) {
+        rapidjson::IStreamWrapper isw(ifs);
+        rapidjson::Document document;
+        document.ParseStream(isw);
+
+        if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) {
+            return string(document[attr.c_str()].GetString());
+        }
+    }
+
+    return genError("Attribute " + attr + " was not found in " + get_data_json_path);
+}
+
 Maybe<string>
 getMgmtObjUid(const string &command_output)
 {
@@ -193,6 +228,11 @@ getMgmtObjUid(const string &command_output)
         return command_output;
     }
 
+    Maybe<string> obj_uuid = getAttrFromCpsdwanGetDataJson("uuid");
+    if (obj_uuid.ok()) {
+        return obj_uuid.unpack();
+    }
+
     static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
     auto file_stream = std::make_shared<std::ifstream>(obj_path);
     if (!file_stream->is_open()) {
@@ -310,7 +350,12 @@ getSmbObjectName(const string &command_output)
     if (command_output.empty() || command_output[0] != centrally_managed_comd_output) {
         return genError("Object name was not found");
     }
-    
+
+    Maybe<string> obj_name = getAttrFromCpsdwanGetDataJson("name");
+    if (obj_name.ok()) {
+        return obj_name.unpack();
+    }
+
     static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
     auto ifs = std::make_shared<std::ifstream>(obj_path);
     if (!ifs->is_open()) {

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -66,8 +66,22 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
     "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
     getQUID)
 SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
-    "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''",
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''",
     getQUID)
+SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
+    "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
+    getOtlpAgentGaiaOsRole)
+SHELL_CMD_HANDLER(
+    "IS_AIOPS_RUNNING",
+    "FS_PATH=<FILESYSTEM-PREFIX>; "
+    "PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
+    "[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
+    getIsAiopsRunning)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(
     "canUpdateSDWanData",
@@ -175,8 +189,7 @@ SHELL_CMD_HANDLER(
 )
 SHELL_CMD_HANDLER(
     "managements",
-    "sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |"
-    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    "echo 1",
     extractManagements
 )
 #endif //gaia
@@ -232,8 +245,7 @@ SHELL_CMD_HANDLER(
 
 SHELL_CMD_HANDLER(
     "managements",
-    "sed -n '/:masters (/,$p' /tmp/local.cfg |"
-    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    "echo 1",
     extractManagements
 )
 #endif//smb

components/security_apps/orchestration/env_details/env_details.cc

@@ -28,6 +28,7 @@ EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
     auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
     if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
     token = retrieveToken();
+    agent_namespace = retrieveNamespace();
     if (!token.empty()) {
         auto env_res = getenv("deployment_type");
         env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
@@ -46,12 +47,24 @@ EnvDetails::getToken()
     return token;
 }
 
+string
+EnvDetails::getNameSpace()
+{
+    return agent_namespace;
+}
+
 string
 EnvDetails::retrieveToken()
 {
     return readFileContent(k8s_service_account + "/token");
 }
 
+string
+EnvDetails::retrieveNamespace()
+{
+    return readFileContent(k8s_service_account + "/namespace");
+}
+
 string
 EnvDetails::readFileContent(const string &file_path)
 {

components/security_apps/orchestration/hybrid_mode_telemetry.cc

@@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &)
 {
     auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
     auto maybe_cmd_output = shell_cmd->getExecOutput(
-        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count"
+        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count",
+        1000,
+        false
     );
 
     // get wd process restart count

components/security_apps/orchestration/include/declarative_policy_utils.h

@@ -79,8 +79,8 @@ public:
     ) override;
     std::string getUpdate(CheckUpdateRequest &request) override;
     bool shouldApplyPolicy() override;
-    void turnOffApplyPolicyFlag() override;
-    void turnOnApplyPolicyFlag() override;
+    void turnOffApplyLocalPolicyFlag() override;
+    void turnOnApplyLocalPolicyFlag() override;
 
     std::string getCurrPolicy() override { return curr_policy; }
 
@@ -94,7 +94,7 @@ private:
     std::string curr_version;
     std::string curr_policy;
     std::string curr_checksum;
-    bool should_apply_policy;
+    bool should_apply_local_policy;
 };
 
 #endif // __DECLARATIVE_POLICY_UTILS_H__

components/security_apps/orchestration/include/i_declarative_policy.h

@@ -22,8 +22,8 @@ public:
 
     virtual std::string getCurrPolicy() = 0;
 
-    virtual void turnOffApplyPolicyFlag() = 0;
-    virtual void turnOnApplyPolicyFlag() = 0;
+    virtual void turnOffApplyLocalPolicyFlag() = 0;
+    virtual void turnOnApplyLocalPolicyFlag() = 0;
 
 protected:
     virtual ~I_DeclarativePolicy() {}

components/security_apps/orchestration/orchestration_comp.cc

@@ -1485,11 +1485,10 @@ private:
     }
 
     void
-    setUpgradeTime()
+    setDelayedUpgradeTime()
     {
         if (getConfigurationFlag("service_startup") != "true") return;
-        if (i_service_controller->getServiceToPortMap().empty()) return;
-
+        if (!i_agent_details->isOpenAppsecAgent() && i_service_controller->getServiceToPortMap().empty()) return;
         try {
             string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN");
             int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30;
@@ -1506,6 +1505,7 @@ private:
     void
     run()
     {
+        loadExistingPolicy();
         sleep_interval = policy.getErrorSleepInterval();
         Maybe<void> registration_status(genError("Not running yet."));
         while (!(registration_status = registerToTheFog()).ok()) {
@@ -1530,7 +1530,6 @@ private:
                 << " seconds";
             Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(seconds(sleep_interval));
         }
-        loadExistingPolicy();
         failure_count = 0;
 
         Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(chrono::seconds(1));
@@ -1587,7 +1586,7 @@ private:
             ).notify();
         }
 
-        setUpgradeTime();
+        setDelayedUpgradeTime();
         while (true) {
             Singleton::Consume<I_Environment>::by<OrchestrationComp>()->startNewTrace(false);
             if (shouldReportAgentDetailsMetadata()) {
@@ -2034,7 +2033,7 @@ private:
         }
         auto policy_mgmt_mode = getSettingWithDefault<string>("management", "profileManagedMode");
         if (getOrchestrationMode() == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative") {
-            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyPolicyFlag();
+            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyLocalPolicyFlag();
         }
 
         auto policy_version = i_service_controller->getPolicyVersion();

components/security_apps/orchestration/service_controller/service_controller.cc

@@ -793,7 +793,7 @@ ServiceController::Impl::updateServiceConfiguration(
             << "Policy file was not updated. Sending reload command regarding settings and data";
         auto signal_services = sendSignalForServices(nano_services_to_update, "");
         if (!signal_services.ok()) return signal_services.passErr();
-        Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+        Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
         return Maybe<void>();
     }
 
@@ -940,7 +940,7 @@ ServiceController::Impl::updateServiceConfiguration(
         if (new_policy_path.compare(config_file_path) == 0) {
             dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file";
             policy_version = version_value;
-            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
             return Maybe<void>();
         }
 
@@ -959,7 +959,7 @@ ServiceController::Impl::updateServiceConfiguration(
     }
 
     if (!was_policy_updated && !send_signal_for_services_err.empty()) return genError(send_signal_for_services_err);
-    Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+    Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
     return Maybe<void>();
 }
 

components/security_apps/orchestration/update_communication/declarative_policy_utils.cc

@@ -17,7 +17,7 @@ void
 DeclarativePolicyUtils::init()
 {
     local_policy_path = getFilesystemPathConfig() + "/conf/local_policy.yaml";
-    should_apply_policy = true;
+    should_apply_local_policy = true;
     Singleton::Consume<I_RestApi>::by<DeclarativePolicyUtils>()->addRestCall<ApplyPolicyRest>(
         RestAction::SET, "apply-policy"
     );
@@ -40,7 +40,7 @@ DeclarativePolicyUtils::upon(const ApplyPolicyEvent &event)
 {
     dbgTrace(D_ORCHESTRATOR) << "Apply policy event";
     local_policy_path = event.getPolicyPath();
-    should_apply_policy = true;
+    should_apply_local_policy = true;
 }
 // LCOV_EXCL_STOP
 
@@ -48,19 +48,24 @@ bool
 DeclarativePolicyUtils::shouldApplyPolicy()
 {
     auto env_type = Singleton::Consume<I_EnvDetails>::by<DeclarativePolicyUtils>()->getEnvType();
-    return env_type == EnvType::K8S ? true : should_apply_policy;
+    if (env_type == EnvType::K8S) {
+        I_OrchestrationTools *orch_tools = Singleton::Consume<I_OrchestrationTools>::by<DeclarativePolicyUtils>();
+        auto maybe_new_version = orch_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger");
+        return maybe_new_version != curr_version;
+    }
+    return should_apply_local_policy;
 }
 
 void
-DeclarativePolicyUtils::turnOffApplyPolicyFlag()
+DeclarativePolicyUtils::turnOffApplyLocalPolicyFlag()
 {
-    should_apply_policy = false;
+    should_apply_local_policy = false;
 }
 
 void
-DeclarativePolicyUtils::turnOnApplyPolicyFlag()
+DeclarativePolicyUtils::turnOnApplyLocalPolicyFlag()
 {
-    should_apply_policy = true;
+    should_apply_local_policy = true;
 }
 
 Maybe<string>
@@ -211,6 +216,6 @@ DeclarativePolicyUtils::periodicPolicyLoad()
 
     if (*new_checksum == curr_checksum) return;
 
-    should_apply_policy = true;
+    should_apply_local_policy = true;
     curr_checksum = *new_checksum;
 }

components/security_apps/orchestration/update_communication/fog_communication.cc

@@ -74,7 +74,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
                 << " to: "
                 << policy_mgmt_mode;
             profile_mode = policy_mgmt_mode;
-            i_declarative_policy->turnOnApplyPolicyFlag();
+            i_declarative_policy->turnOnApplyLocalPolicyFlag();
         }
 
         if (i_declarative_policy->shouldApplyPolicy()) {

components/security_apps/rate_limit/rate_limit.cc

@@ -293,8 +293,13 @@ public:
             return ACCEPT;
         }
 
-        burst = rule.getRateLimit();
-        limit = calcRuleLimit(rule);
+        auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1;
+        if (replicas == 0) {
+            dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1";
+            replicas = 1;
+        }
+        burst = static_cast<float>(rule.getRateLimit()) / replicas;
+        limit = static_cast<float>(calcRuleLimit(rule)) / replicas;
 
         dbgTrace(D_RATE_LIMIT)
             << "found rate limit rule with: "

components/security_apps/waap/include/i_transaction.h

@@ -70,6 +70,7 @@ public:
     virtual const std::string getParam() const = 0;
     virtual const std::vector<std::string> getKeywordMatches() const = 0;
     virtual const std::vector<std::string> getKeywordsCombinations() const = 0;
+    virtual const std::vector<std::string> getKeywordsAfterFilter() const = 0;
     virtual const std::string getContentTypeStr() const = 0;
     virtual Waap::Util::ContentType getContentType() const = 0;
     virtual const std::string getKeywordMatchesStr() const = 0;
@@ -84,6 +85,7 @@ public:
     virtual const std::string getUriStr() const = 0;
     virtual const std::string& getSourceIdentifier() const = 0;
     virtual double getScore() const = 0;
+    virtual double getOtherModelScore() const = 0;
     virtual const std::vector<double> getScoreArray() const = 0;
     virtual Waap::CSRF::State& getCsrfState() = 0;
     virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;

components/security_apps/waap/waap_clib/CMakeLists.txt

@@ -87,7 +87,10 @@ add_library(waap_clib
     ParserPairs.cc
     Waf2Util2.cc
     ParserPDF.cc
+    ParserKnownBenignSkipper.cc
+    ParserScreenedJson.cc
     ParserBinaryFile.cc
+    RegexComparator.cc
 )
 
 add_definitions("-Wno-unused-function")

components/security_apps/waap/waap_clib/CidrMatch.cc

@@ -25,6 +25,51 @@
 #include "log_generator.h"
 #include <stdexcept>
 
+static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) {
+    in_addr maskedAddr;
+    if (prefixLength == 0) {
+        maskedAddr.s_addr = 0;
+    } else {
+        uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1));  // Create mask
+        maskedAddr.s_addr = addr.s_addr & mask;  // Apply mask
+    }
+    return maskedAddr;
+}
+
+// Function to apply a network mask to an IPv6 address
+static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) {
+    in6_addr maskedAddr = addr;
+    int fullBytes = prefixLength / 8;
+    int remainingBits = prefixLength % 8;
+
+    // Mask full bytes
+    for (int i = fullBytes; i < 16; ++i) {
+        maskedAddr.s6_addr[i] = 0;
+    }
+
+    // Mask remaining bits
+    if (remainingBits > 0) {
+        uint8_t mask = ~((1 << (8 - remainingBits)) - 1);
+        maskedAddr.s6_addr[fullBytes] &= mask;
+    }
+
+    return maskedAddr;
+}
+
+// Helper function to convert an IPv4 address to string
+static std::string ipv4ToString(const in_addr& ipv4) {
+    char str[INET_ADDRSTRLEN];
+    inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN);
+    return std::string(str);
+}
+
+// Helper function to convert an IPv6 address to string
+static std::string ipv6ToString(const in6_addr& ipv6) {
+    char str[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN);
+    return std::string(str);
+}
+
 USE_DEBUG_FLAG(D_WAAP);
 namespace Waap {
 namespace Util {
@@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const {
         isIPV6 == other.isIPV6;
 }
 
+bool CIDRData::operator<(const CIDRData &other) const {
+    if (isIPV6) {
+        if (!other.isIPV6) return false;
+        return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0;
+    }
+    if (other.isIPV6) return true;
+    return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr);
+}
+
 bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) {
     if (bits == 0) {
         // C99 6.5.7 (3): u32 << 32 is undefined behaviour
@@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
     memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr));
 
     if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) {
+        cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits);
         cidr.isIPV6 = false;
     }
     else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) {
+        cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits);
         cidr.isIPV6 = true;
     }
     else
@@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
 
     return true;
 }
+
 bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
     CIDRData cidrData;
 
@@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
 
     return cidrMatch(sourceip, cidrData);
 }
+
 bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
     struct in_addr source_inaddr;
     struct in6_addr source_inaddr6;
@@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
     dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions.";
     return false;
 }
+
+bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) {
+    if (first.isIPV6 != second.isIPV6) return false;  // IPv4 and IPv6 cannot overlap
+    if (first.networkBits >= second.networkBits) return false;
+
+    if (!first.isIPV6) {
+        // IPv4 containment check
+        in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits);
+        return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr);
+    }
+    // IPv6 containment check
+    in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits);
+
+    for (int i = 0; i < 16; ++i) {
+        if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+std::string cidrsToString(const std::vector<CIDRData>& cidrs) {
+    std::stringstream ss;
+    bool is_first = true;
+    ss << "[";
+    for (const auto& cidr : cidrs) {
+        if (!is_first) ss << ", ";
+        if (cidr.isIPV6) {
+            ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast<int>(cidr.networkBits);
+        } else {
+            ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast<int>(cidr.networkBits);
+        }
+        is_first = false;
+    }
+    ss << "]";
+    return ss.str();
+}
+
 }
 }

components/security_apps/waap/waap_clib/CidrMatch.h

@@ -18,6 +18,7 @@
 #include <sys/socket.h>
 #include <arpa/inet.h>
 #include <errno.h>
+#include <vector>
 
 namespace Waap {
 namespace Util {
@@ -29,11 +30,14 @@ struct CIDRData {
     uint8_t networkBits;
     bool isIPV6;
     bool operator==(const CIDRData &other) const;
+    bool operator<(const CIDRData &other) const;
 };
 
 bool isCIDR(const std::string& strCIDR, CIDRData& cidr);
 bool cidrMatch(const std::string& sourceip, const CIDRData& cidr);
 bool cidrMatch(const std::string &sourceip, const std::string &target);
+bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second);
+std::string cidrsToString(const std::vector<CIDRData>& cidrs);
 
 }
 }

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -28,6 +28,8 @@
 #include "ParserDelimiter.h"
 #include "ParserPDF.h"
 #include "ParserBinaryFile.h"
+#include "ParserKnownBenignSkipper.h"
+#include "ParserScreenedJson.h"
 #include "WaapAssetState.h"
 #include "Waf2Regex.h"
 #include "Waf2Util.h"
@@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
             isRefererParamPayload,
             isUrlPayload,
             isUrlParamPayload,
+            isCookiePayload,
             flags,
             parser_depth,
             base64BinaryFileType
@@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
             isRefererParamPayload,
             isUrlPayload,
             isUrlParamPayload,
+            isCookiePayload,
             flags,
             parser_depth,
             base64BinaryFileType
@@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
                 isRefererParamPayload,
                 isUrlPayload,
                 isUrlParamPayload,
+                isCookiePayload,
                 flags,
                 parser_depth,
                 base64ParamFound,
@@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
     bool isRefererParamPayload,
     bool isUrlPayload,
     bool isUrlParamPayload,
+    bool isCookiePayload,
     int flags,
     size_t parser_depth,
     bool base64ParamFound,
@@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
             isRefererParamPayload,
             isUrlPayload,
             isUrlParamPayload,
+            isCookiePayload,
             flags,
             parser_depth,
             b64FileType
@@ -918,6 +925,7 @@ bool isRefererPayload,
 bool isRefererParamPayload,
 bool isUrlPayload,
 bool isUrlParamPayload,
+bool isCookiePayload,
 int flags,
 size_t parser_depth
 ) {
@@ -959,6 +967,7 @@ DeepParser::createInternalParser(
     bool isRefererParamPayload,
     bool isUrlPayload,
     bool isUrlParamPayload,
+    bool isCookiePayload,
     int flags,
     size_t parser_depth,
     Waap::Util::BinaryFileType b64FileType
@@ -978,7 +987,19 @@ DeepParser::createInternalParser(
         << "\n\tflags: "
         << flags
         << "\n\tparser_depth: "
-        << parser_depth;
+        << parser_depth
+        << "\n\tisBodyPayload: "
+        << isBodyPayload
+        << "\n\tisRefererPayload: "
+        << isRefererPayload
+        << "\n\tisRefererParamPayload: "
+        << isRefererParamPayload
+        << "\n\tisUrlPayload: "
+        << isUrlPayload
+        << "\n\tisUrlParamPayload: "
+        << isUrlParamPayload
+        << "\n\tisCookiePayload: "
+        << isCookiePayload;
     bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false,
         isAmperType = false;
     bool isKeyValDelimited = false;
@@ -1045,6 +1066,53 @@ DeepParser::createInternalParser(
         }
     }
 
+    if (Waap::Util::isScreenedJson(cur_val)) {
+        dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON";
+        m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserScreenedJson>>(*this, parser_depth + 1));
+        offset = 0;
+        return offset;
+    }
+
+    dbgTrace(D_WAAP_DEEP_PARSER)
+        << "Offset = "
+        << offset
+        << " depth = "
+        << m_depth
+        << " isBodyPayload = "
+        << isBodyPayload;
+    //Detect sensor_data format in body and just use dedicated filter for it
+    if (m_depth == 1
+        && isBodyPayload
+        && Waap::Util::detectKnownSource(cur_val) ==  Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
+        m_parsersDeque.push_back(
+            std::make_shared<BufferedParser<ParserKnownBenignSkipper>>(
+                *this,
+                parser_depth + 1,
+                Waap::Util::SOURCE_TYPE_SENSOR_DATA
+            )
+        );
+        offset = 0;
+        dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it";
+        return offset;
+    }
+    // Detect cookie parameter sensorsdata2015jssdkcross
+    // and causes false positives due to malformed JSON. Make preprocessing to parse it correctly
+    if (m_depth == 2
+        && isCookiePayload) {
+        offset = Waap::Util::definePrefixedJson(cur_val);
+        if (offset >= 0) {
+            m_parsersDeque.push_back(
+                std::make_shared<BufferedParser<ParserJson>>(
+                    *this,
+                    parser_depth + 1,
+                    m_pTransaction
+                )
+            );
+            dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data";
+            return offset;
+        }
+    }
+
     // Detect wbxml (binary XML) data type
     if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) {
         m_is_wbxml = true;
@@ -1308,7 +1376,7 @@ DeepParser::createInternalParser(
                         *this,
                         parser_depth + 1,
                         '&',
-                        valueStats.isUrlEncoded)
+                        valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
                 );
             } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
                 dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)";
@@ -1323,7 +1391,7 @@ DeepParser::createInternalParser(
                             *this,
                             parser_depth + 1,
                             '&',
-                            valueStats.isUrlEncoded)
+                            valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
                     );
                     offset = 0;
                     return offset;
@@ -1374,6 +1442,7 @@ DeepParser::createInternalParser(
             isRefererParamPayload,
             isUrlPayload,
             isUrlParamPayload,
+            isCookiePayload,
             flags,
             parser_depth
             );
@@ -1545,5 +1614,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const
     static const std::string PDF_header("%PDF-");
     if (cur_val.size() < 10)
         return false;
-    return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
+    return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size())
+                .find(PDF_header) != std::string::npos;
 }

components/security_apps/waap/waap_clib/DeepParser.h

@@ -129,6 +129,7 @@ private:
         bool isRefererParamPayload,
         bool isUrlPayload,
         bool isUrlParamPayload,
+        bool isCookiePayload,
         int flags,
         size_t parser_depth,
         Waap::Util::BinaryFileType b64FileType
@@ -144,6 +145,7 @@ private:
         bool isRefererParamPayload,
         bool isUrlPayload,
         bool isUrlParamPayload,
+        bool isCookiePayload,
         int flags,
         size_t parser_depth
     );
@@ -160,6 +162,7 @@ private:
         bool isRefererParamPayload,
         bool isUrlPayload,
         bool isUrlParamPayload,
+        bool isCookiePayload,
         int flags,
         size_t parser_depth,
         bool base64ParamFound,

components/security_apps/waap/waap_clib/ParserBinaryFile.cc

@@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len)
                     }
                 } else {
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
-                    c = strstr(buf + len - tail.size(), tail.c_str());
+                    size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
+                    c = strstr(buf + tail_lookup_offset, tail.c_str());
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
                     if (c) {
                         m_state = s_end;

components/security_apps/waap/waap_clib/ParserBinaryFile.h

@@ -20,7 +20,7 @@
 
 #define MIN_HEADER_LOOKUP 16
 #define MAX_HEADER_LOOKUP 64
-#define MAX_TAIL_LOOKUP 5
+#define MAX_TAIL_LOOKUP 20
 
 class ParserBinaryFile : public ParserBase {
 public:

components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.cc

@@ -0,0 +1,139 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ParserKnownBenignSkipper.h"
+#include "Waf2Util.h"
+#include "debug.h"
+#include <string.h>
+USE_DEBUG_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER);
+USE_DEBUG_FLAG(D_WAAP);
+
+const std::string ParserKnownBenignSkipper::m_parserName = "ParserKnownBenignSkipper";
+const char* DATA_SENSOR_TAIL = "\"}";
+
+ParserKnownBenignSkipper::ParserKnownBenignSkipper(
+    IParserStreamReceiver &receiver,
+    size_t parser_depth,
+    Waap::Util::KnownSourceType source_type
+) :
+    m_receiver(receiver),
+    m_state(s_start),
+    m_parser_depth(parser_depth),
+    m_source_type(source_type)
+{}
+
+ParserKnownBenignSkipper::~ParserKnownBenignSkipper()
+{}
+
+size_t
+ParserKnownBenignSkipper::push(const char *buf, size_t len)
+{
+    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+        << "buf='"
+        << std::string(buf, std::min((size_t)200, len))
+        << (len > 200 ? "..." : "")
+        << "' len="
+        << len
+        << " depth="
+        << depth();
+
+    const char *c;
+
+    if (m_state == s_error) {
+        return 0;
+    }
+    if (len == 0)
+    {
+        dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+            << "ParserKnownBenignSkipper::push(): end of stream. m_state="
+            << m_state;
+
+        if (m_state == s_end) {
+            m_receiver.onKvDone();
+        } else {
+            m_state = s_error;
+        }
+        return 0;
+    }
+
+    size_t tail_lookup_offset = 0;
+
+    switch (m_state) {
+            case s_start:
+                m_state = s_body;
+                CP_FALL_THROUGH;
+            case s_body:
+                {
+                    if (m_source_type == Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
+                        tail_lookup_offset =
+                            (len > MAX_DATA_SENSOR_TAIL_LOOKUP) ? len - MAX_DATA_SENSOR_TAIL_LOOKUP : 0;
+                        c = strstr(buf + tail_lookup_offset, DATA_SENSOR_TAIL);
+                        if (c) {
+                            dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                                << "ParserKnownBenignSkipper::push(): found end of sensor data";
+                            m_state = s_end;
+                            CP_FALL_THROUGH;
+                        } else {
+                            break;
+                        }
+                    } else {
+                        dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                            << "ParserKnownBenignSkipper::push(): unknown source type";
+                        m_state = s_error;
+                        break;
+                    }
+                }
+            case s_end:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state = end";
+                if (m_receiver.onKey("SENSOR_DATA", 11) != 0) {
+                    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state moving to error onKey";
+                    m_state = s_error;
+                    return 0;
+                }
+                if (m_receiver.onValue("", 0) != 0) {
+                    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state moving to error onValue";
+                    m_state = s_error;
+                    return 0;
+                }
+                break;
+            case s_error:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state = error";
+                break;
+            default:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                    << "ParserKnownBenignSkipper::push(): unknown state: "
+                    << m_state;
+                m_state = s_error;
+                return 0;
+    }
+    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+        << "ParserKnownBenignSkipper::push(): final state: "
+        << m_state;
+    return len;
+}
+
+
+void ParserKnownBenignSkipper::finish()
+{
+    push(NULL, 0);
+}
+
+const std::string& ParserKnownBenignSkipper::name() const
+{
+    return m_parserName;
+}
+
+bool ParserKnownBenignSkipper::error() const
+{
+    return m_state == s_error;
+}

components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.h

@@ -0,0 +1,52 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_BENIGN_SKIPPER_H__
+#define __PARSER_BENIGN_SKIPPER_H__
+
+#include "ParserBase.h"
+#include "Waf2Util.h"
+#include <string.h>
+#include "Waf2Util.h"
+
+#define MAX_DATA_SENSOR_TAIL_LOOKUP 5
+
+class ParserKnownBenignSkipper : public ParserBase {
+public:
+    ParserKnownBenignSkipper(
+        IParserStreamReceiver &receiver,
+        size_t parser_depth,
+        Waap::Util::KnownSourceType source_type=Waap::Util::SOURCE_TYPE_UNKNOWN);
+    virtual ~ParserKnownBenignSkipper();
+    virtual size_t push(const char *buf, size_t len);
+    virtual void finish();
+    virtual const std::string &name() const;
+    virtual bool error() const;
+    virtual size_t depth() { return 1; }
+
+private:
+    enum state {
+        s_start,
+        s_body,
+        s_end,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    enum state m_state;
+    static const std::string m_parserName;
+    size_t m_parser_depth;
+    Waap::Util::KnownSourceType m_source_type;
+};
+
+#endif // __PARSER_BENIGN_SKIPPER_H__

components/security_apps/waap/waap_clib/ParserPDF.cc

@@ -38,9 +38,10 @@ size_t
 ParserPDF::push(const char *buf, size_t len)
 {
     dbgTrace(D_WAAP_PARSER_PDF)
-        << "buf="
-        << buf
-        << "len="
+        << "buf='"
+        << std::string(buf, std::min((size_t)200, len))
+        << (len > 200 ? "..." : "")
+        << "' len="
         << len;
 
     const char *c;
@@ -65,13 +66,18 @@ ParserPDF::push(const char *buf, size_t len)
                 m_state = s_body;
                 CP_FALL_THROUGH;
             case s_body:
-                c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
-                dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
-                if (c) {
-                    m_state = s_end;
-                    CP_FALL_THROUGH;
-                } else {
-                    break;
+                {
+                    size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0;
+                    c = strstr(buf + tail_lookup_offset, PDF_TAIL);
+                    dbgTrace(D_WAAP_PARSER_PDF)
+                        << "string to search: " << std::string(buf + tail_lookup_offset)
+                        << " c=" << c;
+                    if (c) {
+                        m_state = s_end;
+                        CP_FALL_THROUGH;
+                    } else {
+                        break;
+                    }
                 }
             case s_end:
                 if (m_receiver.onKey("PDF", 3) != 0) {

components/security_apps/waap/waap_clib/ParserPDF.h

@@ -17,8 +17,8 @@
 #include "ParserBase.h"
 #include <string.h>
 
-#define MAX_HEADER_LOOKUP 64
-#define MAX_TAIL_LOOKUP 5
+#define MAX_PDF_HEADER_LOOKUP 64
+#define MAX_PDF_TAIL_LOOKUP 20
 
 class ParserPDF : public ParserBase {
 public:

components/security_apps/waap/waap_clib/ParserScreenedJson.cc

@@ -0,0 +1,187 @@
+#include "ParserScreenedJson.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_WAAP_PARSER_SCREENED_JSON);
+
+const std::string ParserScreenedJson::m_parserName = "ParserScreenedJson";
+
+ParserScreenedJson::ParserScreenedJson(IParserStreamReceiver &receiver, size_t parser_depth) :
+    m_receiver(receiver),
+    m_state(s_start),
+    m_unscreenedLen(0),
+    m_leftoverLen(0),
+    m_parser_depth(parser_depth)
+{
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+        << "parser_depth="
+        << parser_depth;
+
+    memset(m_unscreened, 0, sizeof(m_unscreened));
+}
+
+ParserScreenedJson::~ParserScreenedJson()
+{}
+
+size_t
+ParserScreenedJson::push(const char *buf, size_t len)
+{
+    size_t i = 0;
+    char c;
+
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON) << "ParserScreenedJson::push(): starting (len=" << len << ")";
+
+    if (len == 0) {
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON) << "ParserScreenedJson::push(): end of data signal! m_state=" << m_state;
+        // flush unescaped data collected (if any)
+        if (m_leftoverLen > 0) {
+            // No need any processing for leftover data - last char must be doublequote, else - error
+            m_state = s_error;
+            dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                << "ParserScreenedJson::push(): end of data and leftover detected m_state="
+                << m_state;
+            return i;
+        }
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+            << "ParserScreenedJson::push(): s_value, pushing m_unscreened = "
+            << m_unscreened
+            << ", m_leftoverLen = "
+            << m_leftoverLen
+            << ", m_unscreenedLen = "
+            << m_unscreenedLen;
+
+        if (m_receiver.onKey("json_unscreened", 15) != 0) {
+            m_state = s_error;
+            return i;
+        }
+
+        if (m_receiver.onValue(m_unscreened, m_unscreenedLen) != 0) {
+            m_state = s_error;
+            return i;
+        }
+
+        if (m_receiver.onKvDone() != 0)
+        {
+            m_state = s_error;
+            return i;
+        }
+        return 0;
+    }
+
+    while (i < len)
+    {
+        c = buf[i];
+
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+            << "ParserScreenedJson::push(): state="
+            << m_state
+            << "; c='"
+            << c
+            << "'"
+            << "; i="
+            << i
+            << ", m_leftoverLen = "
+            << m_leftoverLen
+            << ", m_unscreenedLen = "
+            << m_unscreenedLen
+            << ", m_unscreened = "
+            << m_unscreened;
+
+        switch (m_state)
+        {
+            case s_start:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_start";
+                m_state = s_value;
+
+                // fallthrough  not required, removing 1st doublequote, it denoted by regex //
+                //CP_FALL_THROUGH;
+                break;
+            }
+            case s_value:
+            {
+                if (c == '\\') {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\\';
+                        m_leftoverLen = 0;
+                        m_unscreenedLen++;
+                    } else {
+                        m_leftoverLen++;
+                    }
+                } else  if (c =='\"') {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\"';
+                        m_unscreenedLen++;
+                        if (m_leftoverLen > 0) {
+                            m_leftoverLen = 0;
+                        }
+                    }
+                } else {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\\';
+                        m_unscreenedLen++;
+                        m_leftoverLen = 0;
+                    }
+                    m_unscreened[m_unscreenedLen] = c;
+                    m_unscreenedLen++;
+                }
+                if (m_unscreenedLen >= MAX_UNSCREENED_JSON_SIZE) {
+                    if (m_receiver.onKey("json_unscreened", 15) != 0) {
+                        m_state = s_error;
+                        return i;
+                    }
+                    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_value, pushing m_unscreened = "
+                    << m_unscreened
+                    << ", m_leftoverLen = "
+                    << m_leftoverLen
+                    << ", m_unscreenedLen = "
+                    << m_unscreenedLen;
+                    if (m_receiver.onValue(m_unscreened, m_unscreenedLen) != 0) {
+                        m_state = s_error;
+                        return i;
+                    }
+                    m_unscreenedLen = 0;
+                }
+            break;
+            }
+            case s_error:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_error";
+                return 0;
+            }
+            default:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): JSON parser unrecoverable error";
+                m_state = s_error;
+                return 0;
+            }
+        }
+        ++i;
+    }
+
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+        << "ParserScreenedJson::push(): finished: len="
+        << len;
+    return len;
+}
+
+void
+ParserScreenedJson::finish()
+{
+    push(NULL, 0);
+}
+
+const std::string &
+ParserScreenedJson::name() const
+{
+    return m_parserName;
+}
+
+bool
+ParserScreenedJson::error() const
+{
+    return m_state == s_error;
+}

components/security_apps/waap/waap_clib/ParserScreenedJson.h

@@ -0,0 +1,52 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_SCREENED_JSON_H_
+#define __PARSER_SCREENED_JSON_H_
+
+#include "ParserBase.h"
+#include <string.h>
+
+#define MAX_UNSCREENED_JSON_SIZE 4095
+
+class ParserScreenedJson : public ParserBase {
+public:
+    ParserScreenedJson(IParserStreamReceiver &receiver, size_t parser_depth);
+    virtual ~ParserScreenedJson();
+    size_t push(const char *data, size_t data_len);
+    void finish();
+    virtual const std::string &name() const;
+    bool error() const;
+    // LCOV_EXCL_START Reason: The function not in use, compliance with the interface
+    virtual size_t depth() { return 1; }
+    // LCOV_EXCL_STOP
+
+private:
+    enum state
+    {
+        s_start,
+        s_value,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    enum state m_state;
+    size_t m_unscreenedLen;
+    char m_unscreened[MAX_UNSCREENED_JSON_SIZE];
+    size_t m_leftoverLen;
+    static const std::string m_parserName;
+    size_t m_parser_depth;
+};
+
+#endif
+

components/security_apps/waap/waap_clib/RegexComparator.cc

@@ -0,0 +1,36 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "RegexComparator.h"
+#include <sstream>
+
+namespace Waap {
+namespace Util {
+
+std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet) {
+    std::stringstream ss;
+    ss << "[";
+    bool first = true;
+    for (const auto &regexPtr : regexSet) {
+        if (!first) ss << ", ";
+        if (regexPtr) {
+            first = false;
+            ss << regexPtr->str();
+        }
+    }
+    ss << "]";
+    return ss.str();
+}
+
+}
+}

components/security_apps/waap/waap_clib/RegexComparator.h

@@ -0,0 +1,32 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <set>
+#include <boost/regex.hpp>
+
+namespace Waap {
+namespace Util {
+
+// Custom comparator for std::shared_ptr<boost::regex>
+struct RegexComparator {
+    bool operator()(const std::shared_ptr<boost::regex>& lhs, const std::shared_ptr<boost::regex>& rhs) const {
+        // Compare the actual regex patterns by string representation
+        return lhs->str() < rhs->str();
+    }
+};
+
+std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet);
+
+}
+}

components/security_apps/waap/waap_clib/ScanResult.cc

@@ -23,9 +23,11 @@ unescaped_line(),
 param_name(),
 location(),
 score(0.0f),
+other_model_score(0.0f),
 scoreNoFilter(0.0f),
 scoreArray(),
 keywordCombinations(),
+keywordsAfterFilter(),
 attack_types(),
 m_isAttackInParam(false)
 {
@@ -41,9 +43,11 @@ void Waf2ScanResult::clear()
     param_name.clear();
     location.clear();
     score = 0;
+    other_model_score = 0;
     scoreNoFilter = 0;
     scoreArray.clear();
     keywordCombinations.clear();
+    keywordsAfterFilter.clear();
     attack_types.clear();
 }
 

components/security_apps/waap/waap_clib/ScanResult.h

@@ -29,9 +29,12 @@ struct Waf2ScanResult {
     std::string param_name;
     std::string location;
     double score;
+    double other_model_score;
     double scoreNoFilter;
     std::vector<double> scoreArray;
+    std::vector<double> coefArray;
     std::vector<std::string> keywordCombinations;
+    std::vector<std::string> keywordsAfterFilter;
     std::set<std::string> attack_types;
     bool m_isAttackInParam;
     void clear(); // clear Waf2ScanResult

components/security_apps/waap/waap_clib/ScoreBuilder.cc

@@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
             m_keywordsDataMap[it->first] = it->second;
         }
     }
+
+    m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept;
+    m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef;
+    m_stats.isLinModel = baseScores.m_stats.isLinModel;
 }
 
 
@@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
     m_pWaapAssetState(pWaapAssetState)
 {
     restore();
-
     // merge
     mergeScores(baseScores);
 }
@@ -352,16 +355,42 @@ void ScoreBuilder::snap()
         const std::string &poolName = pool.first;
         const KeywordsScorePool& keywordScorePool = pool.second;
         m_snapshotKwScoreMap[poolName];
+        m_snapshotKwCoefMap[poolName];
+        if (keywordScorePool.m_keywordsDataMap.empty()) {
+            m_snapshotStatsMap[poolName] = KeywordsStats();
+        } else {
+            m_snapshotStatsMap[poolName] = keywordScorePool.m_stats;
+        }
 
         for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
         {
             const std::string &kwName = kwData.first;
-            double kwScore = kwData.second.score;
-            m_snapshotKwScoreMap[poolName][kwName] = kwScore;
+            if (keywordScorePool.m_stats.isLinModel) {
+                m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score;
+            } else {
+                m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score;
+            }
+            m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef;
         }
     }
 }
 
+KeywordsStats
+ScoreBuilder::getSnapshotStats(const std::string &poolName) const
+{
+    std::map<std::string, KeywordsStats>::const_iterator poolIt = m_snapshotStatsMap.find(poolName);
+    if (poolIt == m_snapshotStatsMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool";
+        poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE);
+        if (poolIt == m_snapshotStatsMap.end()) {
+            dbgWarning(D_WAAP_SCORE_BUILDER) <<
+                "base pool does not exist! This is probably a bug. Returning empty stats";
+            return KeywordsStats();
+        }
+    }
+    return poolIt->second;
+}
+
 double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore,
     const std::string &poolName) const
 {
@@ -369,12 +398,11 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
     if (poolIt == m_snapshotKwScoreMap.end()) {
         dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool";
         poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE);
-    }
-
-    if (poolIt == m_snapshotKwScoreMap.end()) {
-        dbgDebug(D_WAAP_SCORE_BUILDER) <<
-            "base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
-        return defaultScore;
+        if (poolIt == m_snapshotKwScoreMap.end()) {
+            dbgDebug(D_WAAP_SCORE_BUILDER) <<
+                "base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
+            return defaultScore;
+        }
     }
 
     const KeywordScoreMap &kwScoreMap = poolIt->second;
@@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
     return kwScoreFound->second;
 }
 
+double
+ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const
+{
+    std::map<std::string, KeywordScoreMap>::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName);
+    if (poolIt == m_snapshotKwCoefMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool";
+        poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE);
+    }
+
+    if (poolIt == m_snapshotKwCoefMap.end()) {
+        dbgWarning(D_WAAP_SCORE_BUILDER) <<
+            "base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef;
+        return defaultCoef;
+    }
+
+    const KeywordScoreMap &kwCoefMap = poolIt->second;
+
+    KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword);
+    if (kwCoefFound == kwCoefMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef <<
+            " (default, keyword not found in pool '" << poolName << "')";
+        return defaultCoef;
+    }
+
+    dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" <<
+        poolName << "')";
+    return kwCoefFound->second;
+}
+
 keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip)
 {
     return m_fpStore.ipItems[ip];

components/security_apps/waap/waap_clib/ScoreBuilder.h

@@ -26,6 +26,8 @@
 #include <cereal/types/string.hpp>
 #include "WaapDefines.h"
 
+USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER);
+
 struct ScoreBuilderData {
     std::string m_sourceIdentifier;
     std::string m_userAgent;
@@ -52,11 +54,15 @@ enum KeywordType {
 };
 
 struct KeywordData {
-    KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {}
+    KeywordData() :
+        truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN)
+    {}
 
     unsigned int truePositiveCtr;
     unsigned int falsePositiveCtr;
     double score;
+    double coef;
+    double new_score;
     KeywordType type;
 
     template <class Archive>
@@ -65,20 +71,42 @@ struct KeywordData {
             cereal::make_nvp("true_positives", truePositiveCtr),
             cereal::make_nvp("score", score),
             cereal::make_nvp("type", type));
+        try {
+            ar(cereal::make_nvp("coef", coef),
+                cereal::make_nvp("new_score", new_score));
+        } catch (const cereal::Exception &e) {
+            ar.setNextName(nullptr);
+            coef = 0;
+            new_score = 0;
+        }
     }
 };
 
 struct KeywordsStats {
-    KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {}
+    KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0)
+    {}
 
     template <class Archive>
     void serialize(Archive& ar) {
         ar(cereal::make_nvp("false_positives", falsePositiveCtr),
-            cereal::make_nvp("true_positives", truePositiveCtr));
+                cereal::make_nvp("true_positives", truePositiveCtr));
+        try {
+            ar(cereal::make_nvp("intercept", linModelIntercept),
+                cereal::make_nvp("log_nnz_coef", linModelNNZCoef));
+            isLinModel = true;
+        } catch (const cereal::Exception &e) {
+            ar.setNextName(nullptr);
+            linModelIntercept = 0;
+            linModelNNZCoef = 0;
+            isLinModel = false;
+        }
     }
 
     unsigned int truePositiveCtr;
     unsigned int falsePositiveCtr;
+    double linModelIntercept;
+    double linModelNNZCoef;
+    bool isLinModel;
 };
 
 typedef std::unordered_set<std::string> keywords_set;
@@ -106,6 +134,7 @@ struct KeywordsScorePool {
 
     KeywordsScorePool();
 
+    // LCOV_EXCL_START Reason: no test exist
     template <typename _A>
     KeywordsScorePool(_A &iarchive)
     {
@@ -120,6 +149,7 @@ struct KeywordsScorePool {
             m_keywordsDataMap[key] = item.second;
         }
     }
+    // LCOV_EXCL_STOP
 
     template <class Archive>
     void serialize(Archive& ar) {
@@ -148,6 +178,8 @@ public:
 
     void snap();
     double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const;
+    double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const;
+    KeywordsStats getSnapshotStats(const std::string &poolName) const;
 
     keywords_set getIpItemKeywordsSet(std::string ip);
     keywords_set getUaItemKeywordsSet(std::string userAgent);
@@ -200,6 +232,8 @@ protected:
     SerializedData m_serializedData;
     std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
     std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
+    std::map<std::string, KeywordScoreMap> m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap()
+    std::map<std::string, KeywordsStats> m_snapshotStatsMap; // the snapshot is updated only by a call to snap()
     std::list<std::string> m_falsePositivesSetsIntersection;
     I_WaapAssetState* m_pWaapAssetState;
 };

components/security_apps/waap/waap_clib/Serializator.cc

@@ -195,6 +195,9 @@ void SerializeToFileBase::saveData()
     } else {
         ss.str(string((const char *)res.output, res.num_output_bytes));
     }
+    if (res.output) free(res.output);
+    res.output = nullptr;
+    res.num_output_bytes = 0;
 
 
     filestream << ss.str();

components/security_apps/waap/waap_clib/WaapAssetState.cc

@@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
 #endif
 }
 
+// std::string nicePrint() - is a function used to create std::string that will represent all data that is
+// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
+// into the object easier.
+
+std::string
+WaapAssetState::nicePrint(Waf2ScanResult &res) const
+{
+    std::string result = "Waf2ScanResult:\n";
+    result += "keyword_matches:\n";
+    for (const auto &keyword : res.keyword_matches) {
+        result += keyword + "\n";
+    }
+    result += "regex_matches:\n";
+    for (const auto &regex : res.regex_matches) {
+        result += regex + "\n";
+    }
+    result += "filtered_keywords:\n";
+    for (const auto &filtered : res.filtered_keywords) {
+        result += filtered + "\n";
+    }
+    result += "found_patterns:\n";
+    for (const auto &pattern : res.found_patterns) {
+        result += pattern.first + ":\n";
+        for (const auto &value : pattern.second) {
+            result += value + "\n";
+        }
+    }
+    result += "unescaped_line: " + res.unescaped_line + "\n";
+    result += "param_name: " + res.param_name + "\n";
+    result += "location: " + res.location + "\n";
+    result += "score: " + std::to_string(res.score) + "\n";
+    result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n";
+    result += "scoreArray:\n";
+    for (const auto &score : res.scoreArray) {
+        result += std::to_string(score) + "\n";
+    }
+    result += "keywordCombinations:\n";
+    for (const auto &combination : res.keywordCombinations) {
+        result += combination + "\n";
+    }
+    result += "attack_types:\n";
+    for (const auto &attack : res.attack_types) {
+        result += attack + "\n";
+    }
+    result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n";
+    return result;
+}
+
+
 bool
 checkBinaryData(const std::string &line, bool binaryDataFound)
 {
@@ -1033,7 +1082,7 @@ WaapAssetState::apply(
     // Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
     // This is done to improve performance of regex matching.
     SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
     checkRegex(
         unescapedLineSample,
         m_Signatures->specific_acuracy_keywords_regex,
@@ -1111,7 +1160,7 @@ WaapAssetState::apply(
     }
 
     bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns);
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res);
     if (os_cmd_ev) {
         dbgTrace(D_WAAP_EVASIONS) << "os command evasion found";
 
@@ -1295,6 +1344,47 @@ WaapAssetState::apply(
         }
     }
 
+    bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns);
+    dbgTrace(D_WAAP_EVASIONS)
+        << "path_traversal_ev = " << path_traversal_ev
+        << " sample = " << res.unescaped_line
+        << " res.unescaped_line.find(2f) =  " << res.unescaped_line.find("2f");
+    if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) {
+        // Possible path traversal evasion .2f. detected: - clean up and scan with regexes again.
+        dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line
+            << "Status beroe evasion checking " << nicePrint(res);
+
+        std::string unescaped = line;
+        replaceAll(unescaped, "2f", "/");
+        size_t kwCount = res.keyword_matches.size();
+
+        if (res.unescaped_line != unescaped) {
+            SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
+            checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
+                res.found_patterns, longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+        }
+
+        if (kwCount == res.keyword_matches.size()) {
+            // Remove the evasion keyword if no real evasion found
+            keywordsToRemove.push_back("path_traversal");
+            path_traversal_ev = false;
+        }
+        else if (!binaryDataFound) {
+            // Recalculate repetition and/or probing indicators
+            unsigned int newWordsCount = 0;
+            calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
+                newWordsCount);
+            // Take minimal words count because empirically it means evasion was probably succesfully decoded
+            wordsCount = std::min(wordsCount, newWordsCount);
+        }
+        dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
+    }
+
+
     bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
         "quotes_space_ev_fast_reg",
         res.found_patterns
@@ -1726,7 +1816,7 @@ WaapAssetState::apply(
             wordsCount = std::min(wordsCount, newWordsCount);
         }
     }
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
     // Remove evasion keywords that should not be reported because there's no real evasion found
     if (!keywordsToRemove.empty()) {
         dbgTrace(D_WAAP_SAMPLE_SCAN)

components/security_apps/waap/waap_clib/WaapAssetState.h

@@ -49,6 +49,7 @@ private: //ugly but needed for build
         Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
 
     void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
+    std::string nicePrint(Waf2ScanResult &res) const;
 
 public:
     // Load and compile signatures from file

components/security_apps/waap/waap_clib/WaapOverride.cc

@@ -12,7 +12,6 @@
 // limitations under the License.
 
 #include "WaapOverride.h"
-#include "Waf2Util.h"
 
 USE_DEBUG_FLAG(D_WAAP);
 
@@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const
             (m_operand1 == other.m_operand1) &&
             (m_operand2 == other.m_operand2) &&
             (m_tag == other.m_tag) &&
-            Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) &&
-            m_cidr == other.m_cidr &&
+            (m_valuesRegex == other.m_valuesRegex) &&
+            m_ip_addr_values == other.m_ip_addr_values &&
             m_isCidr == other.m_isCidr;
 }
 

components/security_apps/waap/waap_clib/WaapOverride.h

@@ -21,6 +21,7 @@
 #include <memory>
 #include "debug.h"
 #include "CidrMatch.h"
+#include "RegexComparator.h"
 
 USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
 
@@ -47,28 +48,58 @@ public:
             m_tag = to_lower_copy(m_tag);
 
             if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
-                m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
-                m_tag != "responsebody" &&  m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") {
+                m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" &&
+                m_tag != "paramlocation" && m_tag != "responsebody" &&  m_tag != "headername" &&
+                m_tag != "headervalue" && m_tag != "method") {
                 m_isValid = false;
                 dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
             }
-            // The name "value" here is misleading. The real meaning is "regex pattern string"
-            ar(cereal::make_nvp("value", m_value));
+
+            try {
+                ar(cereal::make_nvp("values", m_values));
+                dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead.";
+            } catch (const cereal::Exception &e) {
+                // The name "value" here is misleading. The real meaning is "regex pattern string"
+                ar(cereal::make_nvp("value", m_value));
+                m_values.insert(m_value);
+            }
 
             if (m_tag == "sourceip" || m_tag == "sourceidentifier") {
-                m_isCidr = Waap::Util::isCIDR(m_value, m_cidr);
+                m_isCidr = true;
+                m_ip_addr_values.resize(m_values.size());
+
+                int val_idx = 0;
+                for (const auto &cur_val : m_values) {
+                    if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) {
+                        dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val;
+                        m_isValid = false;
+                        break;
+                    }
+                    val_idx++;
+                }
+                sortAndMergeCIDRs();
+                dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values);
             }
             m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody");
 
             if (!m_isCidr) {
-                // regex build may throw boost::regex_error
-                m_valueRegex = nullptr;
-                try {
-                    m_valueRegex = std::make_shared<boost::regex>(m_value);
-                }
-                catch (const boost::regex_error &err) {
-                    dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" <<
-                        m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'";
+                for (const auto &cur_val : m_values) {
+                    try {
+                        m_valuesRegex.emplace(std::make_shared<boost::regex>(cur_val));
+                    }
+                    catch (const boost::regex_error &err) {
+                        dbgDebug(D_WAAP_OVERRIDE)
+                            << "Waap::Override::Match(): Failed to compile regex pattern '"
+                            << cur_val
+                            << "' on position "
+                            << err.position()
+                            <<  ". Reason: '"
+                            << err.what()
+                            << "'";
+                        m_isValid = false;
+                        m_valuesRegex.clear();
+                        break;
+                    }
                 }
             }
         }
@@ -95,13 +126,21 @@ public:
     template<typename TestFunctor>
     bool match(TestFunctor testFunctor) const {
         if (m_op == "basic" && m_isCidr) {
-            bool result = testFunctor(m_tag, m_cidr);
-            dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result;
+            bool result = testFunctor(m_tag, m_ip_addr_values);
+            dbgTrace(D_WAAP_OVERRIDE)
+                << "Override matching CIDR list: "
+                << cidrsToString(m_ip_addr_values)
+                << " result: "
+                << result;
             return result;
         }
-        else if (m_op == "basic" && m_valueRegex) {
-            bool result = testFunctor(m_tag, *m_valueRegex);
-            dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result;
+        else if (m_op == "basic" && !m_valuesRegex.empty()) {
+            bool result = testFunctor(m_tag, m_valuesRegex);
+            dbgTrace(D_WAAP_OVERRIDE)
+                << "Override matching regex list: "
+                << regexSetToString(m_valuesRegex)
+                << " result: "
+                << result;
             return result;
         }
         if (m_op == "and") {
@@ -125,24 +164,39 @@ public:
         return false;
     }
 
-    bool isOverrideResponse() const {
-        return m_isOverrideResponse;
-    }
+    bool isOverrideResponse() const { return m_isOverrideResponse; }
 
-    bool isValidMatch() const{
-        return m_isValid;
-    }
+    bool isValidMatch() const { return m_isValid; }
 
 private:
+    void sortAndMergeCIDRs() {
+        if (m_ip_addr_values.empty()) return;
+        std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end());
+
+        size_t mergedIndex = 0;
+        for (size_t i = 1; i < m_ip_addr_values.size(); ++i) {
+            Waap::Util::CIDRData &current = m_ip_addr_values[mergedIndex];
+            Waap::Util::CIDRData &next = m_ip_addr_values[i];
+
+            if (!doesFirstCidrContainSecond(current, next)) {
+                ++mergedIndex;
+                if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next;
+            }
+        }
+
+        m_ip_addr_values.resize(mergedIndex + 1);
+    }
+
     std::string m_op;
     std::shared_ptr<Match> m_operand1;
     std::shared_ptr<Match> m_operand2;
     std::string m_tag;
     std::string m_value;
-    std::shared_ptr<boost::regex> m_valueRegex;
-    Waap::Util::CIDRData m_cidr;
-    bool        m_isCidr;
-    bool        m_isOverrideResponse;
+    std::set<std::string> m_values;
+    std::vector<Waap::Util::CIDRData> m_ip_addr_values;
+    std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> m_valuesRegex;
+    bool m_isCidr;
+    bool m_isOverrideResponse;
     bool m_isValid;
 };
 

components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc

@@ -17,100 +17,150 @@
 #include "WaapOverrideFunctor.h"
 #include "Waf2Engine.h"
 #include "CidrMatch.h"
+#include "RegexComparator.h"
 #include "agent_core_utilities.h"
 #include "debug.h"
 
 USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
 
+#define REGX_MATCH(FIELD_FETCH) \
+    NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx)
+#define W2T_REGX_MATCH(FIELD_GETTER) \
+    REGX_MATCH(waf2Transaction.FIELD_GETTER())
+
 WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction)
 {
 }
 
-bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) {
+bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector<Waap::Util::CIDRData> &values) {
+    std::string sourceIp;
     if (tag == "sourceip") {
-        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
-            value.cidrString;
-        std::string sourceIp = waf2Transaction.getRemoteAddr();
-        // match sourceIp against the cidr
-        return Waap::Util::cidrMatch(sourceIp, value);
+        dbgDebug(D_WAAP_OVERRIDE)
+            << "Remote IP Address : "
+            << waf2Transaction.getRemoteAddr();
+
+        sourceIp = waf2Transaction.getRemoteAddr();
+    } else if (tag == "sourceidentifier") {
+        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr();
+        sourceIp = waf2Transaction.getSourceIdentifier();
+    } else {
+        dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag;
+        return false;
     }
-    else if (tag == "sourceidentifier") {
-        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
-            value.cidrString;
-        std::string sourceIp = waf2Transaction.getSourceIdentifier();
-        // match source against the cidr
-        return Waap::Util::cidrMatch(sourceIp, value);
+
+    Waap::Util::CIDRData source_cidr;
+    if (!Waap::Util::isCIDR(sourceIp, source_cidr)) {
+        dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp;
+        return false;
+    }
+
+
+    int left = 0;
+    int right = values.size() - 1;
+
+    while (left <= right) {
+        int mid = left + (right - left) / 2;
+        if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true;
+
+        if (values[mid] < source_cidr) {
+            left = mid + 1;
+        } else {
+            right = mid - 1;
+        }
     }
 
     return false;
 }
 
-bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
+bool WaapOverrideFunctor::operator()(
+    const std::string &tag,
+    const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes)
 {
     boost::cmatch what;
     std::string tagLower = tag;
     std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
+
     try {
         if (tagLower == "method") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getMethod)) return true;
+            }
+            return false;
         }
         else if (tagLower == "url") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getUriStr)) return true;
+            }
+            return false;
         }
         else if (tagLower == "hostname") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getHost)) return true;
+            }
+            return false;
         }
         else if (tagLower == "sourceidentifier") {
-            return NGEN::Regex::regexMatch(
-                __FILE__,
-                __LINE__,
-                waf2Transaction.getSourceIdentifier().c_str(),
-                what,
-                rx
-            );
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getSourceIdentifier)) return true;
+            }
+            return false;
         }
-        else if (tagLower == "keyword") {
-            for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
-                    return true;
+        else if (tagLower == "keyword" || tagLower == "indicator") {
+            for (const auto &rx : rxes) {
+                for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
+                    if (REGX_MATCH(keywordStr)) {
+                        return true;
+                    }
                 }
             }
             return false;
         }
         else if (tagLower == "paramname") {
-            for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
-                    return true;
+            for (const auto &rx : rxes) {
+                for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
+                    if (REGX_MATCH(keywordInfo.getName())) {
+                        return true;
+                    }
                 }
-            }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) {
-                return true;
-            }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) {
-                return true;
+                if (W2T_REGX_MATCH(getParamKey)) return true;
+                if (W2T_REGX_MATCH(getParam)) return true;
             }
             return false;
         }
         else if (tagLower == "paramvalue") {
-            for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
-                    return true;
+            for (const auto &rx : rxes) {
+                for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
+                    if (REGX_MATCH(keywordInfo.getValue())) {
+                        return true;
+                    }
                 }
+                if (W2T_REGX_MATCH(getSample)) return true;
             }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) {
-                return true;
-            }
+
             return false;
         }
         else if (tagLower == "paramlocation") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getLocation)) return true;
+            }
+            return false;
         }
         else if (tagLower == "responsebody") {
             waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
             if (!waf2Transaction.getResponseBody().empty()) {
-                boost::smatch matcher;
-                return NGEN::Regex::regexSearch(__FILE__, __LINE__,
-                        waf2Transaction.getResponseBody().c_str(), matcher, rx);
+                for (const auto &rx : rxes) {
+                    boost::smatch matcher;
+                    if (NGEN::Regex::regexSearch(
+                        __FILE__,
+                        __LINE__,
+                        waf2Transaction.getResponseBody().c_str(),
+                        matcher,
+                        *rx
+                    )) {
+                        return true;
+                    }
+                }
+                return false;
             } else {
                 return false;
             }
@@ -119,11 +169,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
                 dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
                 return false;
             }
-            for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
-                std::string value = hdr_pair.first;
-                std::transform(value.begin(), value.end(), value.begin(), ::tolower);
-                if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
-                    return true;
+            for (const auto &rx : rxes) {
+                for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
+                    std::string value = hdr_pair.first;
+                    std::transform(value.begin(), value.end(), value.begin(), ::tolower);
+                    if (REGX_MATCH(value)) {
+                        return true;
+                    }
                 }
             }
             return false;
@@ -132,11 +184,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
                 dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
                 return false;
             }
-            for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
-                std::string value = hdr_pair.second;
-                std::transform(value.begin(), value.end(), value.begin(), ::tolower);
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
-                    return true;
+            for (const auto &rx : rxes) {
+                for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
+                    std::string value = hdr_pair.second;
+                    std::transform(value.begin(), value.end(), value.begin(), ::tolower);
+                    if (REGX_MATCH(value)) {
+                        return true;
+                    }
                 }
             }
             return false;
@@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
         dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();
         return false;
     }
-
     // Unknown tag: should not occur
     dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
     return false;

components/security_apps/waap/waap_clib/WaapOverrideFunctor.h

@@ -16,6 +16,7 @@
 namespace Waap {
     namespace Util {
         struct CIDRData; // forward decleration
+        struct RegexComparator;
     }
 }
 
@@ -24,12 +25,15 @@ class Waf2Transaction;
 // Functor used to match Override rules against request data
 class WaapOverrideFunctor {
 public:
-    WaapOverrideFunctor(Waf2Transaction& waf2Transaction);
+    WaapOverrideFunctor(Waf2Transaction &waf2Transaction);
 
-    bool operator()(const std::string& tag, const Waap::Util::CIDRData& value);
+    bool operator()(const std::string &tag, const std::vector<Waap::Util::CIDRData> &values);
 
-    bool operator()(const std::string& tag, const boost::regex& rx);
+    bool operator()(
+        const std::string &tag,
+        const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes
+    );
 
 private:
-    Waf2Transaction& waf2Transaction;
+    Waf2Transaction &waf2Transaction;
 };

components/security_apps/waap/waap_clib/WaapScanner.cc

@@ -13,6 +13,7 @@
 
 #include "WaapScanner.h"
 #include "WaapScores.h"
+#include "Waf2Engine.h"
 #include "i_transaction.h"
 #include <string>
 #include "debug.h"
@@ -105,20 +106,54 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
     }
 
     std::sort(newKeywords.begin(), newKeywords.end());
+    res.keywordsAfterFilter.clear();
+    for (auto keyword : newKeywords) {
+        res.keywordsAfterFilter.push_back(keyword);
+    }
+    double res_score = getScoreFromPool(res, newKeywords, poolName);
 
+    std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
+    Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
+
+    if (applyLearning && poolName != other_pool_name &&
+        modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
+        double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
+        dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
+                                << ", vs. pool " << other_pool_name << ": " << other_score
+                                << ", score difference: " << res_score - other_score
+                                << ", sample: " << res.unescaped_line;
+        res.other_model_score = other_score;
+    } else {
+        res.other_model_score = res_score;
+    }
+    return res_score;
+}
+
+double Waap::Scanner::getScoreFromPool(
+    Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
+)
+{
     res.scoreArray.clear();
+    res.coefArray.clear();
     res.keywordCombinations.clear();
+    KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
 
     if (!newKeywords.empty()) {
         // Collect scores of individual keywords
         Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
-            res.scoreArray);
+            res.scoreArray, res.coefArray);
         // Collect keyword combinations and their scores. Append scores to scoresArray,
         // and also populate m_scanResultKeywordCombinations list
         Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
-            res.scoreArray, res.keywordCombinations);
+            res.scoreArray, res.coefArray, res.keywordCombinations);
     }
 
+    if (stats.isLinModel) {
+        return Waap::Scores::calcLogisticRegressionScore(
+            res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef
+        );
+    }
+    // use base_scores calculation
     return Waap::Scores::calcArrayScore(res.scoreArray);
 }
 
@@ -127,7 +162,7 @@ bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res,
 {
     if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
         if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
-                (key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
+            (key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
             dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
             return true;
         }

components/security_apps/waap/waap_clib/WaapScanner.h

@@ -44,6 +44,9 @@ namespace Waap {
         static const std::string xmlEntityAttributeId;
     private:
         double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
+        double getScoreFromPool(
+            Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
+        );
         bool shouldIgnoreOverride(const Waf2ScanResult &res);
         bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);
         

components/security_apps/waap/waap_clib/WaapScores.cc

@@ -14,6 +14,8 @@
 #include "WaapScores.h"
 #include <vector>
 #include <string>
+#include <cmath>
+
 #include "ScoreBuilder.h"
 #include "WaapDefines.h"
 #include "debug.h"
@@ -24,11 +26,44 @@ namespace Waap {
 namespace Scores {
 
 std::string getScorePoolNameByLocation(const std::string &location) {
-    std::string poolName = KEYWORDS_SCORE_POOL_BASE;
-    if (location == "header") {
-        poolName = KEYWORDS_SCORE_POOL_HEADERS;
+    auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.scorePoolName");
+    std::string res = KEYWORDS_SCORE_POOL_BASE;
+    if (maybePoolName.ok()) {
+        res = maybePoolName.unpack();
     }
-    return poolName;
+    else if (location == "header") {
+        res = KEYWORDS_SCORE_POOL_HEADERS;
+    }
+    return res;
+}
+
+std::string getOtherScorePoolName()
+{
+    auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.otherScorePoolName");
+    if (maybePoolName.ok()) {
+        return maybePoolName.unpack();
+    }
+    return KEYWORDS_SCORE_POOL_BASE;
+}
+
+ModelLoggingSettings getModelLoggingSettings()
+{
+    ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF,
+                                    .logToS3 = false,
+                                    .logToStream = true};
+    auto maybeLogS3 = getProfileAgentSetting<bool>("agent.waap.modelLogToS3");
+    if (maybeLogS3.ok()) {
+        settings.logToS3 = maybeLogS3.unpack();
+    }
+    auto maybeLogKusto = getProfileAgentSetting<bool>("agent.waap.modelLogToStream");
+    if (maybeLogKusto.ok()) {
+        settings.logToStream = maybeLogKusto.unpack();
+    }
+    auto maybeLogLevel = getProfileAgentSetting<uint>("agent.waap.modelLogLevel");
+    if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) {
+        settings.logLevel = static_cast<ModelLogLevel>(maybeLogLevel.unpack());
+    }
+    return settings;
 }
 
 void
@@ -37,9 +72,16 @@ addKeywordScore(
     const std::string &poolName,
     std::string keyword,
     double defaultScore,
-    std::vector<double>& scoresArray)
+    double defaultCoef,
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray)
 {
-    scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName));
+    double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName);
+    double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName);
+    dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef
+                                    << " keyword: '" << keyword << "' pool: " << poolName;
+    scoresArray.push_back(score);
+    coefArray.push_back(coef);
 }
 
 // Calculate score of individual keywords
@@ -48,13 +90,14 @@ calcIndividualKeywords(
     const ScoreBuilder& scoreBuilder,
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
-    std::vector<double>& scoresArray)
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray)
 {
     std::vector<std::string> keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!)
     std::sort(keywords.begin(), keywords.end());
 
     for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) {
-        addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray);
+        addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray);
     }
 }
 
@@ -65,10 +108,12 @@ calcCombinations(
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
     std::vector<double>& scoresArray,
+    std::vector<double>& coefArray,
     std::vector<std::string>& keyword_combinations)
 {
     keyword_combinations.clear();
     static const double max_combi_score = 1.0f;
+    double default_coef = 0.8f;
 
     for (size_t i = 0; i < keyword_matches.size(); ++i) {
         std::vector<std::string> combinations;
@@ -93,7 +138,7 @@ calcCombinations(
             }
             // set default combination score to be the sum of its keywords, bounded by 1
             default_score = std::min(default_score, max_combi_score);
-            addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
+            addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray);
             keyword_combinations.push_back(combination);
         }
     }
@@ -105,7 +150,6 @@ calcArrayScore(std::vector<double>& scoreArray)
     // Calculate cumulative score from array of individual scores
     double score = 1.0f;
     for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) {
-        dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore;
         double left = 10.0f - score;
         double divisor = (*pScore / 3.0f + 10.0f);  // note: divisor can't be empty because
                                                     // *pScore is always positive and there's a +10 offset
@@ -115,5 +159,20 @@ calcArrayScore(std::vector<double>& scoreArray)
     return score;
 }
 
+double
+calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef)
+{
+    // Sparse logistic regression model, with boolean feature values
+    // Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features
+    // An additional feature was added for the log of the number of non-zero features, as a regularization term
+    double log_odds = intercept + nnzCoef * log(static_cast<double>(coefArray.size()) + 1);
+    for (double &pCoef : coefArray) {
+        log_odds += pCoef;
+    }
+    // Apply the expit function to the log-odds to obtain the probability,
+    // and multiply by 10 to obtain a 'score' in the range [0, 10]
+    return 1.0f / (1.0f + exp(-log_odds)) * 10.0f;
+}
+
 }
 }

components/security_apps/waap/waap_clib/WaapScores.h

@@ -20,7 +20,21 @@
 namespace Waap {
 namespace Scores {
 
+enum class ModelLogLevel {
+    OFF = 0,
+    DIFF = 1,
+    ALL = 2
+};
+
+struct ModelLoggingSettings {
+    ModelLogLevel logLevel;
+    bool logToS3;
+    bool logToStream;
+};
+
 std::string getScorePoolNameByLocation(const std::string &location);
+std::string getOtherScorePoolName();
+ModelLoggingSettings getModelLoggingSettings();
 
 void
 addKeywordScore(
@@ -28,7 +42,9 @@ addKeywordScore(
     const std::string &poolName,
     std::string keyword,
     double defaultScore,
-    std::vector<double>& scoresArray);
+    double defaultCoef,
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray);
 
 // Calculate score of individual keywords
 void
@@ -36,7 +52,8 @@ calcIndividualKeywords(
     const ScoreBuilder& scoreBuilder,
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
-    std::vector<double>& scoresArray);
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray);
 
 // Calculate keyword combinations and their scores
 void
@@ -45,9 +62,11 @@ calcCombinations(
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
     std::vector<double>& scoresArray,
+    std::vector<double>& coefArray,
     std::vector<std::string>& keyword_combinations);
 
 double calcArrayScore(std::vector<double>& scoreArray);
+double calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef=0.0);
 
 }
 }

components/security_apps/waap/waap_clib/WaapTrigger.h

@@ -125,6 +125,7 @@ struct Log {
 struct Trigger {
     template <typename _A>
     void serialize(_A &ar) {
+        ar(cereal::make_nvp("id", triggerId));
         ar(cereal::make_nvp("$triggerType", triggerType));
         triggerType = to_lower_copy(triggerType);
 
@@ -137,6 +138,7 @@ struct Trigger {
     Trigger();
     bool operator==(const Trigger &other) const;
 
+    std::string triggerId;
     std::string triggerType;
     std::shared_ptr<Log> log;
 };

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
         std::string tag = scanStage + "_param";
         m_deepParser.m_key.push(tag.data(), tag.size());
         size_t buff_len = uriEnd - p;
-        dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
-        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
+
+        bool should_decode = checkUrlEncoded(p, buff_len);
+        if (should_decode) {
+            std::string url_as_string(p, buff_len);
+            should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true);
+       }
+        dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
+
+        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
         up.push(p, buff_len);
         up.finish();
         m_deepParser.m_key.pop(tag.c_str());
@@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) {
     m_ignoreScore = ignoreScore;
 }
 
+bool Waf2Transaction::get_ignoreScore() const
+{
+    auto maybe_override_ignore_score = getProfileAgentSetting<std::string>("agent.waap.alwaysReportScore");
+    if (maybe_override_ignore_score.ok()) {
+        return maybe_override_ignore_score.unpack() == "true";
+    }
+    return m_ignoreScore;
+}
+
 void
 Waf2Transaction::decide(
     bool& bForceBlock,
@@ -1342,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
         dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
         return false;
     }
-    std::string body(data);
+    std::string body(data, data_len);
     if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
     {
         dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
@@ -1645,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
     waapLog << LogField("sourcePort", m_remote_port);
     waapLog << LogField("httpHostName", m_hostStr);
     waapLog << LogField("httpMethod", m_methodStr);
+    if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
+    if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
+
     const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
         m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
     bool send_extended_log = shouldSendExtendedLog(triggerLog);
@@ -1889,6 +1908,14 @@ Waf2Transaction::sendLog()
         return;
     }
 
+    std::set<std::string> triggers_set;
+    for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
+        triggers_set.insert(trigger.triggerId);
+        dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
+    }
+    ScopedContext ctx;
+    ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
+
     auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
     switch (decision_type)
     {
@@ -2319,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
         exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
         exceptions_dict["url"].insert(getUriStr());
         exceptions_dict["hostName"].insert(m_hostStr);
+        exceptions_dict["method"].insert(m_methodStr);
 
         for (auto &keyword : res.keyword_matches) {
             exceptions_dict["indicator"].insert(keyword);
@@ -2331,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
         auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
                 getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
         for (const auto &behavior : behaviors) {
+            dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
             if (!res.filtered_keywords.empty() || res.score > 0) {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
+                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
                 std::string overrideId = behavior.getId();
                 if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
                     m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
@@ -2351,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
             }
             if (behavior == action_ignore)
             {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
+                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
                 std::string overrideId = behavior.getId();
                 if (!overrideId.empty()) {
                     m_matchedOverrideIds.insert(overrideId);

components/security_apps/waap/waap_clib/Waf2Engine.h

@@ -20,6 +20,8 @@
 #include "DeepParser.h"
 #include "WaapAssetState.h"
 #include "PatternMatcher.h"
+#include "generic_rulebase/rulebase_config.h"
+#include "generic_rulebase/evaluators/trigger_eval.h"
 #include "Waf2Util.h"
 #include "WaapConfigApplication.h"
 #include "WaapConfigApi.h"
@@ -91,6 +93,7 @@ public:
     const std::vector<std::string> getFilteredKeywords() const;
     const std::map<std::string, std::vector<std::string>> getFilteredVerbose() const;
     virtual const std::vector<std::string> getKeywordsCombinations() const;
+    virtual const std::vector<std::string> getKeywordsAfterFilter() const;
     const std::vector<DeepParser::KeywordInfo>& getKeywordInfo() const;
     const std::vector<std::pair<std::string, std::string> >& getKvPairs() const;
     const std::string getKeywordMatchesStr() const;
@@ -99,6 +102,9 @@ public:
     const std::string getLastScanSample() const;
     virtual const std::string& getLastScanParamName() const;
     double getScore() const;
+    // LCOV_EXCL_START Reason: model testing
+    double getOtherModelScore() const;
+    // LCOV_EXCL_STOP
     const std::vector<double> getScoreArray() const;
     Waap::CSRF::State& getCsrfState();
     const std::set<std::string> getFoundPatterns() const;
@@ -161,7 +167,7 @@ public:
 
     // decision functions
     void set_ignoreScore(bool ignoreScore);
-    bool get_ignoreScore() const { return m_ignoreScore; }
+    bool get_ignoreScore() const;
     void decide(
         bool& bForceBlock,
         bool& bForceException,

components/security_apps/waap/waap_clib/Waf2EngineGetters.cc

@@ -189,6 +189,15 @@ const std::vector<std::string> Waf2Transaction::getKeywordsCombinations() const
     }
     return std::vector<std::string>();
 }
+const std::vector<std::string> Waf2Transaction::getKeywordsAfterFilter() const
+{
+    if (m_scanResult)
+    {
+        return m_scanResult->keywordsAfterFilter;
+    }
+    return std::vector<std::string>();
+}
+
 const std::vector<DeepParser::KeywordInfo>& Waf2Transaction::getKeywordInfo() const
 {
     return m_deepParser.m_keywordInfo;
@@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const
     }
     return 0;
 }
+// LCOV_EXCL_START Reason: model testing
+double Waf2Transaction::getOtherModelScore() const
+{
+    if (m_scanResult) {
+        return m_scanResult->other_model_score;
+    }
+    return 0;
+}
+// LCOV_EXCL_STOP
 const std::vector<double> Waf2Transaction::getScoreArray() const
 {
     if (m_scanResult) {
@@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
     waap_log << LogField("waapUriFalsePositiveScore", (int)(
         autonomousSecurityDecision->getFpMitigationScore() * 100));
     waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100));
+    waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100));
     waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100));
     waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel());
 }

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -35,6 +35,7 @@
 #include "user_identifiers_config.h"
 #include "Waf2Regex.h"
 #include "ParserBinaryFile.h"
+#include "ParserKnownBenignSkipper.h"
 
 using boost::algorithm::to_lower_copy;
 using namespace std;
@@ -508,6 +509,8 @@ const char* g_htmlTags[] = {
 };
 
 static const string b64_prefix("base64,");
+// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded
+static const double base64_entropy_threshold = 4.01;
 
 const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]);
 
@@ -966,7 +969,8 @@ base64_decode_status decodeBase64Chunk(
         string::const_iterator it,
         string::const_iterator end,
         string& decoded,
-        bool clear_on_error)
+        bool clear_on_error,
+        bool called_with_prefix)
 {
     decoded.clear();
     uint32_t acc = 0;
@@ -974,6 +978,7 @@ base64_decode_status decodeBase64Chunk(
     int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them.
     uint32_t nonPrintableCharsCount = 0;
     uint32_t spacer_count = 0;
+    uint32_t length = end - it;
 
     dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
     string::const_iterator begin = it;
@@ -986,6 +991,8 @@ base64_decode_status decodeBase64Chunk(
             return B64_DECODE_INVALID;
         }
 
+        std::unordered_map<char, double> frequency;
+
         while (it != end) {
             unsigned char c = *it;
 
@@ -1008,6 +1015,7 @@ base64_decode_status decodeBase64Chunk(
 
                 // allow for more terminator characters
                 it++;
+                frequency[c]++;
                 continue;
             }
 
@@ -1032,6 +1040,7 @@ base64_decode_status decodeBase64Chunk(
                 // Start tracking terminator characters
                 terminatorCharsSeen++;
                 it++;
+                frequency[c]++;
                 continue;
             }
             else {
@@ -1062,6 +1071,7 @@ base64_decode_status decodeBase64Chunk(
             }
 
             it++;
+            frequency[c]++;
         }
 
         // end of encoded sequence decoded.
@@ -1078,6 +1088,27 @@ base64_decode_status decodeBase64Chunk(
             << "; decoded='"
             << decoded << "'";
 
+        // Check if entropy is correlates with b64 threshold (initially > 4.5)
+        if (!called_with_prefix) {
+            double entropy = 0;
+            double p = 0;
+            for (const auto& pair : frequency) {
+                p = pair.second / length;
+                entropy -= p * std::log2(p);
+            }
+            dbgTrace(D_WAAP_BASE64) << " ===b64Test===:  base entropy = " << entropy << "length = " << length;
+            // Add short payload factor
+            if (length < 16)
+                entropy = entropy * 16 / length;
+            // Enforce tailoring '=' characters
+            entropy+=terminatorCharsSeen;
+
+            dbgTrace(D_WAAP_BASE64) << " ===b64Test===:  corrected entropy = " << entropy << "length = " << length;
+            if (entropy <= base64_entropy_threshold) {
+                return B64_DECODE_INVALID;
+            }
+        }
+
         // Return success only if decoded.size>=5 and there are less than 10% of non-printable
         // characters in output.
         if (decoded.size() >= 5) {
@@ -1188,21 +1219,21 @@ static const SingleRegex csp_report_policy_re(
     "csp_report_policy"
 );
 static const SingleRegex base64_key_value_detector_re(
-        "^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
-        err,
-        "base64_key_value");
+    "^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
+    err,
+    "base64_key_value");
 static const SingleRegex json_key_value_detector_re(
     "\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
-        err,
-        "json_key_value");
+    err,
+    "json_key_value");
 static const SingleRegex base64_key_detector_re(
-        "^[^<>{};,&\\?|=\\s]+={1}",
-        err,
-        "base64_key");
+    "^[^<>{};,&\\?|=\\s]+={1}",
+    err,
+    "base64_key");
 static const SingleRegex base64_prefix_detector_re(
-        "data:\\S*;base64,\\S+|base64,\\S+",
-        err,
-        "base64_prefix");
+    "data:\\S*;base64,\\S+|base64,\\S+",
+    err,
+    "base64_prefix");
 
 // looks for combination <param>={<some text>*:<some text>*}
 //used to allow parsing param=JSON to reduce false positives
@@ -1323,10 +1354,11 @@ processDecodedChunk(
     string::const_iterator start,
     string::const_iterator end,
     string &value,
-    BinaryFileType &binaryFileType
+    BinaryFileType &binaryFileType,
+    bool called_with_prefix = false
 )
 {
-    base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
+    base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix);
     dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
                             << " value.size()=" << value.size();
     if (retVal != B64_DECODE_INVALID && !value.empty()) {
@@ -1349,7 +1381,7 @@ bool isBase64PrefixProcessingOK (
         if (detectBase64Chunk(s, start, end)) {
             dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
             if ((start != s.end()) && (end == s.end())) {
-                retVal = processDecodedChunk(s, start, end, value, binaryFileType);
+                retVal = processDecodedChunk(s, start, end, value, binaryFileType, true);
             }
         } else if (start != s.end()) {
             dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -871,7 +871,8 @@ decodeBase64Chunk(
     std::string::const_iterator it,
     std::string::const_iterator end,
     std::string &decoded,
-    bool clear_on_error = true);
+    bool clear_on_error = true,
+    bool called_with_prefix = false);
 
 bool
 b64DecodeChunk(
@@ -893,6 +894,16 @@ namespace Util {
 
     bool isValidJson(const std::string &input);
 
+    enum KnownSourceType {
+        SOURCE_TYPE_UNKNOWN = 0,
+        SOURCE_TYPE_SENSOR_DATA = 1
+    };
+
+    KnownSourceType detectKnownSource(const std::string &input);
+    bool isScreenedJson(const std::string &input);
+
+    int definePrefixedJson(const std::string &input);
+
     bool detectJSONasParameter(const std::string &s,
             std::string &key,
             std::string &value);

components/security_apps/waap/waap_clib/Waf2Util2.cc

@@ -1,5 +1,7 @@
 #include "Waf2Util.h"
+#include "Waf2Regex.h"
 #include <string>
+#include "debug.h"
 
 namespace Waap {
 namespace Util {
@@ -628,5 +630,52 @@ isValidJson(const std::string &input)
     return false;
 }
 
+KnownSourceType
+detectKnownSource(const std::string &input)
+{
+    static bool err = false;
+    static const SingleRegex known_source_sensor_data_re(
+        "^\\{\\\"sensor_data\\\":\\\"",
+        err,
+        "known_source_sensor_data"
+    );
+    if (known_source_sensor_data_re.hasMatch(input)) {
+        return SOURCE_TYPE_SENSOR_DATA;
+    }
+    return SOURCE_TYPE_UNKNOWN;
+}
+
+int
+definePrefixedJson(const std::string &input)
+{
+    static const size_t MAX_JSON_PREFIX_LEN = 32;
+    static const size_t MIN_PARAMETER_LEN = 4;
+    if (input.size() < MIN_PARAMETER_LEN) {
+        return -1;
+    }
+
+    for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) {
+        if (input[i] == '-' && input[i+1] == '{') return i + 1;
+    }
+
+    return -1;
+}
+
+bool
+isScreenedJson(const std::string &input)
+{
+    static bool err = false;
+    static const SingleRegex screened_json_re(
+        R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])",
+        err,
+        "screened_json"
+    );
+
+    if (screened_json_re.hasMatch(input)) {
+        return true;
+    }
+    return false;
+}
+
 } // namespace Util
 } // namespace Waap

components/security_apps/waap/waap_component_impl.cc

@@ -535,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
         verdict = drop_response.getVerdict();
     }
 
+    bool sould_inject_response = waf2Transaction.shouldInjectResponse();
+    // in Chunked transfer encoding the last chunk is always empty - and we leave it empty
+    bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0;
+    dbgTrace(D_WAAP)
+        << (sould_inject_response ? "should Inject Response" : "should not Inject Response")
+        << (should_stay_empty_chunk ? " empty last chunk will stay empty" : "");
     if (verdict == pending_response.getVerdict() &&
-        waf2Transaction.shouldInjectResponse() &&
-        !event.isLastChunk()
+        sould_inject_response &&
+        !should_stay_empty_chunk
     ) {
         // Inject if needed. Note that this is only reasonable to do if there was no DROP decision above
 

components/utils/generic_rulebase/match_query.cc

@@ -111,7 +111,7 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
                     is_specific_label = false;
                 }
             }
-            is_ignore_keyword = (key == "indicator");
+            is_ignore_keyword = (key == "indicator" || key == "keyword");
 
             if (condition_type != Conditions::Exist) {
                 archive_in(cereal::make_nvp("value", value));
@@ -157,6 +157,9 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
                         dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what();
                     }
                 }
+                if (isKeyTypeIp()) {
+                    sortAndMergeIpRangesValues();
+                }
                 first_value = *(value.begin());
             }
             break;
@@ -241,9 +244,10 @@ MatchQuery::getAllKeys() const
 bool
 MatchQuery::matchAttributes(
         const unordered_map<string, set<string>> &key_value_pairs,
-        set<string> &matched_override_keywords ) const
+        set<string> &matched_override_keywords) const
 {
 
+    dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes";
     if (type == MatchType::Condition) {
         auto key_value_pair = key_value_pairs.find(key);
         if (key_value_pair == key_value_pairs.end()) {
@@ -254,9 +258,11 @@ MatchQuery::matchAttributes(
     } else if (type == MatchType::Operator && operator_type == Operators::And) {
         for (const MatchQuery &inner_match: items) {
             if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator";
                 return false;
             }
         }
+        dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator";
         return true;
     } else if (type == MatchType::Operator && operator_type == Operators::Or) {
         // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
@@ -269,6 +275,7 @@ MatchQuery::matchAttributes(
                 res = true;
             }
         }
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res;
         return res;
     } else {
         dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type";
@@ -282,6 +289,7 @@ MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs)
     MatchQuery::MatchResult matches;
     matches.matched_keywords = make_shared<set<string>>();
     matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
+    dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match;
     return matches;
 }
 
@@ -301,12 +309,15 @@ MatchQuery::matchAttributes(
     bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn;
     bool match = false;
 
-    if (isIP()) {
+    if (isKeyTypeIp()) {
         match = matchAttributesIp(values);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for IP address: " << match;
     } else if (isRegEx()) {
         match = matchAttributesRegEx(values, matched_override_keywords);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for regex: " << match;
     } else {
         match = matchAttributesString(values);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match;
     }
 
     return negate ? !match : match;
@@ -321,6 +332,8 @@ MatchQuery::matchAttributesRegEx(
     boost::cmatch value_matcher;
     for (const boost::regex &val_regex : regex_values) {
         for (const string &requested_match_value : values) {
+            dbgTrace(D_RULEBASE_CONFIG) << "Matching value: '" << requested_match_value
+                                    << "' with regex: '" << val_regex << "'";
             if (NGEN::Regex::regexMatch(
                 __FILE__,
                 __LINE__,
@@ -352,10 +365,18 @@ MatchQuery::matchAttributesString(const set<string> &values) const
 bool
 MatchQuery::matchAttributesIp(const set<string> &values) const
 {
-    for (const IPRange &rule_ip_range : ip_addr_value) {
-        for (const string &requested_value : values) {
-            IpAddress ip_addr = IPUtilities::createIpFromString(requested_value);
-            if (IPUtilities::isIpAddrInRange(rule_ip_range, ip_addr)) return true;
+    for (const string &requested_value : values) {
+        int left = 0;
+        int right = ip_addr_value.size() - 1;
+        IpAddress ip_addr = IPUtilities::createIpFromString(requested_value);
+        while (left <= right) {
+            int mid = left + (right - left) / 2;
+            if (IPUtilities::isIpAddrInRange(ip_addr_value[mid], ip_addr)) return true;
+            if (ip_addr_value[mid].start < ip_addr) {
+                left = mid + 1;
+            } else {
+                right = mid - 1;
+            }
         }
     }
     return false;
@@ -367,8 +388,23 @@ MatchQuery::isRegEx() const
     return key != "protectionName";
 }
 
-bool
-MatchQuery::isIP() const
+void
+MatchQuery::sortAndMergeIpRangesValues()
 {
-    return key == "sourceIP" || key == "destinationIP";
+    if (ip_addr_value.empty()) return;
+
+    sort(ip_addr_value.begin(), ip_addr_value.end());
+    size_t mergedIndex = 0;
+    for (size_t i = 1; i < ip_addr_value.size(); ++i) {
+        if (ip_addr_value[i].start <= ip_addr_value[mergedIndex].end) {
+            if (ip_addr_value[mergedIndex].end <= ip_addr_value[i].end) {
+                ip_addr_value[mergedIndex].end = ip_addr_value[i].end;
+            }
+        } else {
+            ++mergedIndex;
+            ip_addr_value[mergedIndex] = ip_addr_value[i];
+        }
+    }
+
+    ip_addr_value.resize(mergedIndex + 1);
 }

components/utils/generic_rulebase/parameters_config.cc

@@ -119,10 +119,13 @@ ParameterException::getBehavior(
     for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
         MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
         if (match_res.is_match) {
-            dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches.";
+            dbgTrace(D_RULEBASE_CONFIG)
+                << "Successfully matched an exception from a list of matches, behavior: "
+                << match_behavior_pair.behavior.getId();
             // When matching indicators with action=ignore, we expect no behavior override.
             // Instead, a matched keywords list should be returned which will be later removed from score calculation
             if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore";
                 matched_override_keywords.insert(match_res.matched_keywords->begin(),
                         match_res.matched_keywords->end());
             } else {

components/utils/ip_utilities/ip_utilities.cc

@@ -22,7 +22,10 @@ bool
 operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
 {
     if (this_ip_addr.ip_type < other_ip_addr.ip_type) return true;
-    if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr < other_ip_addr.addr4_t.s_addr;
+    if (this_ip_addr.ip_type > other_ip_addr.ip_type) return false;
+    if (this_ip_addr.ip_type == IP_VERSION_4) {
+        return ntohl(this_ip_addr.addr4_t.s_addr) < ntohl(other_ip_addr.addr4_t.s_addr);
+    }
     return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) < 0;
 }
 
@@ -33,6 +36,19 @@ operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
     if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr == other_ip_addr.addr4_t.s_addr;
     return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) == 0;
 }
+
+bool
+operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
+{
+    if (this_ip_addr < other_ip_addr || this_ip_addr == other_ip_addr) return true;
+    return false;
+}
+
+bool
+operator<(const IPRange &range1, const IPRange &range2)
+{
+    return range1.start < range2.start || (range1.start == range2.start && range1.end < range2.end);
+}
 // LCOV_EXCL_STOP
 
 Maybe<pair<string, int>>

components/utils/keywords/keywords_rule.cc

@@ -5,6 +5,8 @@
 
 using namespace std;
 
+USE_DEBUG_FLAG(D_KEYWORD);
+
 static const string whitespaces = " \t";
 
 static string
@@ -14,6 +16,13 @@ getSubStrNoPadding(const string &str, uint start, uint end)
     auto r_end = str.find_last_not_of(whitespaces, end-1);
 
     if (r_end==string::npos || r_start==string::npos || r_start>r_end) {
+        dbgWarning(D_KEYWORD)
+            << "Can't extract substring from '"
+            << str
+            << "', padded start: "
+            << r_start
+            <<  ", padded end: "
+            << r_end;
         throw KeywordError("Found an empty section in the '"+ str + "'");
     }
 
@@ -45,13 +54,24 @@ split(const string &str, const string &delim, uint start = 0)
             }
             default:
                 if (!in_string && delim.find(str[index])!=string::npos) {
-                    res.push_back(getSubStrNoPadding(str, part_start, index));
+                    if (part_start == index) {
+                        dbgTrace(D_KEYWORD) << "Encountered consecutive delimiter in: " << str;
+                    } else {
+                        res.push_back(getSubStrNoPadding(str, part_start, index));
+                    }
                     part_start = index+1;
                 }
         }
     }
 
-    if (escape||in_string) throw KeywordError("Split has ended in the middle of the parsing");
+    if (escape||in_string) {
+        dbgWarning(D_KEYWORD)
+            << "Failed to split "
+            << str
+            << ". Split ended in "
+            << (escape ? "escape" : "middle of string");
+        throw KeywordError("Split has ended in the middle of the parsing");
+    }
 
     if (str.find_first_not_of(whitespaces, part_start)!=string::npos) {
         res.push_back(getSubStrNoPadding(str, part_start, str.size()));

components/utils/keywords/keywords_ut/keywords_ut.cc

@@ -55,6 +55,13 @@ TEST_F(KeywordsRuleTest, data_basic_test) {
     EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;"));
 }
 
+TEST_F(KeywordsRuleTest, consecutive_delimiter_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_TRUE(ruleRun("data: \"234\" , part  HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"75\", part  HTTP_RESPONSE_BODY;"));
+}
+
 TEST_F(KeywordsRuleTest, data_relative_test) {
     appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
 

core/CMakeLists.txt

@@ -37,7 +37,7 @@ target_link_libraries(
     ngen_core
     -Wl,-whole-archive
     "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest"
-    "config;intelligence_is_v2;event_is;memory_consumption;connkey"
+    "compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey"
     "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities"
     "report_messaging"
     -Wl,-no-whole-archive

core/agent_details/agent_details.cc

@@ -304,6 +304,12 @@ AgentDetails::getOrchestrationMode() const
     return orchestration_mode;
 }
 
+bool
+AgentDetails::isOpenAppsecAgent() const
+{
+    return (orchestration_mode == OrchestrationMode::HYBRID) || (tenant_id.rfind("org_", 0) == 0);
+}
+
 string
 AgentDetails::getAccessToken() const
 {

core/agent_details_reporter/agent_details_reporter.cc

@@ -117,7 +117,7 @@ private:
                 return;
             }
 
-            dbgInfo(D_AGENT_DETAILS)
+            dbgDebug(D_AGENT_DETAILS)
                 << "Successfully handled attributes persistence. Operation: "
                 << operation
                 << ", Path "

core/config/config.cc

@@ -315,22 +315,13 @@ void
 ConfigComponent::Impl::preload()
 {
     I_Environment *environment = Singleton::Consume<I_Environment>::by<ConfigComponent>();
-    auto executable = environment->get<string>("Executable Name");
+    auto executable = environment->get<string>("Base Executable Name");
     if (!executable.ok() || *executable == "") {
         dbgWarning(D_CONFIG)
             << "Could not load nano service's settings since \"Executable Name\" in not found in the environment";
         return;
     }
-
     executable_name = *executable;
-    auto file_path_end = executable_name.find_last_of("/");
-    if (file_path_end != string::npos) {
-        executable_name = executable_name.substr(file_path_end + 1);
-    }
-    auto file_sufix_start = executable_name.find_first_of(".");
-    if (file_sufix_start != string::npos) {
-        executable_name = executable_name.substr(0, file_sufix_start);
-    }
 
     config_file_paths.insert(executable_name + "-conf.json");
     config_file_paths.insert(executable_name + "-debug-conf.json");

core/debug_is/debug.cc

@@ -766,22 +766,8 @@ Debug::findDebugFilePrefix(const string &file_name)
 string
 Debug::getExecutableName()
 {
-    auto executable = env->get<string>("Executable Name");
-    if (!executable.ok() || *executable == "") {
-        return "";
-    }
-
-    string executable_name = *executable;
-    auto file_path_end = executable_name.find_last_of("/");
-    if (file_path_end != string::npos) {
-        executable_name = executable_name.substr(file_path_end + 1);
-    }
-    auto file_sufix_start = executable_name.find_first_of(".");
-    if (file_sufix_start != string::npos) {
-        executable_name = executable_name.substr(0, file_sufix_start);
-    }
-
-    return executable_name;
+    auto executable = env->get<string>("Base Executable Name");
+    return executable.ok() ? *executable : "";
 }
 
 void

core/debug_is/debug_is_ut/debug_ut.cc

@@ -836,7 +836,7 @@ TEST_F(DebugConfigTest, testSetConfig)
     EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
 
     env.preload();
-    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "debug-ut");
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "debug-ut");
     env.init();
 
     Debug::init();

core/environment/environment.cc

@@ -265,18 +265,9 @@ Environment::Impl::getCurrentHeadersMap()
         tracing_headers["X-Span-Id"] = span_id;
     }
 
-    auto exec_name = get<string>("Executable Name");
+    auto exec_name = get<string>("Base Executable Name");
     if (exec_name.ok() && *exec_name != "") {
-        string executable_name = *exec_name;
-        auto file_path_end = executable_name.find_last_of("/");
-        if (file_path_end != string::npos) {
-            executable_name = executable_name.substr(file_path_end + 1);
-        }
-        auto file_sufix_start = executable_name.find_first_of(".");
-        if (file_sufix_start != string::npos) {
-            executable_name = executable_name.substr(0, file_sufix_start);
-        }
-        tracing_headers["X-Calling-Service"] = executable_name;
+        tracing_headers["X-Calling-Service"] = *exec_name;
     }
 
     return tracing_headers;

core/environment/environment_ut/tracing_ut.cc

@@ -346,7 +346,7 @@ TEST_F(TracingCompRoutinesTest, 2SpansDifFlow)
 {
     I_MainLoop::Routine routine = [&] () {
         string service_name = "test-service-name";
-        i_env->registerValue("Executable Name", service_name);
+        i_env->registerValue("Base Executable Name", service_name);
 
         i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9");
         trace_id = i_env->getCurrentTrace();

core/include/services_sdk/interfaces/i_agent_details.h

@@ -46,6 +46,7 @@ public:
     virtual OrchestrationMode getOrchestrationMode()                  const = 0;
     virtual std::string getAccessToken()                              const = 0;
     virtual void loadAccessToken()                                          = 0;
+    virtual bool isOpenAppsecAgent()                                  const = 0;
 
     // OpenSSL
     virtual void setOpenSSLDir(const std::string &openssl_dir)  = 0;

core/include/services_sdk/interfaces/i_env_details.h

@@ -24,6 +24,7 @@ class I_EnvDetails
 public:
     virtual EnvType getEnvType() = 0;
     virtual std::string getToken() = 0;
+    virtual std::string getNameSpace() = 0;
 
 protected:
     virtual ~I_EnvDetails() {}

core/include/services_sdk/interfaces/messaging/http_response.h

@@ -31,15 +31,26 @@ public:
     HTTPResponse() = default;
 // LCOV_EXCL_STOP
 
-    HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {}
+    HTTPResponse(
+        HTTPStatusCode _status_code,
+        const std::string &_body,
+        std::unordered_map<std::string, std::string> _headers = std::unordered_map<std::string, std::string>()
+    )
+            :
+        status_code(_status_code),
+        body(_body),
+        headers(_headers)
+    {}
 
     HTTPStatusCode getHTTPStatusCode() const;
     const std::string & getBody() const;
     std::string toString() const;
+    Maybe<std::string> getHeaderVal(const std::string &header_key);
 
 private:
     HTTPStatusCode status_code;
     std::string body;
+    std::unordered_map<std::string, std::string> headers;
 };
 
 #endif // __HTTP_RESPONSE_H__

core/include/services_sdk/interfaces/messaging/messaging_enums.h

@@ -63,6 +63,7 @@ enum class HTTPStatusCode
     HTTP_PROXY_AUTHENTICATION_REQUIRED = 407,
     HTTP_REQUEST_TIME_OUT = 408,
     HTTP_PAYLOAD_TOO_LARGE = 413,
+    HTTP_TOO_MANY_REQUESTS = 429,
     // 5xx - Server error responses.
     HTTP_INTERNAL_SERVER_ERROR = 500,
     HTTP_NOT_IMPLEMENTED = 501,

core/include/services_sdk/interfaces/messaging/messaging_metadata.h

@@ -8,6 +8,7 @@
 #include "config.h"
 #include "singleton.h"
 #include "i_agent_details.h"
+#include "i_time_get.h"
 
 class MessageProxySettings
 {
@@ -54,7 +55,7 @@ private:
     uint16_t proxy_port = 0;
 };
 
-class MessageMetadata
+class MessageMetadata : Singleton::Consume<I_TimeGet>
 {
 public:
     inline MessageMetadata();
@@ -215,6 +216,38 @@ public:
         return is_to_fog;
     }
 
+    void
+    setSniHostName(const std::string &_host_name)
+    {
+        sni_host_name = _host_name;
+    }
+
+    Maybe<std::string>
+    getSniHostName() const
+    {
+        return sni_host_name;
+    }
+
+    void
+    setRateLimitBlock(uint block_time)
+    {
+        is_rate_limit_block = true;
+        auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+        auto current_timeout = timer->getMonotonicTime() + std::chrono::seconds(block_time);
+        rate_limit_block_time = current_timeout.count();
+    }
+
+    bool
+    isRateLimitBlock() const
+    {
+        if (is_rate_limit_block) {
+            auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+            uint current_time = timer->getMonotonicTime().count();
+            if (current_time < rate_limit_block_time) return true;
+        }
+        return false;
+    }
+
     template <class Archive>
     void
     serialize(Archive &ar)
@@ -231,12 +264,15 @@ public:
             cereal::make_nvp("is_to_fog", is_to_fog),
             cereal::make_nvp("ca_path", ca_path),
             cereal::make_nvp("client_cert_path", client_cert_path),
-            cereal::make_nvp("client_key_path", client_key_path)
+            cereal::make_nvp("client_key_path", client_key_path),
+            cereal::make_nvp("is_rate_limit_block", is_rate_limit_block),
+            cereal::make_nvp("rate_limit_block_time", rate_limit_block_time)
         );
     }
 
 private:
     std::string host_name = "";
+    Maybe<std::string> sni_host_name = genError("SNI host name not set");
     std::string ca_path = "";
     std::string client_cert_path = "";
     std::string client_key_path = "";
@@ -249,6 +285,8 @@ private:
     std::string external_certificate = "";
     bool should_buffer = false;
     bool is_to_fog = false;
+    bool is_rate_limit_block = false;
+    uint rate_limit_block_time = 0;
 };
 
 #endif // __MESSAGING_METADATA_H__

core/include/services_sdk/interfaces/mock/mock_agent_details.h

@@ -28,6 +28,7 @@ public:
     MOCK_CONST_METHOD0(getAgentId, std::string());
     MOCK_METHOD0(loadAccessToken, void());
     MOCK_CONST_METHOD0(getAccessToken, std::string());
+    MOCK_CONST_METHOD0(isOpenAppsecAgent, bool());
 
     // OpenSSL
     MOCK_METHOD1(setOpenSSLDir, void(const std::string&));

core/include/services_sdk/resources/agent_details.h

@@ -73,6 +73,7 @@ public:
     Maybe<std::string> getOpenSSLDir()       const;
     std::string getClusterId()               const;
     OrchestrationMode getOrchestrationMode() const;
+    bool isOpenAppsecAgent()                 const;
     std::string getAccessToken()             const;
     void loadAccessToken();
 

core/include/services_sdk/resources/component_is/components_list_impl.h

@@ -165,6 +165,14 @@ public:
         }
 
         registerGlobalValue<std::string>("Executable Name", arg_vec.front());
+
+        auto file_path_end = arg_vec.front().find_last_of("/");
+        auto executable_name = arg_vec.front().substr(file_path_end + 1);
+        auto file_sufix_start = executable_name.find_first_of(".");
+        if (file_sufix_start != std::string::npos) {
+            executable_name = executable_name.substr(0, file_sufix_start);
+        }
+        registerGlobalValue<std::string>("Base Executable Name", executable_name);
     }
 
     void

core/include/services_sdk/resources/debug_flags.h

@@ -76,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
             DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
             DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP)
+            DEFINE_FLAG(D_WAAP_MODEL_LOGGER, D_WAAP)
             DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP)
             DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
             DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
@@ -106,6 +107,8 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
             DEFINE_FLAG(D_WAAP_PARSER_BINARY_FILE, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_SCREENED_JSON, D_WAAP_PARSER)
 
     DEFINE_FLAG(D_IPS, D_COMPONENT)
     DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT)
@@ -190,6 +193,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
+    DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
 
 DEFINE_FLAG(D_FLOW, D_ALL)
     DEFINE_FLAG(D_DROP, D_FLOW)

core/include/services_sdk/resources/generic_metric.h

@@ -24,6 +24,8 @@
 #include "i_mainloop.h"
 #include "i_time_get.h"
 #include "i_agent_details.h"
+#include "i_encryptor.h"
+#include "i_instance_awareness.h"
 #include "i_environment.h"
 #include "i_messaging.h"
 #include "i_rest_api.h"
@@ -52,9 +54,11 @@ class GenericMetric
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_TimeGet>,
     Singleton::Consume<I_AgentDetails>,
+    Singleton::Consume<I_InstanceAwareness>,
     Singleton::Consume<I_Environment>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Encryptor>,
     public Listener<AllMetricEvent>
 {
 public:
@@ -111,7 +115,9 @@ private:
 
     void handleMetricStreamSending();
     void generateLog();
+    void generatePrometheus();
     void generateDebug();
+    void generateAiopsLog();
 
     I_MainLoop *i_mainloop;
     I_TimeGet *i_time;

core/include/services_sdk/resources/log_generator.h

@@ -102,6 +102,8 @@ public:
 
     std::string getLogInsteadOfSending();
 
+    void addMarkerSuffix(const std::string &suffix);
+
 private:
     std::chrono::microseconds getCurrentTime() const;
     void loadBaseLogFields();

core/include/services_sdk/resources/metric/average.h

@@ -54,6 +54,12 @@ public:
         return (count > 0) ? double(sum)/count : 0;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(getAverage());
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/counter.h

@@ -44,6 +44,12 @@ public:
         return counter;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(counter);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/last_reported_value.h

@@ -44,6 +44,12 @@ public:
         return last_reported;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(last_reported);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/max.h

@@ -56,6 +56,12 @@ public:
         return max;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(max);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/metric_calc.h

@@ -18,21 +18,202 @@
 #error metric/metric_calc.h should not be included directly
 #endif // __GENERIC_METRIC_H_
 
+#include <cmath>
 #include <cereal/archives/json.hpp>
 
 #include "report/report.h"
+#include "customized_cereal_map.h"
+#include "compression_utils.h"
+#include "i_encryptor.h"
 
 class GenericMetric;
 
 enum class MetricType { GAUGE, COUNTER };
 
+struct PrometheusData
+{
+    std::string name;
+    std::string type;
+    std::string desc;
+    std::string label;
+    std::string value;
+};
+
+class AiopsMetricData
+{
+public:
+    AiopsMetricData(
+        const std::string &_name,
+        const std::string &_type,
+        const std::string &_units,
+        const std::string &_description,
+        std::map<std::string, std::string> _resource_attributes,
+        float _value)
+            :
+        name(_name),
+        type(_type),
+        units(_units),
+        description(_description),
+        resource_attributes(_resource_attributes),
+        value(_value)
+    {
+        timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
+        // convert timestamp to RFC 3339 format
+        std::size_t pos = timestamp.find('.');
+        if (pos != std::string::npos) {
+            timestamp = timestamp.substr(0, pos) + "Z";
+        }
+        asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
+    }
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        ar(cereal::make_nvp("Timestamp", timestamp));
+        ar(cereal::make_nvp("MetricName", name));
+        ar(cereal::make_nvp("MetricType", type));
+        ar(cereal::make_nvp("MetricUnit", units));
+        ar(cereal::make_nvp("MetricDescription", description));
+        ar(cereal::make_nvp("MetricValue", value));
+        ar(cereal::make_nvp("ResourceAttributes", resource_attributes));
+        ar(cereal::make_nvp("MetricAttributes", metric_attributes));
+        ar(cereal::make_nvp("AssetID", asset_id));
+    }
+
+    std::string
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        return ss.str();
+    }
+
+    void
+    addMetricAttribute(const std::string &label, const std::string &value)
+    {
+        metric_attributes[label] = value;
+    }
+
+private:
+    std::string timestamp = "";
+    std::string asset_id = "";
+    std::string name;
+    std::string type;
+    std::string units;
+    std::string description;
+    std::map<std::string, std::string> resource_attributes;
+    std::map<std::string, std::string> metric_attributes;
+    float value = 0;
+};
+
+class AiopsMetricList
+{
+public:
+    void
+    addMetrics(const std::vector<AiopsMetricData> &_metrics)
+    {
+        metrics.insert(metrics.end(), _metrics.begin(), _metrics.end());
+    }
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        ar(cereal::make_nvp("Metrics", metrics));
+    }
+
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+    Maybe<std::string>
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        auto res = compressAndEncodeData(ss.str());
+        if (!res.ok()) {
+            return genError("Failed to compress and encode the data");
+        }
+        return res.unpack();
+    }
+// LCOV_EXCL_STOP
+
+private:
+    Maybe<std::string>
+    compressAndEncodeData(const std::string &unhandled_data) const
+    {
+        std::string data_holder = unhandled_data;
+        auto compression_stream = initCompressionStream();
+        CompressionResult compression_response = compressData(
+            compression_stream,
+            CompressionType::GZIP,
+            data_holder.size(),
+            reinterpret_cast<const unsigned char *>(data_holder.c_str()),
+            true
+        );
+        finiCompressionStream(compression_stream);
+        if (!compression_response.ok) {
+            // send log to Kibana
+            return genError("Failed to compress(gzip) data");
+        }
+
+        std::string compressed_data =
+            std::string((const char *)compression_response.output, compression_response.num_output_bytes);
+
+        auto encryptor = Singleton::Consume<I_Encryptor>::by<GenericMetric>();
+        Maybe<std::string> handled_data = encryptor->base64Encode(compressed_data);
+
+        if (compression_response.output) free(compression_response.output);
+        compression_response.output = nullptr;
+        compression_response.num_output_bytes = 0;
+        return handled_data;
+    }
+
+    std::vector<AiopsMetricData> metrics;
+};
+
+class CompressAndEncodeAIOPSMetrics
+{
+public:
+    CompressAndEncodeAIOPSMetrics(const AiopsMetricList &_aiops_metrics) : aiops_metrics(_aiops_metrics) {}
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        auto metric_str = aiops_metrics.toString();
+        if (!metric_str.ok()) {
+            return;
+        }
+        ar(cereal::make_nvp("records", metric_str.unpack()));
+    }
+
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+    Maybe<std::string>
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        return ss.str();
+    }
+// LCOV_EXCL_STOP
+
+private:
+    AiopsMetricList aiops_metrics;
+};
+
 class MetricCalc
 {
 public:
     template<typename ... Args>
     MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args)
     {
-        setMetadata("BaseName", calc_title);
+        setMetricName(calc_title);
         addMetric(metric);
         parseMetadata(args ...);
     }
@@ -47,7 +228,11 @@ public:
     std::string getMetircDescription() const { return getMetadata("Description"); }
     std::string getMetadata(const std::string &metadata) const;
     virtual MetricType getMetricType() const { return MetricType::GAUGE; }
+    virtual std::vector<PrometheusData> getPrometheusMetrics() const;
+    virtual float getValue() const = 0;
+    virtual std::vector<AiopsMetricData> getAiopsMetrics() const;
 
+    void setMetricName(const std::string &name) { setMetadata("BaseName", name); }
     void setMetricDotName(const std::string &name) { setMetadata("DotName", name); }
     void setMetircUnits(const std::string &units) { setMetadata("Units", units); }
     void setMetircDescription(const std::string &description) { setMetadata("Description", description); }
@@ -55,6 +240,7 @@ public:
 
 protected:
     void addMetric(GenericMetric *metric);
+    std::map<std::string, std::string> getBasicLabels() const;
 
     template <typename Metadata, typename ... OtherMetadata>
     void

core/include/services_sdk/resources/metric/metric_map.h

@@ -54,6 +54,37 @@ class MetricMap : public MetricCalc
             return first->second.getMetricType();
         }
 
+        std::vector<PrometheusData>
+        getPrometheusMetrics(const std::string &label, const std::string &name) const
+        {
+            std::vector<PrometheusData> res;
+
+            for (auto &metric : inner_map) {
+                auto sub_res =  metric.second.getPrometheusMetrics();
+                for (auto &sub_metric : sub_res) {
+                    sub_metric.label += "," + label + "=\"" + metric.first + "\"";
+                    sub_metric.name = name;
+                }
+                res.insert(res.end(), sub_res.begin(), sub_res.end());
+            }
+
+            return res;
+        }
+
+        std::vector<AiopsMetricData>
+        getAiopsMetrics(const std::string &label) const
+        {
+            std::vector<AiopsMetricData> aiops_metrics;
+            for (auto &metric : inner_map) {
+                auto metric_data = metric.second.getAiopsMetrics();
+                for (auto &sub_metric : metric_data) {
+                    sub_metric.addMetricAttribute(label,  metric.first);
+                }
+                aiops_metrics.insert(aiops_metrics.end(), metric_data.begin(), metric_data.end());
+            }
+            return aiops_metrics;
+        }
+
         typename std::map<std::string, Metric>::const_iterator begin() const { return inner_map.begin(); }
         typename std::map<std::string, Metric>::const_iterator end() const { return inner_map.end(); }
 
@@ -63,9 +94,17 @@ class MetricMap : public MetricCalc
 
 public:
     template <typename ... Args>
-    MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args)
+    MetricMap(
+        const Metric &sub_metric,
+        GenericMetric *metric,
+        const std::string &l,
+        const std::string &title,
+        const Args & ... args
+    )
             :
-        MetricCalc(metric, title, args ...)
+        MetricCalc(metric, title, args ...),
+        base_metric(sub_metric),
+        label(l)
     {
     }
 
@@ -75,6 +114,14 @@ public:
         if (getMetricType() == MetricType::GAUGE) metric_map.clear();
     }
 
+// LCOV_EXCL_START Reason: Covered by printPromeathusMultiMap unit-test, but misdetected by the coverage
+    float
+    getValue() const override
+    {
+        return std::nanf("");
+    }
+// LCOV_EXCL_STOP
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {
@@ -89,7 +136,9 @@ public:
     {
         std::stringstream string_key;
         string_key << key;
-        auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first;
+        auto new_metric = base_metric;
+        new_metric.setMetricName(string_key.str());
+        auto metric = metric_map.emplace(string_key.str(), std::move(new_metric)).first;
         metric->second.report(new_values...);
     }
 
@@ -105,8 +154,22 @@ public:
         return field;
     }
 
+    std::vector<PrometheusData>
+    getPrometheusMetrics() const override
+    {
+        return metric_map.getPrometheusMetrics(label, getMetricName());
+    }
+
+    std::vector<AiopsMetricData>
+    getAiopsMetrics() const
+    {
+        return metric_map.getAiopsMetrics(label);
+    }
+
 private:
     InnerMap metric_map;
+    Metric base_metric;
+    std::string label;
 };
 
 } // namespace MetricCalculations