Merge pull request #200 from openappsec/Oct_14_2024-Dev

Oct 14 2024 dev

components/include/env_details.h

@@ -29,12 +29,15 @@ public:
 
     virtual EnvType getEnvType() override;
     virtual std::string getToken() override;
+    virtual std::string getNameSpace() override;
 
 private:
     std::string retrieveToken();
+    std::string retrieveNamespace();
     std::string readFileContent(const std::string &file_path);
 
     std::string token;
+    std::string agent_namespace;
     EnvType env_type;
 };
 

components/include/generic_rulebase/match_query.h

@@ -91,7 +91,7 @@ private:
     bool matchAttributesString(const std::set<std::string> &values) const;
     bool matchAttributesIp(const std::set<std::string> &values) const;
     bool isRegEx() const;
-    bool isIP() const;
+    void sortAndMergeIpRangesValues();
 
     MatchType type;
     Operators operator_type;

components/include/ip_utilities.h

@@ -28,8 +28,9 @@
 
 // LCOV_EXCL_START Reason: temporary until we add relevant UT until 07/10
 bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
-
 bool operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<(const IPRange &range1, const IPRange &range2);
 // LCOV_EXCL_STOP
 
 Maybe<std::pair<std::string, int>> extractAddressAndMaskSize(const std::string &cidr);

components/include/waap.h

@@ -33,6 +33,7 @@ class I_WaapAssetStatesManager;
 class I_Messaging;
 class I_AgentDetails;
 class I_Encryptor;
+class I_WaapModelResultLogger;
 
 const std::string WAAP_APPLICATION_NAME = "waap application";
 
@@ -50,7 +51,8 @@ class WaapComponent
     Singleton::Consume<I_AgentDetails>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_Encryptor>,
-    Singleton::Consume<I_Environment>
+    Singleton::Consume<I_Environment>,
+    Singleton::Consume<I_WaapModelResultLogger>
 {
 public:
     WaapComponent();

components/security_apps/local_policy_mgmt_gen/access_control_practice.cc

@@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
 
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
-    parseAppsecJSONKey<string>("practiceMode", mode, archive_in);
+    parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Access control practice mode invalid: " + mode);
+    }
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<AccessControlRateLimit>("rateLimit", rate_limit, archive_in);
 }

components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc

@@ -438,19 +438,30 @@ WebAppSection::WebAppSection(
     csrf_protection_mode("Disabled"),
     open_redirect_mode("Disabled"),
     error_disclosure_mode("Disabled"),
+    schema_validation_mode("Disabled"),
+    schema_validation_enforce_level("fullSchema"),
     practice_advanced_config(parsed_appsec_spec),
     anti_bots(parsed_appsec_spec.getAntiBot()),
     trusted_sources({ parsed_trusted_sources })
 {
+    auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
+    if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << mitigation_sevirity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
     web_attack_mitigation_severity =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
+        web_attack_mitigation_severity;
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
@@ -479,6 +490,9 @@ WebAppSection::WebAppSection(
     const string &_web_attack_mitigation_severity,
     const string &_web_attack_mitigation_mode,
     const string &_bot_protection,
+    const string &_schema_validation_mode,
+    const string &_schema_validation_enforce_level,
+    const vector<string> &_schema_validation_oas,
     const PracticeAdvancedConfig &_practice_advanced_config,
     const AppsecPracticeAntiBotSection &_anti_bots,
     const LogTriggerSection &parsed_log_trigger,
@@ -493,19 +507,29 @@ WebAppSection::WebAppSection(
     practice_id(_practice_id),
     practice_name(_practice_name),
     context(_context),
-    web_attack_mitigation_severity(_web_attack_mitigation_severity),
     web_attack_mitigation_mode(_web_attack_mitigation_mode),
     bot_protection(_bot_protection),
+    schema_validation_mode(_schema_validation_mode),
+    schema_validation_enforce_level(_schema_validation_enforce_level),
+    schema_validation_oas(_schema_validation_oas),
     practice_advanced_config(_practice_advanced_config),
     anti_bots(_anti_bots),
     trusted_sources({ parsed_trusted_sources })
 {
+    if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << _web_attack_mitigation_severity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
@@ -516,6 +540,7 @@ WebAppSection::WebAppSection(
     for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
         overrides.push_back(AppSecOverride(source_ident));
     }
+
 }
 
 // LCOV_EXCL_STOP
@@ -523,7 +548,6 @@ WebAppSection::WebAppSection(
 void
 WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
 {
-    string disabled_str = "Disabled";
     vector<string> empty_list;
     out_ar(
         cereal::make_nvp("context",                         context),
@@ -541,9 +565,10 @@ WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
         cereal::make_nvp("assetName",                       asset_name),
         cereal::make_nvp("ruleId",                          rule_id),
         cereal::make_nvp("ruleName",                        rule_name),
-        cereal::make_nvp("schemaValidation",               false),
+        cereal::make_nvp("schemaValidation",                schema_validation_mode == "Prevent"),
-        cereal::make_nvp("schemaValidation_v2",            disabled_str),
+        cereal::make_nvp("schemaValidation_v2",             schema_validation_mode),
-        cereal::make_nvp("oas",                            empty_list),
+        cereal::make_nvp("oas",                             schema_validation_oas),
+        cereal::make_nvp("schemaValidationEnforceLevel",    schema_validation_enforce_level),
         cereal::make_nvp("triggers",                        triggers),
         cereal::make_nvp("applicationUrls",                 application_urls),
         cereal::make_nvp("overrides",                       overrides),

components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h

@@ -291,6 +291,9 @@ public:
         const std::string &_web_attack_mitigation_severity,
         const std::string &_web_attack_mitigation_mode,
         const std::string &_bot_protection,
+        const std::string &schema_validation_mode,
+        const std::string &schema_validation_enforce_level,
+        const std::vector<std::string> &schema_validation_oas,
         const PracticeAdvancedConfig &_practice_advanced_config,
         const AppsecPracticeAntiBotSection &_anti_bots,
         const LogTriggerSection &parsed_log_trigger,
@@ -302,6 +305,7 @@ public:
     bool operator< (const WebAppSection &other) const;
 
 private:
+    bool                                    web_attack_mitigation;
     std::string                             application_urls;
     std::string                             asset_id;
     std::string                             asset_name;
@@ -317,12 +321,14 @@ private:
     std::string                             open_redirect_mode;
     std::string                             error_disclosure_mode;
     std::string                             bot_protection;
-    bool web_attack_mitigation;
+    std::string                             schema_validation_mode;
-    std::vector<TriggersInWaapSection> triggers;
+    std::string                             schema_validation_enforce_level;
+    std::vector<std::string>                schema_validation_oas;
     PracticeAdvancedConfig                  practice_advanced_config;
     AppsecPracticeAntiBotSection            anti_bots;
-    std::vector<AppSecTrustedSources> trusted_sources;
     std::vector<AppSecOverride>             overrides;
+    std::vector<AppSecTrustedSources>       trusted_sources;
+    std::vector<TriggersInWaapSection>      triggers;
 };
 
 class WebAPISection
@@ -410,7 +416,7 @@ class ParsedRule
 {
 public:
     ParsedRule() {}
-    ParsedRule(const std::string &_host) : host(_host) {}
+    ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
     const std::vector<std::string> & getExceptions() const;

components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h

@@ -24,6 +24,7 @@
 #include "maybe_res.h"
 #include "i_orchestration_tools.h"
 #include "i_shell_cmd.h"
+#include "i_encryptor.h"
 #include "i_messaging.h"
 #include "i_env_details.h"
 #include "i_agent_details.h"
@@ -40,6 +41,7 @@ class K8sPolicyUtils
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_ShellCmd>,
     Singleton::Consume<I_EnvDetails>,
+    Singleton::Consume<I_Encryptor>,
     Singleton::Consume<I_AgentDetails>
 {
 public:
@@ -80,6 +82,8 @@ private:
 
     void createSnortFile(std::vector<NewAppSecPracticeSpec> &practices) const;
 
+    void createSchemaValidationOas(std::vector<NewAppSecPracticeSpec> &practices) const;
+
     template<class T>
     std::vector<T> extractV1Beta2ElementsFromCluster(
         const std::string &crd_plural,
@@ -112,6 +116,7 @@ private:
     I_Messaging* messaging = nullptr;
     EnvType env_type;
     std::string token;
+    std::string agent_ns;
 };
 
 #endif // __K8S_POLICY_UTILS_H__

components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h

@@ -49,6 +49,13 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
     { "WebUserResponse", TriggerType::WebUserResponse }
 };
 
+static const std::unordered_map<std::string, std::string> key_to_mitigation_severity = {
+    { "high", "High"},
+    { "medium", "Medium"},
+    { "critical", "Critical"},
+    { "Transparent", "Transparent"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -57,6 +64,14 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val =
     { "inactive", "Inactive"}
 };
 
+static const std::unordered_map<std::string, std::string> key_to_practices_mode_val = {
+    { "prevent-learn", "Prevent"},
+    { "detect-learn", "Detect"},
+    { "prevent", "Prevent"},
+    { "detect", "Detect"},
+    { "inactive", "Disabled"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val2 = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -66,6 +81,8 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val2
 };
 
 static const std::string default_appsec_url = "http://*:*";
+static const std::string default_appsec_name = "Any";
+
 
 class PolicyGenException : public std::exception
 {

components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h

@@ -31,7 +31,7 @@ class NewParsedRule
 {
 public:
     NewParsedRule() {}
-    NewParsedRule(const std::string &_host) : host(_host) {}
+    NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
 

components/security_apps/local_policy_mgmt_gen/include/new_practice.h

@@ -23,6 +23,8 @@
 #include "config.h"
 #include "debug.h"
 #include "local_policy_common.h"
+#include "i_orchestration_tools.h"
+#include "i_encryptor.h"
 
 bool isModeInherited(const std::string &mode);
 
@@ -88,6 +90,8 @@ public:
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
+    bool operator<(const IpsProtectionsSection &other) const;
+
 private:
     std::string                                 context;
     std::string                                 name;
@@ -105,7 +109,7 @@ public:
     // LCOV_EXCL_START Reason: no test exist
     IPSSection() {};
 
-    IPSSection(const std::vector<IpsProtectionsSection> &_ips) : ips(_ips) {};
+    IPSSection(const std::vector<IpsProtectionsSection> &_ips);
     // LCOV_EXCL_STOP
 
     void save(cereal::JSONOutputArchive &out_ar) const;
@@ -138,6 +142,12 @@ public:
     const std::string & getMode(const std::string &default_mode = "inactive") const;
 
 private:
+
+    const std::string & getRulesMode(
+        const std::string &mode,
+        const std::string &default_mode = "inactive"
+    ) const;
+
     std::string override_mode;
     std::string max_performance_impact;
     std::string min_severity_level;
@@ -487,15 +497,16 @@ private:
     SnortSection snort;
 };
 
-class NewSnortSignaturesAndOpenSchemaAPI
+class NewSnortSignatures
 {
 public:
-    NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {};
+    NewSnortSignatures() : is_temporary(false) {};
 
     void load(cereal::JSONInputArchive &archive_in);
 
     void addFile(const std::string &file_name);
     const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
     const std::vector<std::string> & getConfigMap() const;
     const std::vector<std::string> & getFiles() const;
     bool isTemporary() const;
@@ -503,17 +514,40 @@ public:
 
 private:
     std::string override_mode;
+    std::string enforcement_level;
     std::vector<std::string> config_map;
     std::vector<std::string> files;
     bool is_temporary;
 };
 
+class NewOpenApiSchema : Singleton::Consume<I_OrchestrationTools>, Singleton::Consume<I_Encryptor>
+{
+public:
+    NewOpenApiSchema() {};
+
+    void load(cereal::JSONInputArchive &archive_in);
+
+    void addOas(const std::string &file);
+    const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
+    const std::vector<std::string> & getConfigMap() const;
+    const std::vector<std::string> & getFiles() const;
+    const std::vector<std::string> & getOas() const;
+
+private:
+    std::string override_mode;
+    std::string enforcement_level;
+    std::vector<std::string> config_map;
+    std::vector<std::string> files;
+    std::vector<std::string> oas;
+};
+
 class NewAppSecPracticeAntiBot
 {
 public:
     const std::vector<std::string> & getIjectedUris() const;
     const std::vector<std::string> & getValidatedUris() const;
-    const std::string & getMode() const;
+    const std::string & getMode(const std::string &default_mode = "inactive") const;
 
     void load(cereal::JSONInputArchive &archive_in);
     void save(cereal::JSONOutputArchive &out_ar) const;
@@ -569,8 +603,8 @@ class NewAppSecPracticeSpec
 public:
     void load(cereal::JSONInputArchive &archive_in);
 
-    NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures();
+    NewSnortSignatures & getSnortSignatures();
-    const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const;
+    NewOpenApiSchema & getOpenSchemaValidation();
     const NewAppSecPracticeWebAttacks & getWebAttacks() const;
     const NewAppSecPracticeAntiBot & getAntiBot() const;
     const NewIntrusionPrevention & getIntrusionPrevention() const;
@@ -583,8 +617,8 @@ public:
 private:
     NewFileSecurity                             file_security;
     NewIntrusionPrevention                      intrusion_prevention;
-    NewSnortSignaturesAndOpenSchemaAPI          openapi_schema_validation;
+    NewOpenApiSchema                            openapi_schema_validation;
-    NewSnortSignaturesAndOpenSchemaAPI          snort_signatures;
+    NewSnortSignatures                          snort_signatures;
     NewAppSecPracticeWebAttacks                 web_attacks;
     NewAppSecPracticeAntiBot                    anti_bot;
     std::string                                 appsec_class_name;

components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h

@@ -123,6 +123,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
@@ -145,6 +146,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 

components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc

@@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key)
     }
 }
 
+string
+getAppSecScopeType()
+{
+    auto env_res = getenv("CRDS_SCOPE");
+    if (env_res != nullptr) return env_res;
+    return "cluster";
+}
+
 void
 K8sPolicyUtils::init()
 {
@@ -42,6 +50,7 @@ K8sPolicyUtils::init()
     env_type = env_details->getEnvType();
     if (env_type == EnvType::K8S) {
         token = env_details->getToken();
+        agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : "";
         messaging = Singleton::Consume<I_Messaging>::by<K8sPolicyUtils>();
     }
 }
@@ -140,10 +149,12 @@ extractElementsFromNewRule(
     const NewParsedRule &rule,
     map<AnnotationTypes, unordered_set<string>> &policy_elements_names)
 {
+    if (rule.getExceptions().size() > 0) {
         policy_elements_names[AnnotationTypes::EXCEPTION].insert(
             rule.getExceptions().begin(),
             rule.getExceptions().end()
         );
+    }
     policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert(
         rule.getPractices().begin(),
         rule.getPractices().end()
@@ -152,14 +163,24 @@ extractElementsFromNewRule(
         rule.getAccessControlPractices().begin(),
         rule.getAccessControlPractices().end()
     );
+    if (rule.getLogTriggers().size() > 0) {
         policy_elements_names[AnnotationTypes::TRIGGER].insert(
             rule.getLogTriggers().begin(),
             rule.getLogTriggers().end()
         );
+    }
+    if (rule.getCustomResponse() != "" ) {
         policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
+    }
+    if (rule.getSourceIdentifiers() != "" ) {
         policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
+    }
+    if (rule.getTrustedSources() != "" ) {
         policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
+    }
+    if (rule.getUpgradeSettings() != "" ) {
         policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
+    }
 }
 
 map<AnnotationTypes, unordered_set<string>>
@@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster(
     dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural;
     vector<T> elements;
     for (const string &element_name : elements_names) {
+        string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+        string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
         dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name;
         auto maybe_appsec_element = getObjectFromCluster<AppsecSpecParser<T>>(
-            "/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name
+            "/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name
         );
 
         if (!maybe_appsec_element.ok()) {
@@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
         practice.getSnortSignatures().setTemporary(true);
         for (const string &config_map : practice.getSnortSignatures().getConfigMap())
         {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
             auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
-                "/api/v1/namespaces/default/configmaps/" + config_map
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
             );
             if (!maybe_configmap.ok())  {
                 dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
@@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
     }
 }
 
+void
+K8sPolicyUtils::createSchemaValidationOas(vector<NewAppSecPracticeSpec> &practices) const
+{
+    for (NewAppSecPracticeSpec &practice : practices) {
+        vector<string> res;
+        for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap())
+        {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
+            auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
+            );
+            if (!maybe_configmap.ok())  {
+                dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
+                continue;
+            }
+            string file_content = maybe_configmap.unpack().getFileContent();
+            string res = Singleton::Consume<I_Encryptor>::by<K8sPolicyUtils>()->base64Encode(file_content);
+            practice.getOpenSchemaValidation().addOas(res);
+        }
+    }
+}
+
 Maybe<V1beta2AppsecLinuxPolicy>
 K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     const AppsecSpecParser<NewAppsecPolicySpec> &appsec_policy_spec,
@@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     }
 
     if (default_rule.getMode().empty() && !ingress_mode.empty()) {
+        dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode;
         default_rule.setMode(ingress_mode);
     }
 
@@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
         );
 
     createSnortFile(threat_prevention_practices);
+    createSchemaValidationOas(threat_prevention_practices);
 
     vector<AccessControlPracticeSpec> access_control_practices =
         extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
@@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
                 maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
             dbgWarning(D_LOCAL_POLICY
             ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
+            string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+            string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
             auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
-                "/apis/openappsec.io/v1beta2/policies/" + policy_name
+                "/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name
             );
+
             if (!maybe_v1beta2_appsec_policy_spec.ok()) {
                 dbgWarning(D_LOCAL_POLICY)
                     << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
@@ -535,10 +586,11 @@ K8sPolicyUtils::createPolicy(
     if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) {
         policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
     }
+    auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
     if (item.getSpec().doesDefaultBackendExist()) {
         dbgTrace(D_LOCAL_POLICY)
             << "Inserting Any host rule to the specific asset set";
-        K ingress_rule = K("*");
+        K ingress_rule = K("*", default_mode);
         policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
     }
 
@@ -556,12 +608,11 @@ K8sPolicyUtils::createPolicy(
                     << "' uri: '"
                     << uri.getPath()
                     << "'";
-                K ingress_rule = K(host);
+                K ingress_rule = K(host, default_mode);
                 policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
             }
         }
     }
-
 }
 
 std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>

components/security_apps/local_policy_mgmt_gen/new_practice.cc

@@ -22,6 +22,7 @@ static const set<string> performance_impacts    = {"low", "medium", "high"};
 static const set<string> severity_levels        = {"low", "medium", "high", "critical"};
 static const set<string> size_unit              = {"bytes", "KB", "MB", "GB"};
 static const set<string> confidences_actions    = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
+static const set<string> valied_enforcement_level = {"fullSchema", "endpointOnly"};
 static const set<string> valid_modes = {
     "prevent",
     "detect",
@@ -32,38 +33,38 @@ static const set<string> valid_modes = {
     "inherited"
 };
 static const set<string> valid_confidences      = {"medium", "high", "critical"};
-static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
+static const unordered_map<string, string> key_to_performance_impact_val = {
     { "low", "Low or lower"},
     { "medium", "Medium or lower"},
     { "high", "High or lower"}
 };
-static const std::unordered_map<std::string, std::string> key_to_severity_level_val = {
+static const unordered_map<string, string> key_to_severity_level_val = {
     { "low", "Low or above"},
     { "medium", "Medium or above"},
     { "high", "High or above"},
     { "critical", "Critical"}
 };
-static const std::unordered_map<std::string, std::string> key_to_mode_val = {
+static const unordered_map<string, string> key_to_mode_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Detect"},
     { "prevent", "Prevent"},
     { "detect", "Detect"},
     { "inactive", "Inactive"}
 };
-static const std::unordered_map<std::string, std::string> anti_bot_key_to_mode_val = {
+static const unordered_map<string, string> anti_bot_key_to_mode_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Detect"},
     { "prevent", "Prevent"},
     { "detect", "Detect"},
     { "inactive", "Disabled"}
 };
-static const std::unordered_map<std::string, uint64_t> unit_to_int = {
+static const unordered_map<string, uint64_t> unit_to_int = {
     { "bytes", 1},
     { "KB", 1024},
     { "MB", 1048576},
     { "GB", 1073741824}
 };
-static const std::string TRANSPARENT_MODE = "Transparent";
+static const string TRANSPARENT_MODE = "Transparent";
 
 bool
 isModeInherited(const string &mode)
@@ -71,11 +72,11 @@ isModeInherited(const string &mode)
     return mode == "as-top-level" || mode == "inherited";
 }
 
-const std::string &
+const string &
 getModeWithDefault(
-    const std::string &mode,
+    const string &mode,
-    const std::string &default_mode,
+    const string &default_mode,
-    const std::unordered_map<std::string, std::string> &key_to_val)
+    const unordered_map<string, string> &key_to_val)
 {
     if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
         dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
@@ -88,36 +89,35 @@ getModeWithDefault(
     return key_to_val.at(mode);
 }
 
-const std::vector<std::string> &
+const vector<string> &
 NewAppSecPracticeAntiBot::getIjectedUris() const
 {
     return injected_uris;
 }
 
-const std::vector<std::string> &
+const vector<string> &
 NewAppSecPracticeAntiBot::getValidatedUris() const
 {
     return validated_uris;
 }
 
-const std::string &
+const string &
-NewAppSecPracticeAntiBot::getMode() const
+NewAppSecPracticeAntiBot::getMode(const string &default_mode) const
 {
-    return override_mode;
+    return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val);
 }
 
 void
 NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
-    string mode;
     parseAppsecJSONKey<vector<string>>("injectedUris", injected_uris, archive_in);
     parseAppsecJSONKey<vector<string>>("validatedUris", validated_uris, archive_in);
-    parseMandatoryAppsecJSONKey<string>("overrideMode", mode, archive_in, "inactive");
+    parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
-    if (valid_modes.count(mode) == 0) {
+    if (valid_modes.count(override_mode) == 0) {
-        dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode;
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode);
     }
-    override_mode = anti_bot_key_to_mode_val.at(mode);
 }
 
 void
@@ -242,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const
 }
 
 SnortProtectionsSection::SnortProtectionsSection(
-    const std::string               &_context,
+    const string               &_context,
-    const std::string               &_asset_name,
+    const string               &_asset_name,
-    const std::string               &_asset_id,
+    const string               &_asset_id,
-    const std::string               &_practice_name,
+    const string               &_practice_name,
-    const std::string               &_practice_id,
+    const string               &_practice_id,
-    const std::string               &_source_identifier,
+    const string               &_source_identifier,
-    const std::string               &_mode,
+    const string               &_mode,
-    const std::vector<std::string>  &_files)
+    const vector<string>  &_files)
         :
     context(_context),
     asset_name(_asset_name),
@@ -278,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 DetectionRules::DetectionRules(
-    const std::string                   &_type,
+    const string                   &_type,
-    const std::string                   &_SSM,
+    const string                   &_SSM,
-    const std::string                   &_keywords,
+    const string                   &_keywords,
-    const std::vector<std::string>      &_context)
+    const vector<string>      &_context)
         :
     type(_type),
     SSM(_SSM),
@@ -314,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const
 
 ProtectionMetadata::ProtectionMetadata(
     bool                                _silent,
-    const std::string                   &_protection_name,
+    const string                   &_protection_name,
-    const std::string                   &_severity,
+    const string                   &_severity,
-    const std::string                   &_confidence_level,
+    const string                   &_confidence_level,
-    const std::string                   &_performance_impact,
+    const string                   &_performance_impact,
-    const std::string                   &_last_update,
+    const string                   &_last_update,
-    const std::string                   &_maintrain_id,
+    const string                   &_maintrain_id,
-    const std::vector<std::string>      &_tags,
+    const vector<string>      &_tags,
-    const std::vector<std::string>      &_cve_list)
+    const vector<string>      &_cve_list)
         :
     silent(_silent),
     protection_name(_protection_name),
@@ -394,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 ProtectionsSection::ProtectionsSection(
-    const std::vector<ProtectionsProtectionsSection>    &_protections,
+    const vector<ProtectionsProtectionsSection>    &_protections,
-    const std::string                                   &_name,
+    const string                                   &_name,
-    const std::string                                   &_modification_time)
+    const string                                   &_modification_time)
         :
     protections(_protections),
     name(_name),
@@ -460,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
+NewSnortSignatures::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
     parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
     parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
     parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode);
+    }
     is_temporary = false;
     if (valid_modes.count(override_mode) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
@@ -474,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
+NewSnortSignatures::addFile(const string &file_name)
 {
     files.push_back(file_name);
 }
 
 const string &
-NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
+NewSnortSignatures::getOverrideMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getFiles() const
+NewSnortSignatures::getFiles() const
 {
     return files;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const
+NewSnortSignatures::getConfigMap() const
 {
     return config_map;
 }
 
 bool
-NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const
+NewSnortSignatures::isTemporary() const
 {
     return is_temporary;
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val)
+NewSnortSignatures::setTemporary(bool val)
 {
     is_temporary = val;
 }
 
+void
+NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in)
+{
+    dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice";
+    parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
+    parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
+    parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    parseAppsecJSONKey<string>("enforcementLevel", enforcement_level, archive_in, "fullSchema");
+    if (valied_enforcement_level.count(enforcement_level) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level;
+        throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level);
+    }
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode);
+    }
+    for (const string &file : files)
+    {
+        auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<NewOpenApiSchema>();
+        auto file_content = i_orchestration_tools->readFile(file);
+        if (!file_content.ok()) {
+            dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file";
+            continue;
+        }
+        oas.push_back(Singleton::Consume<I_Encryptor>::by<NewOpenApiSchema>()->base64Encode(file_content.unpack()));
+    }
+}
+
+void
+NewOpenApiSchema::addOas(const string &file)
+{
+    oas.push_back(file);
+}
+
+const string &
+NewOpenApiSchema::getOverrideMode(const string &default_mode) const
+{
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2);
+    return res;
+}
+
+const string &
+NewOpenApiSchema::getEnforceLevel() const
+{
+    return enforcement_level;
+}
+
+const vector<string> &
+NewOpenApiSchema::getFiles() const
+{
+    return files;
+}
+
+const vector<string> &
+NewOpenApiSchema::getConfigMap() const
+{
+    return config_map;
+}
+
+const vector<string> &
+NewOpenApiSchema::getOas() const
+{
+    return oas;
+}
+
 void
 IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -548,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection(
 {
 }
 
-std::string &
+string &
 IpsProtectionsSection::getMode()
 {
     return mode;
@@ -570,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
     );
 }
 
+bool
+IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const
+{
+    // for sorting from the most specific to the least specific rule
+    if (name == default_appsec_name) return false;
+    if (other.name == default_appsec_name) return true;
+    return name.size() > other.name.size();
+}
+
+IPSSection::IPSSection(const vector<IpsProtectionsSection> &_ips) : ips(_ips)
+{
+    sort(ips.begin(), ips.end());
+}
+
 void
 IPSSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -648,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     vector<IpsProtectionsRulesSection> ips_rules;
     IpsProtectionsRulesSection high_rule(
         min_cve_Year,
-        getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(high_confidence_event_action, default_mode),
         string("High"),
         max_performance_impact,
         string(""),
@@ -658,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection med_rule(
         min_cve_Year,
-        getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(medium_confidence_event_action, default_mode),
         string("Medium"),
         max_performance_impact,
         string(""),
@@ -668,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection low_rule(
         min_cve_Year,
-        getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(low_confidence_event_action, default_mode),
         string("Low"),
         max_performance_impact,
         string(""),
@@ -679,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     return ips_rules;
 }
 
-const std::string &
+const string &
-NewIntrusionPrevention::getMode(const std::string &default_mode) const
+NewIntrusionPrevention::getMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
+const string &
+NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const
+{
+    if (isModeInherited(mode)) return default_mode;
+
+    if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) {
+        dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
+        return key_to_practices_mode_val.at("inactive");
+    }
+    return key_to_practices_mode_val.at(mode);
+}
+
 FileSecurityProtectionsSection::FileSecurityProtectionsSection(
     uint64_t                _file_size_limit,
     uint64_t                _archive_file_size_limit,
     bool                    _allow_files_without_name,
     bool                    _required_file_size_limit,
     bool                    _required_archive_extraction,
-    const std::string       &_context,
+    const string       &_context,
-    const std::string       &_name,
+    const string       &_name,
-    const std::string       &_asset_id,
+    const string       &_asset_id,
-    const std::string       &_practice_name,
+    const string       &_practice_name,
-    const std::string       &_practice_id,
+    const string       &_practice_id,
-    const std::string       &_action,
+    const string       &_action,
-    const std::string       &_files_without_name_action,
+    const string       &_files_without_name_action,
-    const std::string       &_high_confidence_action,
+    const string       &_high_confidence_action,
-    const std::string       &_medium_confidence_action,
+    const string       &_medium_confidence_action,
-    const std::string       &_low_confidence_action,
+    const string       &_low_confidence_action,
-    const std::string       &_severity_level,
+    const string       &_severity_level,
-    const std::string       &_file_size_limit_action,
+    const string       &_file_size_limit_action,
-    const std::string       &_multi_level_archive_action,
+    const string       &_multi_level_archive_action,
-    const std::string       &_unopened_archive_action)
+    const string       &_unopened_archive_action)
         :
     file_size_limit(_file_size_limit),
     archive_file_size_limit(_archive_file_size_limit),
@@ -831,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const
     return extract_archive_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const
 {
     return archived_files_within_archived_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const
 {
     return archived_files_where_content_extraction_failed;
@@ -886,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const
     return (file_size_limit * unit_to_int.at(file_size_limit_unit));
 }
 
-const std::string &
+const string &
 NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const
 {
     return files_exceeding_size_limit_action;
@@ -1007,7 +1102,7 @@ void
 NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
-    parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
+    parseAppsecJSONKey<NewOpenApiSchema>(
         "schemaValidation",
         openapi_schema_validation,
         archive_in
@@ -1015,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
     parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
-    parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
+    parseMandatoryAppsecJSONKey<NewSnortSignatures>("snortSignatures", snort_signatures, archive_in);
     parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
     parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
     parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode);
+    }
 }
 
 void
@@ -1028,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name)
     practice_name = _name;
 }
 
-const NewSnortSignaturesAndOpenSchemaAPI &
+NewOpenApiSchema &
-NewAppSecPracticeSpec::getOpenSchemaValidation() const
+NewAppSecPracticeSpec::getOpenSchemaValidation()
 {
     return openapi_schema_validation;
 }
 
-NewSnortSignaturesAndOpenSchemaAPI &
+NewSnortSignatures &
 NewAppSecPracticeSpec::getSnortSignatures()
 {
     return snort_signatures;

components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc

@@ -23,6 +23,14 @@ using namespace std;
 USE_DEBUG_FLAG(D_NGINX_POLICY);
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
+static const std::unordered_map<std::string, std::string> key_to_source_identefier_val = {
+    { "sourceip", "Source IP"},
+    { "cookie", "Cookie:"},
+    { "headerkey", "Header:"},
+    { "JWTKey", ""},
+    { "x-forwarded-for", "X-Forwarded-For"}
+};
+
 void
 SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -1038,7 +1046,7 @@ PolicyMakerUtils::createIpsSections(
         practice_name,
         practice_id,
         source_identifier,
-        override_mode,
+        "Inactive",
         apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
     );
 
@@ -1048,8 +1056,7 @@ PolicyMakerUtils::createIpsSections(
 void
 PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary)
 {
-    auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name;
+    auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name;
-    string in_file = is_temporary ? path + ".rule" : path;
 
     if (snort_protections.find(path) != snort_protections.end()) {
         dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists";
@@ -1060,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         << (is_temporary ? " temporary" : "") << " file " << path;
 
     auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
-    auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err";
+    auto tmp_out = "/tmp/" + file_name + ".out";
+    auto tmp_err = "/tmp/" + file_name + ".err";
+    auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err;
 
     auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
 
@@ -1069,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         return;
     }
 
-    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(path + ".out");
+    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(tmp_out);
     if (!maybe_protections.ok()){
         dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr();
         return;
     }
 
     auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
-    if (is_temporary) i_orchestration_tools->removeFile(in_file);
+    if (is_temporary) i_orchestration_tools->removeFile(path);
-    i_orchestration_tools->removeFile(path + ".out");
+    i_orchestration_tools->removeFile(tmp_out);
-    i_orchestration_tools->removeFile(path + ".err");
+    i_orchestration_tools->removeFile(tmp_err);
 
     snort_protections[path] = ProtectionsSection(
         maybe_protections.unpack().getProtections(),
@@ -1208,7 +1217,8 @@ void
 PolicyMakerUtils::createWebAppSection(
     const V1beta2AppsecLinuxPolicy &policy,
     const RulesConfigRulebase& rule_config,
-    const string &practice_id, const string &full_url,
+    const string &practice_id,
+    const string &full_url,
     const string &default_mode,
     map<AnnotationTypes, string> &rule_annotations)
 {
@@ -1225,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection(
         apssec_practice.getWebAttacks().getMaxObjectDepth(),
         apssec_practice.getWebAttacks().getMaxUrlSizeBytes()
     );
+
     WebAppSection web_app = WebAppSection(
         full_url == "Any" ? default_appsec_url : full_url,
         rule_config.getAssetId(),
@@ -1236,7 +1247,10 @@ PolicyMakerUtils::createWebAppSection(
         rule_config.getContext(),
         apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode),
         apssec_practice.getWebAttacks().getMode(practice_mode),
-        apssec_practice.getAntiBot().getMode(),
+        apssec_practice.getAntiBot().getMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getEnforceLevel(),
+        apssec_practice.getOpenSchemaValidation().getOas(),
         practice_advance_config,
         apssec_practice.getAntiBot(),
         log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
@@ -1290,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
     );
     rules_config[rule_config.getAssetName()] = rule_config;
 
-    string current_identifier;
+    string current_identifier, current_identifier_value;
     if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) {
         UsersIdentifiersRulebase user_identifiers = createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
             rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS],
@@ -1299,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         );
         users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers;
         current_identifier = user_identifiers.getIdentifier();
+        current_identifier_value = user_identifiers.getIdentifierValue();
+    }
+
+    string ips_identifier, ips_identifier_value;
+    if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) {
+        ips_identifier = key_to_source_identefier_val.at(current_identifier);
+    }
+    if (current_identifier == "cookie" || current_identifier == "headerkey") {
+        ips_identifier_value = current_identifier_value;
     }
 
     createIpsSections(
@@ -1306,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         rule_config.getAssetName(),
         practice_id,
         rule_annotations[AnnotationTypes::PRACTICE],
-        current_identifier,
+        ips_identifier + ips_identifier_value,
         rule_config.getContext(),
         policy,
         rule_annotations,

components/security_apps/local_policy_mgmt_gen/rules_config_section.cc

@@ -17,6 +17,8 @@ using namespace std;
 
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
+static const string empty_string="";
+
 AssetUrlParser
 AssetUrlParser::parse(const string &uri)
 {
@@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const
 {
     return source_identifier;
 }
+
+const string &
+UsersIdentifier::getIdentifierValue() const
+{
+    if (identifier_values.empty()) return empty_string;
+    return identifier_values[0];
+}
 // LCOV_EXCL_STOP
 
 void
@@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const
     if (source_identifiers.empty()) return source_identifier;
     return source_identifiers[0].getIdentifier();
 }
+
+const string &
+UsersIdentifiersRulebase::getIdentifierValue() const
+{
+    if (source_identifiers.empty()) return empty_string;
+    return source_identifiers[0].getIdentifierValue();
+}
 // LCOV_EXCL_STOP
 
 void

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -66,7 +66,12 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
     "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
     getQUID)
 SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
-    "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''",
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
     getQUID)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(

components/security_apps/orchestration/env_details/env_details.cc

@@ -28,6 +28,7 @@ EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
     auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
     if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
     token = retrieveToken();
+    agent_namespace = retrieveNamespace();
     if (!token.empty()) {
         auto env_res = getenv("deployment_type");
         env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
@@ -46,12 +47,24 @@ EnvDetails::getToken()
     return token;
 }
 
+string
+EnvDetails::getNameSpace()
+{
+    return agent_namespace;
+}
+
 string
 EnvDetails::retrieveToken()
 {
     return readFileContent(k8s_service_account + "/token");
 }
 
+string
+EnvDetails::retrieveNamespace()
+{
+    return readFileContent(k8s_service_account + "/namespace");
+}
+
 string
 EnvDetails::readFileContent(const string &file_path)
 {

components/security_apps/orchestration/orchestration_comp.cc

@@ -1485,11 +1485,10 @@ private:
     }
 
     void
-    setUpgradeTime()
+    setDelayedUpgradeTime()
     {
         if (getConfigurationFlag("service_startup") != "true") return;
-        if (i_service_controller->getServiceToPortMap().empty()) return;
+        if (!i_agent_details->isOpenAppsecAgent() && i_service_controller->getServiceToPortMap().empty()) return;
-
         try {
             string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN");
             int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30;
@@ -1506,6 +1505,7 @@ private:
     void
     run()
     {
+        loadExistingPolicy();
         sleep_interval = policy.getErrorSleepInterval();
         Maybe<void> registration_status(genError("Not running yet."));
         while (!(registration_status = registerToTheFog()).ok()) {
@@ -1530,7 +1530,6 @@ private:
                 << " seconds";
             Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(seconds(sleep_interval));
         }
-        loadExistingPolicy();
         failure_count = 0;
 
         Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(chrono::seconds(1));
@@ -1587,7 +1586,7 @@ private:
             ).notify();
         }
 
-        setUpgradeTime();
+        setDelayedUpgradeTime();
         while (true) {
             Singleton::Consume<I_Environment>::by<OrchestrationComp>()->startNewTrace(false);
             if (shouldReportAgentDetailsMetadata()) {

components/security_apps/waap/include/i_transaction.h

@@ -70,6 +70,7 @@ public:
     virtual const std::string getParam() const = 0;
     virtual const std::vector<std::string> getKeywordMatches() const = 0;
     virtual const std::vector<std::string> getKeywordsCombinations() const = 0;
+    virtual const std::vector<std::string> getKeywordsAfterFilter() const = 0;
     virtual const std::string getContentTypeStr() const = 0;
     virtual Waap::Util::ContentType getContentType() const = 0;
     virtual const std::string getKeywordMatchesStr() const = 0;
@@ -84,6 +85,7 @@ public:
     virtual const std::string getUriStr() const = 0;
     virtual const std::string& getSourceIdentifier() const = 0;
     virtual double getScore() const = 0;
+    virtual double getOtherModelScore() const = 0;
     virtual const std::vector<double> getScoreArray() const = 0;
     virtual Waap::CSRF::State& getCsrfState() = 0;
     virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;

components/security_apps/waap/include/i_waap_model_result_logger.h

@@ -0,0 +1,37 @@
+// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+class IWaf2Transaction;
+struct Waf2ScanResult;
+namespace Waap {
+namespace Scores {
+struct ModelLoggingSettings;
+}
+}
+
+class I_WaapModelResultLogger {
+public:
+    virtual ~I_WaapModelResultLogger() {}
+
+    virtual void
+    logModelResult(
+        Waap::Scores::ModelLoggingSettings &settings,
+        IWaf2Transaction* transaction,
+        Waf2ScanResult &res,
+        std::string modelName,
+        std::string otherModelName,
+        double newScore,
+        double baseScore) = 0;
+};

components/security_apps/waap/waap_clib/CMakeLists.txt

@@ -88,6 +88,8 @@ add_library(waap_clib
     Waf2Util2.cc
     ParserPDF.cc
     ParserBinaryFile.cc
+    RegexComparator.cc
+    WaapModelResultLogger.cc
 )
 
 add_definitions("-Wno-unused-function")

components/security_apps/waap/waap_clib/CidrMatch.cc

@@ -25,6 +25,51 @@
 #include "log_generator.h"
 #include <stdexcept>
 
+static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) {
+    in_addr maskedAddr;
+    if (prefixLength == 0) {
+        maskedAddr.s_addr = 0;
+    } else {
+        uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1));  // Create mask
+        maskedAddr.s_addr = addr.s_addr & mask;  // Apply mask
+    }
+    return maskedAddr;
+}
+
+// Function to apply a network mask to an IPv6 address
+static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) {
+    in6_addr maskedAddr = addr;
+    int fullBytes = prefixLength / 8;
+    int remainingBits = prefixLength % 8;
+
+    // Mask full bytes
+    for (int i = fullBytes; i < 16; ++i) {
+        maskedAddr.s6_addr[i] = 0;
+    }
+
+    // Mask remaining bits
+    if (remainingBits > 0) {
+        uint8_t mask = ~((1 << (8 - remainingBits)) - 1);
+        maskedAddr.s6_addr[fullBytes] &= mask;
+    }
+
+    return maskedAddr;
+}
+
+// Helper function to convert an IPv4 address to string
+static std::string ipv4ToString(const in_addr& ipv4) {
+    char str[INET_ADDRSTRLEN];
+    inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN);
+    return std::string(str);
+}
+
+// Helper function to convert an IPv6 address to string
+static std::string ipv6ToString(const in6_addr& ipv6) {
+    char str[INET6_ADDRSTRLEN];
+    inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN);
+    return std::string(str);
+}
+
 USE_DEBUG_FLAG(D_WAAP);
 namespace Waap {
 namespace Util {
@@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const {
         isIPV6 == other.isIPV6;
 }
 
+bool CIDRData::operator<(const CIDRData &other) const {
+    if (isIPV6) {
+        if (!other.isIPV6) return false;
+        return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0;
+    }
+    if (other.isIPV6) return true;
+    return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr);
+}
+
 bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) {
     if (bits == 0) {
         // C99 6.5.7 (3): u32 << 32 is undefined behaviour
@@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
     memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr));
 
     if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) {
+        cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits);
         cidr.isIPV6 = false;
     }
     else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) {
+        cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits);
         cidr.isIPV6 = true;
     }
     else
@@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
 
     return true;
 }
+
 bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
     CIDRData cidrData;
 
@@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
 
     return cidrMatch(sourceip, cidrData);
 }
+
 bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
     struct in_addr source_inaddr;
     struct in6_addr source_inaddr6;
@@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
     dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions.";
     return false;
 }
+
+bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) {
+    if (first.isIPV6 != second.isIPV6) return false;  // IPv4 and IPv6 cannot overlap
+    if (first.networkBits >= second.networkBits) return false;
+
+    if (!first.isIPV6) {
+        // IPv4 containment check
+        in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits);
+        return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr);
+    }
+    // IPv6 containment check
+    in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits);
+
+    for (int i = 0; i < 16; ++i) {
+        if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) {
+            return false;
+        }
+    }
+    return true;
+}
+
+std::string cidrsToString(const std::vector<CIDRData>& cidrs) {
+    std::stringstream ss;
+    bool is_first = true;
+    ss << "[";
+    for (const auto& cidr : cidrs) {
+        if (!is_first) ss << ", ";
+        if (cidr.isIPV6) {
+            ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast<int>(cidr.networkBits);
+        } else {
+            ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast<int>(cidr.networkBits);
+        }
+        is_first = false;
+    }
+    ss << "]";
+    return ss.str();
+}
+
 }
 }

components/security_apps/waap/waap_clib/CidrMatch.h

@@ -18,6 +18,7 @@
 #include <sys/socket.h>
 #include <arpa/inet.h>
 #include <errno.h>
+#include <vector>
 
 namespace Waap {
 namespace Util {
@@ -29,11 +30,14 @@ struct CIDRData {
     uint8_t networkBits;
     bool isIPV6;
     bool operator==(const CIDRData &other) const;
+    bool operator<(const CIDRData &other) const;
 };
 
 bool isCIDR(const std::string& strCIDR, CIDRData& cidr);
 bool cidrMatch(const std::string& sourceip, const CIDRData& cidr);
 bool cidrMatch(const std::string &sourceip, const std::string &target);
+bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second);
+std::string cidrsToString(const std::vector<CIDRData>& cidrs);
 
 }
 }

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -1308,7 +1308,7 @@ DeepParser::createInternalParser(
                         *this,
                         parser_depth + 1,
                         '&',
-                        valueStats.isUrlEncoded)
+                        valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
                 );
             } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
                 dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)";
@@ -1323,7 +1323,7 @@ DeepParser::createInternalParser(
                             *this,
                             parser_depth + 1,
                             '&',
-                            valueStats.isUrlEncoded)
+                            valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
                     );
                     offset = 0;
                     return offset;
@@ -1545,5 +1545,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const
     static const std::string PDF_header("%PDF-");
     if (cur_val.size() < 10)
         return false;
-    return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
+    return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size())
+                .find(PDF_header) != std::string::npos;
 }

components/security_apps/waap/waap_clib/ParserBinaryFile.cc

@@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len)
                     }
                 } else {
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
-                    c = strstr(buf + len - tail.size(), tail.c_str());
+                    size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
+                    c = strstr(buf + tail_lookup_offset, tail.c_str());
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
                     if (c) {
                         m_state = s_end;

components/security_apps/waap/waap_clib/ParserBinaryFile.h

@@ -20,7 +20,7 @@
 
 #define MIN_HEADER_LOOKUP 16
 #define MAX_HEADER_LOOKUP 64
-#define MAX_TAIL_LOOKUP 5
+#define MAX_TAIL_LOOKUP 20
 
 class ParserBinaryFile : public ParserBase {
 public:

components/security_apps/waap/waap_clib/ParserPDF.cc

@@ -38,9 +38,10 @@ size_t
 ParserPDF::push(const char *buf, size_t len)
 {
     dbgTrace(D_WAAP_PARSER_PDF)
-        << "buf="
+        << "buf='"
-        << buf
+        << std::string(buf, std::min((size_t)200, len))
-        << "len="
+        << (len > 200 ? "..." : "")
+        << "' len="
         << len;
 
     const char *c;
@@ -65,14 +66,19 @@ ParserPDF::push(const char *buf, size_t len)
                 m_state = s_body;
                 CP_FALL_THROUGH;
             case s_body:
-                c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
+                {
-                dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
+                    size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0;
+                    c = strstr(buf + tail_lookup_offset, PDF_TAIL);
+                    dbgTrace(D_WAAP_PARSER_PDF)
+                        << "string to search: " << std::string(buf + tail_lookup_offset)
+                        << " c=" << c;
                     if (c) {
                         m_state = s_end;
                         CP_FALL_THROUGH;
                     } else {
                         break;
                     }
+                }
             case s_end:
                 if (m_receiver.onKey("PDF", 3) != 0) {
                     m_state = s_error;

components/security_apps/waap/waap_clib/ParserPDF.h

@@ -17,8 +17,8 @@
 #include "ParserBase.h"
 #include <string.h>
 
-#define MAX_HEADER_LOOKUP 64
+#define MAX_PDF_HEADER_LOOKUP 64
-#define MAX_TAIL_LOOKUP 5
+#define MAX_PDF_TAIL_LOOKUP 20
 
 class ParserPDF : public ParserBase {
 public:

components/security_apps/waap/waap_clib/RegexComparator.cc

@@ -0,0 +1,36 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "RegexComparator.h"
+#include <sstream>
+
+namespace Waap {
+namespace Util {
+
+std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet) {
+    std::stringstream ss;
+    ss << "[";
+    bool first = true;
+    for (const auto &regexPtr : regexSet) {
+        if (!first) ss << ", ";
+        if (regexPtr) {
+            first = false;
+            ss << regexPtr->str();
+        }
+    }
+    ss << "]";
+    return ss.str();
+}
+
+}
+}

components/security_apps/waap/waap_clib/RegexComparator.h

@@ -0,0 +1,32 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#include <set>
+#include <boost/regex.hpp>
+
+namespace Waap {
+namespace Util {
+
+// Custom comparator for std::shared_ptr<boost::regex>
+struct RegexComparator {
+    bool operator()(const std::shared_ptr<boost::regex>& lhs, const std::shared_ptr<boost::regex>& rhs) const {
+        // Compare the actual regex patterns by string representation
+        return lhs->str() < rhs->str();
+    }
+};
+
+std::string regexSetToString(const std::set<std::shared_ptr<boost::regex>, RegexComparator> &regexSet);
+
+}
+}

components/security_apps/waap/waap_clib/ScanResult.cc

@@ -23,9 +23,11 @@ unescaped_line(),
 param_name(),
 location(),
 score(0.0f),
+other_model_score(0.0f),
 scoreNoFilter(0.0f),
 scoreArray(),
 keywordCombinations(),
+keywordsAfterFilter(),
 attack_types(),
 m_isAttackInParam(false)
 {
@@ -41,9 +43,11 @@ void Waf2ScanResult::clear()
     param_name.clear();
     location.clear();
     score = 0;
+    other_model_score = 0;
     scoreNoFilter = 0;
     scoreArray.clear();
     keywordCombinations.clear();
+    keywordsAfterFilter.clear();
     attack_types.clear();
 }
 

components/security_apps/waap/waap_clib/ScanResult.h

@@ -29,9 +29,12 @@ struct Waf2ScanResult {
     std::string param_name;
     std::string location;
     double score;
+    double other_model_score;
     double scoreNoFilter;
     std::vector<double> scoreArray;
+    std::vector<double> coefArray;
     std::vector<std::string> keywordCombinations;
+    std::vector<std::string> keywordsAfterFilter;
     std::set<std::string> attack_types;
     bool m_isAttackInParam;
     void clear(); // clear Waf2ScanResult

components/security_apps/waap/waap_clib/ScoreBuilder.cc

@@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
             m_keywordsDataMap[it->first] = it->second;
         }
     }
+
+    m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept;
+    m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef;
+    m_stats.isLinModel = baseScores.m_stats.isLinModel;
 }
 
 
@@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
     m_pWaapAssetState(pWaapAssetState)
 {
     restore();
-
     // merge
     mergeScores(baseScores);
 }
@@ -352,16 +355,42 @@ void ScoreBuilder::snap()
         const std::string &poolName = pool.first;
         const KeywordsScorePool& keywordScorePool = pool.second;
         m_snapshotKwScoreMap[poolName];
+        m_snapshotKwCoefMap[poolName];
+        if (keywordScorePool.m_keywordsDataMap.empty()) {
+            m_snapshotStatsMap[poolName] = KeywordsStats();
+        } else {
+            m_snapshotStatsMap[poolName] = keywordScorePool.m_stats;
+        }
 
         for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
         {
             const std::string &kwName = kwData.first;
-            double kwScore = kwData.second.score;
+            if (keywordScorePool.m_stats.isLinModel) {
-            m_snapshotKwScoreMap[poolName][kwName] = kwScore;
+                m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score;
+            } else {
+                m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score;
+            }
+            m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef;
         }
     }
 }
 
+KeywordsStats
+ScoreBuilder::getSnapshotStats(const std::string &poolName) const
+{
+    std::map<std::string, KeywordsStats>::const_iterator poolIt = m_snapshotStatsMap.find(poolName);
+    if (poolIt == m_snapshotStatsMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool";
+        poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE);
+        if (poolIt == m_snapshotStatsMap.end()) {
+            dbgWarning(D_WAAP_SCORE_BUILDER) <<
+                "base pool does not exist! This is probably a bug. Returning empty stats";
+            return KeywordsStats();
+        }
+    }
+    return poolIt->second;
+}
+
 double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore,
     const std::string &poolName) const
 {
@@ -369,13 +398,12 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
     if (poolIt == m_snapshotKwScoreMap.end()) {
         dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool";
         poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE);
-    }
-
         if (poolIt == m_snapshotKwScoreMap.end()) {
             dbgDebug(D_WAAP_SCORE_BUILDER) <<
                 "base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
             return defaultScore;
         }
+    }
 
     const KeywordScoreMap &kwScoreMap = poolIt->second;
 
@@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
     return kwScoreFound->second;
 }
 
+double
+ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const
+{
+    std::map<std::string, KeywordScoreMap>::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName);
+    if (poolIt == m_snapshotKwCoefMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool";
+        poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE);
+    }
+
+    if (poolIt == m_snapshotKwCoefMap.end()) {
+        dbgWarning(D_WAAP_SCORE_BUILDER) <<
+            "base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef;
+        return defaultCoef;
+    }
+
+    const KeywordScoreMap &kwCoefMap = poolIt->second;
+
+    KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword);
+    if (kwCoefFound == kwCoefMap.end()) {
+        dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef <<
+            " (default, keyword not found in pool '" << poolName << "')";
+        return defaultCoef;
+    }
+
+    dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" <<
+        poolName << "')";
+    return kwCoefFound->second;
+}
+
 keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip)
 {
     return m_fpStore.ipItems[ip];

components/security_apps/waap/waap_clib/ScoreBuilder.h

@@ -26,6 +26,8 @@
 #include <cereal/types/string.hpp>
 #include "WaapDefines.h"
 
+USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER);
+
 struct ScoreBuilderData {
     std::string m_sourceIdentifier;
     std::string m_userAgent;
@@ -52,11 +54,15 @@ enum KeywordType {
 };
 
 struct KeywordData {
-    KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {}
+    KeywordData() :
+        truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN)
+    {}
 
     unsigned int truePositiveCtr;
     unsigned int falsePositiveCtr;
     double score;
+    double coef;
+    double new_score;
     KeywordType type;
 
     template <class Archive>
@@ -65,20 +71,42 @@ struct KeywordData {
             cereal::make_nvp("true_positives", truePositiveCtr),
             cereal::make_nvp("score", score),
             cereal::make_nvp("type", type));
+        try {
+            ar(cereal::make_nvp("coef", coef),
+                cereal::make_nvp("new_score", new_score));
+        } catch (const cereal::Exception &e) {
+            ar.setNextName(nullptr);
+            coef = 0;
+            new_score = 0;
+        }
     }
 };
 
 struct KeywordsStats {
-    KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {}
+    KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0)
+    {}
 
     template <class Archive>
     void serialize(Archive& ar) {
         ar(cereal::make_nvp("false_positives", falsePositiveCtr),
                 cereal::make_nvp("true_positives", truePositiveCtr));
+        try {
+            ar(cereal::make_nvp("intercept", linModelIntercept),
+                cereal::make_nvp("log_nnz_coef", linModelNNZCoef));
+            isLinModel = true;
+        } catch (const cereal::Exception &e) {
+            ar.setNextName(nullptr);
+            linModelIntercept = 0;
+            linModelNNZCoef = 0;
+            isLinModel = false;
+        }
     }
 
     unsigned int truePositiveCtr;
     unsigned int falsePositiveCtr;
+    double linModelIntercept;
+    double linModelNNZCoef;
+    bool isLinModel;
 };
 
 typedef std::unordered_set<std::string> keywords_set;
@@ -106,6 +134,7 @@ struct KeywordsScorePool {
 
     KeywordsScorePool();
 
+    // LCOV_EXCL_START Reason: no test exist
     template <typename _A>
     KeywordsScorePool(_A &iarchive)
     {
@@ -120,6 +149,7 @@ struct KeywordsScorePool {
             m_keywordsDataMap[key] = item.second;
         }
     }
+    // LCOV_EXCL_STOP
 
     template <class Archive>
     void serialize(Archive& ar) {
@@ -148,6 +178,8 @@ public:
 
     void snap();
     double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const;
+    double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const;
+    KeywordsStats getSnapshotStats(const std::string &poolName) const;
 
     keywords_set getIpItemKeywordsSet(std::string ip);
     keywords_set getUaItemKeywordsSet(std::string userAgent);
@@ -200,6 +232,8 @@ protected:
     SerializedData m_serializedData;
     std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
     std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
+    std::map<std::string, KeywordScoreMap> m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap()
+    std::map<std::string, KeywordsStats> m_snapshotStatsMap; // the snapshot is updated only by a call to snap()
     std::list<std::string> m_falsePositivesSetsIntersection;
     I_WaapAssetState* m_pWaapAssetState;
 };

components/security_apps/waap/waap_clib/Serializator.cc

@@ -195,6 +195,9 @@ void SerializeToFileBase::saveData()
     } else {
         ss.str(string((const char *)res.output, res.num_output_bytes));
     }
+    if (res.output) free(res.output);
+    res.output = nullptr;
+    res.num_output_bytes = 0;
 
 
     filestream << ss.str();

components/security_apps/waap/waap_clib/WaapModelResultLogger.cc

@@ -0,0 +1,250 @@
+// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "WaapModelResultLogger.h"
+#include "Waf2Engine.h"
+#include "i_time_get.h"
+#include "i_messaging.h"
+#include "i_instance_awareness.h"
+#include "http_manager.h"
+#include "LogGenWrapper.h"
+#include "rest.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
+
+using namespace std;
+
+static const unsigned int MAX_FILES_PER_WINDOW = 5;
+static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
+static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
+
+class WaapModelReport : public RestGetFile
+{
+public:
+    WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
+
+private:
+    C2S_PARAM(vector<WaapModelResult>, data);
+};
+
+class WaapModelResultLogger::Impl
+        :
+    Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
+{
+public:
+    Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
+        last_sent_s3(std::chrono::minutes::zero()),
+        last_kusto_log_window(std::chrono::minutes::zero()) {}
+    virtual ~Impl();
+    void
+    logModelResult(
+        Waap::Scores::ModelLoggingSettings &settings,
+        IWaf2Transaction* transaction,
+        Waf2ScanResult &res,
+        string modelName,
+        string otherModelName,
+        double score,
+        double otherScore) override;
+
+private:
+    void logToStream(WaapModelResult &result, chrono::minutes now);
+    void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
+    bool shouldSendLogsToS3(chrono::minutes now);
+    void sendLogsToS3();
+    size_t max_logs;
+    unsigned int sent_files_count;
+    unsigned int sent_logs_count;
+    std::chrono::minutes last_sent_s3;
+    std::chrono::minutes last_kusto_log_window;
+    std::map<std::string, vector<WaapModelResult>> logs;
+};
+
+WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
+{
+}
+
+WaapModelResultLogger::~WaapModelResultLogger()
+{
+}
+
+void
+WaapModelResultLogger::logModelResult(
+    Waap::Scores::ModelLoggingSettings &settings,
+    IWaf2Transaction* transaction,
+    Waf2ScanResult &res,
+    std::string modelName,
+    std::string otherModelName,
+    double score,
+    double otherScore
+)
+{
+    pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
+}
+
+void
+WaapModelResultLogger::Impl::logModelResult(
+    Waap::Scores::ModelLoggingSettings &settings,
+    IWaf2Transaction* transaction,
+    Waf2ScanResult &res,
+    string modelName,
+    string otherModelName,
+    double score,
+    double otherScore)
+{
+    if (transaction == NULL) return;
+    if (!Singleton::exists<I_Messaging>()) {
+        dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
+        return;
+    }
+
+    double score_diff = score - otherScore;
+    if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
+        ! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
+        (score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
+        return;
+    }
+
+    auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
+    auto now = chrono::duration_cast<chrono::minutes>(current_time);
+
+    WaapModelResult result = WaapModelResult(
+        *transaction,
+        res,
+        modelName,
+        otherModelName,
+        score,
+        otherScore,
+        now.count()
+        );
+
+    if (settings.logToStream) logToStream(result, now);
+    if (settings.logToS3) logToS3(result, transaction, now);
+}
+
+void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
+{
+    auto asset_state = transaction->getAssetState();
+    string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
+    auto asset_logs = logs.find(asset_id);
+    if (asset_logs == logs.end()) {
+        logs.emplace(asset_id, vector<WaapModelResult>());
+    }
+    logs.at(asset_id).push_back(result);
+    if (shouldSendLogsToS3(now)) {
+        sendLogsToS3();
+    }
+}
+
+void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
+{
+    if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
+        last_kusto_log_window = now;
+        sent_logs_count = 0;
+    }
+    else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
+        return;
+    }
+    sent_logs_count++;
+    dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
+
+    auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
+    LogGenWrapper logGenWrapper(
+        maybeLogTriggerConf,
+        "WAAP Model Telemetry",
+        ReportIS::Audience::SECURITY,
+        LogTriggerConf::SecurityType::ThreatPrevention,
+        ReportIS::Severity::CRITICAL,
+        ReportIS::Priority::HIGH,
+        false);
+
+    LogGen& waap_log = logGenWrapper.getLogGen();
+    waap_log.addMarkerSuffix(result.location);
+    waap_log << LogField("httpuripath", result.uri);
+    waap_log << LogField("matchedlocation", result.location);
+    waap_log << LogField("matchedparameter", result.param);
+    waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
+    waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
+    waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
+    waap_log << LogField("waapfinalscore", (int)(result.score * 100));
+    waap_log << LogField("indicatorssource", result.modelName);
+    waap_log << LogField("indicatorsversion", result.otherModelName);
+}
+
+bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
+{
+    if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
+    for (const auto &asset_logs : logs) {
+        if (asset_logs.second.size() >= max_logs) return true;
+    }
+    return false;
+}
+
+void WaapModelResultLogger::Impl::sendLogsToS3()
+{
+    dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
+
+    I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
+
+    for (auto &asset_logs : logs) {
+        if (asset_logs.second.empty()) {
+            continue;
+        }
+        if (sent_files_count >= MAX_FILES_PER_WINDOW) {
+            dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
+            asset_logs.second.clear();
+            continue;
+        }
+        I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
+        string tenant_id = agentDetails->getTenantId();
+        string agent_id = agentDetails->getAgentId();
+        string asset_id = asset_logs.first;
+        if (Singleton::exists<I_InstanceAwareness>()) {
+            I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
+            Maybe<string> uniqueId = instance->getUniqueID();
+            if (uniqueId.ok())
+            {
+                agent_id += "/" + uniqueId.unpack();
+            }
+        }
+        string uri = "/storage/waap/" +
+                    tenant_id + "/" + asset_id + "/waap_model_results/window_" +
+                    to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
+                    "/" + agent_id + "/data.data";
+        WaapModelReport report = WaapModelReport(asset_logs.second);
+
+        dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
+            ", length " << asset_logs.second.size() <<
+            ", uri " << uri;
+        msg->sendAsyncMessage(
+            HTTPMethod::PUT,
+            uri,
+            report,
+            MessageCategory::LOG
+        );
+
+        asset_logs.second.clear();
+    }
+
+    auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
+    auto now = chrono::duration_cast<chrono::minutes>(current_time);
+    if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
+        last_sent_s3 = now;
+        sent_files_count = 0;
+    } else {
+        sent_files_count++;
+    }
+}
+
+WaapModelResultLogger::Impl::~Impl()
+{}

components/security_apps/waap/waap_clib/WaapModelResultLogger.h

@@ -0,0 +1,109 @@
+// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <map>
+#include <chrono>
+#include <vector>
+#include <ostream>
+#include "cereal/archives/json.hpp"
+
+#include "i_waap_model_result_logger.h"
+#include "DeepAnalyzer.h"
+#include "i_transaction.h"
+#include "ScanResult.h"
+#include "WaapAssetState.h"
+#include "WaapScores.h"
+
+class WaapModelResultLogger
+        :
+    Singleton::Provide<I_WaapModelResultLogger>
+{
+public:
+    WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
+    virtual ~WaapModelResultLogger();
+    virtual void logModelResult(
+        Waap::Scores::ModelLoggingSettings &settings,
+        IWaf2Transaction* transaction,
+        Waf2ScanResult &res,
+        std::string modelName,
+        std::string otherModelName,
+        double score,
+        double otherScore
+    );
+    class Impl;
+
+protected:
+    std::unique_ptr<Impl> pimpl;
+    static const size_t MAX_WAAP_MODEL_LOGS = 20000;
+};
+
+class WaapModelResult
+{
+public:
+    WaapModelResult(
+        IWaf2Transaction &transaction,
+        Waf2ScanResult &res,
+        const std::string &modelName,
+        const std::string &otherModelName,
+        double score,
+        double otherScore,
+        uint64_t time
+    ) : uri(transaction.getUri()), location(res.location), param(res.param_name),
+        modelName(modelName), otherModelName(otherModelName),
+        score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
+        sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
+    {
+    }
+
+    template<class Archive>
+    void serialize(Archive &ar) const
+    {
+        ar(cereal::make_nvp("uri", uri));
+        ar(cereal::make_nvp("location", location));
+        ar(cereal::make_nvp("param", param));
+        ar(cereal::make_nvp("modelName", modelName));
+        ar(cereal::make_nvp("otherModelName", otherModelName));
+        ar(cereal::make_nvp("score", score));
+        ar(cereal::make_nvp("otherScore", otherScore));
+        ar(cereal::make_nvp("keywords", keywords));
+        ar(cereal::make_nvp("sample", sample));
+        ar(cereal::make_nvp("id", id));
+        ar(cereal::make_nvp("time", time));
+    }
+
+    std::string toString() const
+    {
+        std::stringstream message_stream;
+        {
+            cereal::JSONOutputArchive ar(message_stream);
+            serialize(ar);
+        }
+        return message_stream.str();
+    }
+
+    std::string uri;
+    std::string location;
+    std::string param;
+    std::string modelName;
+    std::string otherModelName;
+    double score;
+    double otherScore;
+    std::vector<std::string> keywords;
+    std::string sample;
+    uint64_t id;
+    uint64_t time;
+};

components/security_apps/waap/waap_clib/WaapOverride.cc

@@ -12,7 +12,6 @@
 // limitations under the License.
 
 #include "WaapOverride.h"
-#include "Waf2Util.h"
 
 USE_DEBUG_FLAG(D_WAAP);
 
@@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const
             (m_operand1 == other.m_operand1) &&
             (m_operand2 == other.m_operand2) &&
             (m_tag == other.m_tag) &&
-            Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) &&
+            (m_valuesRegex == other.m_valuesRegex) &&
-            m_cidr == other.m_cidr &&
+            m_ip_addr_values == other.m_ip_addr_values &&
             m_isCidr == other.m_isCidr;
 }
 

components/security_apps/waap/waap_clib/WaapOverride.h

@@ -21,6 +21,7 @@
 #include <memory>
 #include "debug.h"
 #include "CidrMatch.h"
+#include "RegexComparator.h"
 
 USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
 
@@ -52,23 +53,52 @@ public:
                 m_isValid = false;
                 dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
             }
+
+            try {
+                ar(cereal::make_nvp("values", m_values));
+                dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead.";
+            } catch (const cereal::Exception &e) {
                 // The name "value" here is misleading. The real meaning is "regex pattern string"
                 ar(cereal::make_nvp("value", m_value));
+                m_values.insert(m_value);
+            }
 
             if (m_tag == "sourceip" || m_tag == "sourceidentifier") {
-                m_isCidr = Waap::Util::isCIDR(m_value, m_cidr);
+                m_isCidr = true;
+                m_ip_addr_values.resize(m_values.size());
+
+                int val_idx = 0;
+                for (const auto &cur_val : m_values) {
+                    if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) {
+                        dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val;
+                        m_isValid = false;
+                        break;
+                    }
+                    val_idx++;
+                }
+                sortAndMergeCIDRs();
+                dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values);
             }
             m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody");
 
             if (!m_isCidr) {
-                // regex build may throw boost::regex_error
+                for (const auto &cur_val : m_values) {
-                m_valueRegex = nullptr;
                     try {
-                    m_valueRegex = std::make_shared<boost::regex>(m_value);
+                        m_valuesRegex.emplace(std::make_shared<boost::regex>(cur_val));
                     }
                     catch (const boost::regex_error &err) {
-                    dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" <<
+                        dbgDebug(D_WAAP_OVERRIDE)
-                        m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'";
+                            << "Waap::Override::Match(): Failed to compile regex pattern '"
+                            << cur_val
+                            << "' on position "
+                            << err.position()
+                            <<  ". Reason: '"
+                            << err.what()
+                            << "'";
+                        m_isValid = false;
+                        m_valuesRegex.clear();
+                        break;
+                    }
                 }
             }
         }
@@ -95,13 +125,21 @@ public:
     template<typename TestFunctor>
     bool match(TestFunctor testFunctor) const {
         if (m_op == "basic" && m_isCidr) {
-            bool result = testFunctor(m_tag, m_cidr);
+            bool result = testFunctor(m_tag, m_ip_addr_values);
-            dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result;
+            dbgTrace(D_WAAP_OVERRIDE)
+                << "Override matching CIDR list: "
+                << cidrsToString(m_ip_addr_values)
+                << " result: "
+                << result;
             return result;
         }
-        else if (m_op == "basic" && m_valueRegex) {
+        else if (m_op == "basic" && !m_valuesRegex.empty()) {
-            bool result = testFunctor(m_tag, *m_valueRegex);
+            bool result = testFunctor(m_tag, m_valuesRegex);
-            dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result;
+            dbgTrace(D_WAAP_OVERRIDE)
+                << "Override matching regex list: "
+                << regexSetToString(m_valuesRegex)
+                << " result: "
+                << result;
             return result;
         }
         if (m_op == "and") {
@@ -125,22 +163,37 @@ public:
         return false;
     }
 
-    bool isOverrideResponse() const {
+    bool isOverrideResponse() const { return m_isOverrideResponse; }
-        return m_isOverrideResponse;
-    }
 
-    bool isValidMatch() const{
+    bool isValidMatch() const { return m_isValid; }
-        return m_isValid;
-    }
 
 private:
+    void sortAndMergeCIDRs() {
+        if (m_ip_addr_values.empty()) return;
+        std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end());
+
+        size_t mergedIndex = 0;
+        for (size_t i = 1; i < m_ip_addr_values.size(); ++i) {
+            Waap::Util::CIDRData &current = m_ip_addr_values[mergedIndex];
+            Waap::Util::CIDRData &next = m_ip_addr_values[i];
+
+            if (!doesFirstCidrContainSecond(current, next)) {
+                ++mergedIndex;
+                if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next;
+            }
+        }
+
+        m_ip_addr_values.resize(mergedIndex + 1);
+    }
+
     std::string m_op;
     std::shared_ptr<Match> m_operand1;
     std::shared_ptr<Match> m_operand2;
     std::string m_tag;
     std::string m_value;
-    std::shared_ptr<boost::regex> m_valueRegex;
+    std::set<std::string> m_values;
-    Waap::Util::CIDRData m_cidr;
+    std::vector<Waap::Util::CIDRData> m_ip_addr_values;
+    std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> m_valuesRegex;
     bool m_isCidr;
     bool m_isOverrideResponse;
     bool m_isValid;

components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc

@@ -17,100 +17,150 @@
 #include "WaapOverrideFunctor.h"
 #include "Waf2Engine.h"
 #include "CidrMatch.h"
+#include "RegexComparator.h"
 #include "agent_core_utilities.h"
 #include "debug.h"
 
 USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
 
+#define REGX_MATCH(FIELD_FETCH) \
+    NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx)
+#define W2T_REGX_MATCH(FIELD_GETTER) \
+    REGX_MATCH(waf2Transaction.FIELD_GETTER())
+
 WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction)
 {
 }
 
-bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) {
+bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector<Waap::Util::CIDRData> &values) {
+    std::string sourceIp;
     if (tag == "sourceip") {
-        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
+        dbgDebug(D_WAAP_OVERRIDE)
-            value.cidrString;
+            << "Remote IP Address : "
-        std::string sourceIp = waf2Transaction.getRemoteAddr();
+            << waf2Transaction.getRemoteAddr();
-        // match sourceIp against the cidr
+
-        return Waap::Util::cidrMatch(sourceIp, value);
+        sourceIp = waf2Transaction.getRemoteAddr();
+    } else if (tag == "sourceidentifier") {
+        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr();
+        sourceIp = waf2Transaction.getSourceIdentifier();
+    } else {
+        dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag;
+        return false;
+    }
+
+    Waap::Util::CIDRData source_cidr;
+    if (!Waap::Util::isCIDR(sourceIp, source_cidr)) {
+        dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp;
+        return false;
+    }
+
+
+    int left = 0;
+    int right = values.size() - 1;
+
+    while (left <= right) {
+        int mid = left + (right - left) / 2;
+        if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true;
+
+        if (values[mid] < source_cidr) {
+            left = mid + 1;
+        } else {
+            right = mid - 1;
         }
-    else if (tag == "sourceidentifier") {
-        dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
-            value.cidrString;
-        std::string sourceIp = waf2Transaction.getSourceIdentifier();
-        // match source against the cidr
-        return Waap::Util::cidrMatch(sourceIp, value);
     }
 
     return false;
 }
 
-bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
+bool WaapOverrideFunctor::operator()(
+    const std::string &tag,
+    const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes)
 {
     boost::cmatch what;
     std::string tagLower = tag;
     std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
+
     try {
         if (tagLower == "method") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getMethod)) return true;
+            }
+            return false;
         }
         else if (tagLower == "url") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getUriStr)) return true;
+            }
+            return false;
         }
         else if (tagLower == "hostname") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getHost)) return true;
+            }
+            return false;
         }
         else if (tagLower == "sourceidentifier") {
-            return NGEN::Regex::regexMatch(
+            for (const auto &rx : rxes) {
-                __FILE__,
+                if (W2T_REGX_MATCH(getSourceIdentifier)) return true;
-                __LINE__,
+            }
-                waf2Transaction.getSourceIdentifier().c_str(),
+            return false;
-                what,
-                rx
-            );
         }
         else if (tagLower == "keyword") {
+            for (const auto &rx : rxes) {
                 for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
+                    if (REGX_MATCH(keywordStr)) {
                         return true;
                     }
                 }
+            }
             return false;
         }
         else if (tagLower == "paramname") {
+            for (const auto &rx : rxes) {
                 for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
+                    if (REGX_MATCH(keywordInfo.getName())) {
                         return true;
                     }
                 }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) {
+                if (W2T_REGX_MATCH(getParamKey)) return true;
-                return true;
+                if (W2T_REGX_MATCH(getParam)) return true;
-            }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) {
-                return true;
             }
             return false;
         }
         else if (tagLower == "paramvalue") {
+            for (const auto &rx : rxes) {
                 for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
+                    if (REGX_MATCH(keywordInfo.getValue())) {
                         return true;
                     }
                 }
-            if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) {
+                if (W2T_REGX_MATCH(getSample)) return true;
-                return true;
             }
+
             return false;
         }
         else if (tagLower == "paramlocation") {
-            return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
+            for (const auto &rx : rxes) {
+                if (W2T_REGX_MATCH(getLocation)) return true;
+            }
+            return false;
         }
         else if (tagLower == "responsebody") {
             waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
             if (!waf2Transaction.getResponseBody().empty()) {
+                for (const auto &rx : rxes) {
                     boost::smatch matcher;
-                return NGEN::Regex::regexSearch(__FILE__, __LINE__,
+                    if (NGEN::Regex::regexSearch(
-                        waf2Transaction.getResponseBody().c_str(), matcher, rx);
+                        __FILE__,
+                        __LINE__,
+                        waf2Transaction.getResponseBody().c_str(),
+                        matcher,
+                        *rx
+                    )) {
+                        return true;
+                    }
+                }
+                return false;
             } else {
                 return false;
             }
@@ -119,26 +169,30 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
                 dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
                 return false;
             }
+            for (const auto &rx : rxes) {
                 for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
                     std::string value = hdr_pair.first;
                     std::transform(value.begin(), value.end(), value.begin(), ::tolower);
-                if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
+                    if (REGX_MATCH(value)) {
                         return true;
                     }
                 }
+            }
             return false;
         } else if (tagLower == "headervalue") {
             if (!waf2Transaction.checkIsHeaderOverrideScanRequired()) {
                 dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
                 return false;
             }
+            for (const auto &rx : rxes) {
                 for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
                     std::string value = hdr_pair.second;
                     std::transform(value.begin(), value.end(), value.begin(), ::tolower);
-                if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
+                    if (REGX_MATCH(value)) {
                         return true;
                     }
                 }
+            }
             return false;
         }
     }
@@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
         dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();
         return false;
     }
-
     // Unknown tag: should not occur
     dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
     return false;

components/security_apps/waap/waap_clib/WaapOverrideFunctor.h

@@ -16,6 +16,7 @@
 namespace Waap {
     namespace Util {
         struct CIDRData; // forward decleration
+        struct RegexComparator;
     }
 }
 
@@ -24,12 +25,15 @@ class Waf2Transaction;
 // Functor used to match Override rules against request data
 class WaapOverrideFunctor {
 public:
-    WaapOverrideFunctor(Waf2Transaction& waf2Transaction);
+    WaapOverrideFunctor(Waf2Transaction &waf2Transaction);
 
-    bool operator()(const std::string& tag, const Waap::Util::CIDRData& value);
+    bool operator()(const std::string &tag, const std::vector<Waap::Util::CIDRData> &values);
 
-    bool operator()(const std::string& tag, const boost::regex& rx);
+    bool operator()(
+        const std::string &tag,
+        const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes
+    );
 
 private:
-    Waf2Transaction& waf2Transaction;
+    Waf2Transaction &waf2Transaction;
 };

components/security_apps/waap/waap_clib/WaapScanner.cc

@@ -13,7 +13,9 @@
 
 #include "WaapScanner.h"
 #include "WaapScores.h"
+#include "Waf2Engine.h"
 #include "i_transaction.h"
+#include "WaapModelResultLogger.h"
 #include <string>
 #include "debug.h"
 #include "reputation_features_events.h"
@@ -105,20 +107,59 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
     }
 
     std::sort(newKeywords.begin(), newKeywords.end());
-
+    res.keywordsAfterFilter.clear();
+    for (auto keyword : newKeywords) {
+        res.keywordsAfterFilter.push_back(keyword);
+    }
     res.scoreArray.clear();
+    res.coefArray.clear();
     res.keywordCombinations.clear();
 
+    double res_score = getScoreFromPool(res, newKeywords, poolName);
+
+    std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
+    Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
+
+    if (applyLearning && poolName != other_pool_name &&
+        modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
+        double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
+
+        dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
+                                << ", vs. pool " << other_pool_name << ": " << other_score
+                                << ", score difference: " << res_score - other_score
+                                << ", sample: " << res.unescaped_line;
+        Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
+            modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
+        );
+        res.other_model_score = other_score;
+    } else {
+        res.other_model_score = res_score;
+    }
+    return res_score;
+}
+
+double Waap::Scanner::getScoreFromPool(
+    Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
+)
+{
+    KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
+
     if (!newKeywords.empty()) {
         // Collect scores of individual keywords
         Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
-            res.scoreArray);
+            res.scoreArray, res.coefArray);
         // Collect keyword combinations and their scores. Append scores to scoresArray,
         // and also populate m_scanResultKeywordCombinations list
         Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
-            res.scoreArray, res.keywordCombinations);
+            res.scoreArray, res.coefArray, res.keywordCombinations);
     }
 
+    if (stats.isLinModel) {
+        return Waap::Scores::calcLogisticRegressionScore(
+            res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef
+        );
+    }
+    // use base_scores calculation
     return Waap::Scores::calcArrayScore(res.scoreArray);
 }
 

components/security_apps/waap/waap_clib/WaapScanner.h

@@ -44,6 +44,9 @@ namespace Waap {
         static const std::string xmlEntityAttributeId;
     private:
         double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
+        double getScoreFromPool(
+            Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
+        );
         bool shouldIgnoreOverride(const Waf2ScanResult &res);
         bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);
         

components/security_apps/waap/waap_clib/WaapScores.cc

@@ -14,6 +14,8 @@
 #include "WaapScores.h"
 #include <vector>
 #include <string>
+#include <cmath>
+
 #include "ScoreBuilder.h"
 #include "WaapDefines.h"
 #include "debug.h"
@@ -24,11 +26,44 @@ namespace Waap {
 namespace Scores {
 
 std::string getScorePoolNameByLocation(const std::string &location) {
-    std::string poolName = KEYWORDS_SCORE_POOL_BASE;
+    auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.scorePoolName");
-    if (location == "header") {
+    std::string res = KEYWORDS_SCORE_POOL_BASE;
-        poolName = KEYWORDS_SCORE_POOL_HEADERS;
+    if (maybePoolName.ok()) {
+        res = maybePoolName.unpack();
     }
-    return poolName;
+    else if (location == "header") {
+        res = KEYWORDS_SCORE_POOL_HEADERS;
+    }
+    return res;
+}
+
+std::string getOtherScorePoolName()
+{
+    auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.otherScorePoolName");
+    if (maybePoolName.ok()) {
+        return maybePoolName.unpack();
+    }
+    return KEYWORDS_SCORE_POOL_BASE;
+}
+
+ModelLoggingSettings getModelLoggingSettings()
+{
+    ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF,
+                                    .logToS3 = false,
+                                    .logToStream = true};
+    auto maybeLogS3 = getProfileAgentSetting<bool>("agent.waap.modelLogToS3");
+    if (maybeLogS3.ok()) {
+        settings.logToS3 = maybeLogS3.unpack();
+    }
+    auto maybeLogKusto = getProfileAgentSetting<bool>("agent.waap.modelLogToStream");
+    if (maybeLogKusto.ok()) {
+        settings.logToStream = maybeLogKusto.unpack();
+    }
+    auto maybeLogLevel = getProfileAgentSetting<uint>("agent.waap.modelLogLevel");
+    if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) {
+        settings.logLevel = static_cast<ModelLogLevel>(maybeLogLevel.unpack());
+    }
+    return settings;
 }
 
 void
@@ -37,9 +72,16 @@ addKeywordScore(
     const std::string &poolName,
     std::string keyword,
     double defaultScore,
-    std::vector<double>& scoresArray)
+    double defaultCoef,
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray)
 {
-    scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName));
+    double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName);
+    double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName);
+    dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef
+                                    << " keyword: '" << keyword << "' pool: " << poolName;
+    scoresArray.push_back(score);
+    coefArray.push_back(coef);
 }
 
 // Calculate score of individual keywords
@@ -48,13 +90,14 @@ calcIndividualKeywords(
     const ScoreBuilder& scoreBuilder,
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
-    std::vector<double>& scoresArray)
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray)
 {
     std::vector<std::string> keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!)
     std::sort(keywords.begin(), keywords.end());
 
     for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) {
-        addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray);
+        addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray);
     }
 }
 
@@ -65,10 +108,12 @@ calcCombinations(
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
     std::vector<double>& scoresArray,
+    std::vector<double>& coefArray,
     std::vector<std::string>& keyword_combinations)
 {
     keyword_combinations.clear();
     static const double max_combi_score = 1.0f;
+    double default_coef = 0.8f;
 
     for (size_t i = 0; i < keyword_matches.size(); ++i) {
         std::vector<std::string> combinations;
@@ -93,7 +138,7 @@ calcCombinations(
             }
             // set default combination score to be the sum of its keywords, bounded by 1
             default_score = std::min(default_score, max_combi_score);
-            addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
+            addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray);
             keyword_combinations.push_back(combination);
         }
     }
@@ -105,7 +150,6 @@ calcArrayScore(std::vector<double>& scoreArray)
     // Calculate cumulative score from array of individual scores
     double score = 1.0f;
     for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) {
-        dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore;
         double left = 10.0f - score;
         double divisor = (*pScore / 3.0f + 10.0f);  // note: divisor can't be empty because
                                                     // *pScore is always positive and there's a +10 offset
@@ -115,5 +159,20 @@ calcArrayScore(std::vector<double>& scoreArray)
     return score;
 }
 
+double
+calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef)
+{
+    // Sparse logistic regression model, with boolean feature values
+    // Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features
+    // An additional feature was added for the log of the number of non-zero features, as a regularization term
+    double log_odds = intercept + nnzCoef * log(static_cast<double>(coefArray.size()) + 1);
+    for (double &pCoef : coefArray) {
+        log_odds += pCoef;
+    }
+    // Apply the expit function to the log-odds to obtain the probability,
+    // and multiply by 10 to obtain a 'score' in the range [0, 10]
+    return 1.0f / (1.0f + exp(-log_odds)) * 10.0f;
+}
+
 }
 }

components/security_apps/waap/waap_clib/WaapScores.h

@@ -20,7 +20,21 @@
 namespace Waap {
 namespace Scores {
 
+enum class ModelLogLevel {
+    OFF = 0,
+    DIFF = 1,
+    ALL = 2
+};
+
+struct ModelLoggingSettings {
+    ModelLogLevel logLevel;
+    bool logToS3;
+    bool logToStream;
+};
+
 std::string getScorePoolNameByLocation(const std::string &location);
+std::string getOtherScorePoolName();
+ModelLoggingSettings getModelLoggingSettings();
 
 void
 addKeywordScore(
@@ -28,7 +42,9 @@ addKeywordScore(
     const std::string &poolName,
     std::string keyword,
     double defaultScore,
-    std::vector<double>& scoresArray);
+    double defaultCoef,
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray);
 
 // Calculate score of individual keywords
 void
@@ -36,7 +52,8 @@ calcIndividualKeywords(
     const ScoreBuilder& scoreBuilder,
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
-    std::vector<double>& scoresArray);
+    std::vector<double>& scoresArray,
+    std::vector<double>& coefArray);
 
 // Calculate keyword combinations and their scores
 void
@@ -45,9 +62,11 @@ calcCombinations(
     const std::string &poolName,
     const std::vector<std::string>& keyword_matches,
     std::vector<double>& scoresArray,
+    std::vector<double>& coefArray,
     std::vector<std::string>& keyword_combinations);
 
 double calcArrayScore(std::vector<double>& scoreArray);
+double calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef=0.0);
 
 }
 }

components/security_apps/waap/waap_clib/WaapTrigger.h

@@ -125,6 +125,7 @@ struct Log {
 struct Trigger {
     template <typename _A>
     void serialize(_A &ar) {
+        ar(cereal::make_nvp("id", triggerId));
         ar(cereal::make_nvp("$triggerType", triggerType));
         triggerType = to_lower_copy(triggerType);
 
@@ -137,6 +138,7 @@ struct Trigger {
     Trigger();
     bool operator==(const Trigger &other) const;
 
+    std::string triggerId;
     std::string triggerType;
     std::shared_ptr<Log> log;
 };

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
         std::string tag = scanStage + "_param";
         m_deepParser.m_key.push(tag.data(), tag.size());
         size_t buff_len = uriEnd - p;
-        dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
+
-        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
+        bool should_decode = checkUrlEncoded(p, buff_len);
+        if (should_decode) {
+            std::string url_as_string(p, buff_len);
+            should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true);
+       }
+        dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
+
+        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
         up.push(p, buff_len);
         up.finish();
         m_deepParser.m_key.pop(tag.c_str());
@@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) {
     m_ignoreScore = ignoreScore;
 }
 
+bool Waf2Transaction::get_ignoreScore() const
+{
+    auto maybe_override_ignore_score = getProfileAgentSetting<std::string>("agent.waap.alwaysReportScore");
+    if (maybe_override_ignore_score.ok()) {
+        return maybe_override_ignore_score.unpack() == "true";
+    }
+    return m_ignoreScore;
+}
+
 void
 Waf2Transaction::decide(
     bool& bForceBlock,
@@ -1889,6 +1905,14 @@ Waf2Transaction::sendLog()
         return;
     }
 
+    std::set<std::string> triggers_set;
+    for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
+        triggers_set.insert(trigger.triggerId);
+        dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
+    }
+    ScopedContext ctx;
+    ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
+
     auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
     switch (decision_type)
     {

components/security_apps/waap/waap_clib/Waf2Engine.h

@@ -20,6 +20,8 @@
 #include "DeepParser.h"
 #include "WaapAssetState.h"
 #include "PatternMatcher.h"
+#include "generic_rulebase/rulebase_config.h"
+#include "generic_rulebase/evaluators/trigger_eval.h"
 #include "Waf2Util.h"
 #include "WaapConfigApplication.h"
 #include "WaapConfigApi.h"
@@ -39,6 +41,7 @@
 #include "i_waap_telemetry.h"
 #include "i_deepAnalyzer.h"
 #include "i_time_get.h"
+#include "i_waap_model_result_logger.h"
 #include "table_opaque.h"
 #include "WaapResponseInspectReasons.h"
 #include "WaapResponseInjectReasons.h"
@@ -91,6 +94,7 @@ public:
     const std::vector<std::string> getFilteredKeywords() const;
     const std::map<std::string, std::vector<std::string>> getFilteredVerbose() const;
     virtual const std::vector<std::string> getKeywordsCombinations() const;
+    virtual const std::vector<std::string> getKeywordsAfterFilter() const;
     const std::vector<DeepParser::KeywordInfo>& getKeywordInfo() const;
     const std::vector<std::pair<std::string, std::string> >& getKvPairs() const;
     const std::string getKeywordMatchesStr() const;
@@ -99,6 +103,9 @@ public:
     const std::string getLastScanSample() const;
     virtual const std::string& getLastScanParamName() const;
     double getScore() const;
+    // LCOV_EXCL_START Reason: model testing
+    double getOtherModelScore() const;
+    // LCOV_EXCL_STOP
     const std::vector<double> getScoreArray() const;
     Waap::CSRF::State& getCsrfState();
     const std::set<std::string> getFoundPatterns() const;
@@ -161,7 +168,7 @@ public:
 
     // decision functions
     void set_ignoreScore(bool ignoreScore);
-    bool get_ignoreScore() const { return m_ignoreScore; }
+    bool get_ignoreScore() const;
     void decide(
         bool& bForceBlock,
         bool& bForceException,

components/security_apps/waap/waap_clib/Waf2EngineGetters.cc

@@ -189,6 +189,15 @@ const std::vector<std::string> Waf2Transaction::getKeywordsCombinations() const
     }
     return std::vector<std::string>();
 }
+const std::vector<std::string> Waf2Transaction::getKeywordsAfterFilter() const
+{
+    if (m_scanResult)
+    {
+        return m_scanResult->keywordsAfterFilter;
+    }
+    return std::vector<std::string>();
+}
+
 const std::vector<DeepParser::KeywordInfo>& Waf2Transaction::getKeywordInfo() const
 {
     return m_deepParser.m_keywordInfo;
@@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const
     }
     return 0;
 }
+// LCOV_EXCL_START Reason: model testing
+double Waf2Transaction::getOtherModelScore() const
+{
+    if (m_scanResult) {
+        return m_scanResult->other_model_score;
+    }
+    return 0;
+}
+// LCOV_EXCL_STOP
 const std::vector<double> Waf2Transaction::getScoreArray() const
 {
     if (m_scanResult) {
@@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
     waap_log << LogField("waapUriFalsePositiveScore", (int)(
         autonomousSecurityDecision->getFpMitigationScore() * 100));
     waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100));
+    waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100));
     waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100));
     waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel());
 }

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -508,6 +508,8 @@ const char* g_htmlTags[] = {
 };
 
 static const string b64_prefix("base64,");
+// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded
+static const double base64_entropy_threshold = 4.01;
 
 const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]);
 
@@ -966,7 +968,8 @@ base64_decode_status decodeBase64Chunk(
         string::const_iterator it,
         string::const_iterator end,
         string& decoded,
-        bool clear_on_error)
+        bool clear_on_error,
+        bool called_with_prefix)
 {
     decoded.clear();
     uint32_t acc = 0;
@@ -974,6 +977,7 @@ base64_decode_status decodeBase64Chunk(
     int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them.
     uint32_t nonPrintableCharsCount = 0;
     uint32_t spacer_count = 0;
+    uint32_t length = end - it;
 
     dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
     string::const_iterator begin = it;
@@ -986,6 +990,8 @@ base64_decode_status decodeBase64Chunk(
             return B64_DECODE_INVALID;
         }
 
+        std::unordered_map<char, double> frequency;
+
         while (it != end) {
             unsigned char c = *it;
 
@@ -1008,6 +1014,7 @@ base64_decode_status decodeBase64Chunk(
 
                 // allow for more terminator characters
                 it++;
+                frequency[c]++;
                 continue;
             }
 
@@ -1032,6 +1039,7 @@ base64_decode_status decodeBase64Chunk(
                 // Start tracking terminator characters
                 terminatorCharsSeen++;
                 it++;
+                frequency[c]++;
                 continue;
             }
             else {
@@ -1062,6 +1070,7 @@ base64_decode_status decodeBase64Chunk(
             }
 
             it++;
+            frequency[c]++;
         }
 
         // end of encoded sequence decoded.
@@ -1078,6 +1087,27 @@ base64_decode_status decodeBase64Chunk(
             << "; decoded='"
             << decoded << "'";
 
+        // Check if entropy is correlates with b64 threshold (initially > 4.5)
+        if (!called_with_prefix) {
+            double entropy = 0;
+            double p = 0;
+            for (const auto& pair : frequency) {
+                p = pair.second / length;
+                entropy -= p * std::log2(p);
+            }
+            dbgTrace(D_WAAP_BASE64) << " ===b64Test===:  base entropy = " << entropy << "length = " << length;
+            // Add short payload factor
+            if (length < 16)
+                entropy = entropy * 16 / length;
+            // Enforce tailoring '=' characters
+            entropy+=terminatorCharsSeen;
+
+            dbgTrace(D_WAAP_BASE64) << " ===b64Test===:  corrected entropy = " << entropy << "length = " << length;
+            if (entropy <= base64_entropy_threshold) {
+                return B64_DECODE_INVALID;
+            }
+        }
+
         // Return success only if decoded.size>=5 and there are less than 10% of non-printable
         // characters in output.
         if (decoded.size() >= 5) {
@@ -1323,10 +1353,11 @@ processDecodedChunk(
     string::const_iterator start,
     string::const_iterator end,
     string &value,
-    BinaryFileType &binaryFileType
+    BinaryFileType &binaryFileType,
+    bool called_with_prefix = false
 )
 {
-    base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
+    base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix);
     dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
                             << " value.size()=" << value.size();
     if (retVal != B64_DECODE_INVALID && !value.empty()) {
@@ -1349,7 +1380,7 @@ bool isBase64PrefixProcessingOK (
         if (detectBase64Chunk(s, start, end)) {
             dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
             if ((start != s.end()) && (end == s.end())) {
-                retVal = processDecodedChunk(s, start, end, value, binaryFileType);
+                retVal = processDecodedChunk(s, start, end, value, binaryFileType, true);
             }
         } else if (start != s.end()) {
             dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -871,7 +871,8 @@ decodeBase64Chunk(
     std::string::const_iterator it,
     std::string::const_iterator end,
     std::string &decoded,
-    bool clear_on_error = true);
+    bool clear_on_error = true,
+    bool called_with_prefix = false);
 
 bool
 b64DecodeChunk(

components/security_apps/waap/waap_component_impl.cc

@@ -50,7 +50,8 @@ WaapComponent::Impl::Impl() :
     drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
     waapStateTable(NULL),
     transactionsCount(0),
-    deepAnalyzer()
+    deepAnalyzer(),
+    waapModelResultLogger()
 {
 }
 

components/security_apps/waap/waap_component_impl.h

@@ -19,6 +19,7 @@
 #include "table_opaque.h"
 #include "i_transaction.h"
 #include "waap_clib/DeepAnalyzer.h"
+#include "waap_clib/WaapModelResultLogger.h"
 #include "waap_clib/WaapAssetState.h"
 #include "waap_clib/WaapAssetStatesManager.h"
 #include "reputation_features_agg.h"
@@ -80,6 +81,7 @@ private:
     uint64_t transactionsCount;
     // instance of singleton classes
     DeepAnalyzer deepAnalyzer;
+    WaapModelResultLogger waapModelResultLogger;
     WaapAssetStatesManager waapAssetStatesManager;
     std::unordered_set<std::string> m_seen_assets_id;
 };

components/utils/generic_rulebase/match_query.cc

@@ -157,6 +157,9 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
                         dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what();
                     }
                 }
+                if (isKeyTypeIp()) {
+                    sortAndMergeIpRangesValues();
+                }
                 first_value = *(value.begin());
             }
             break;
@@ -301,7 +304,7 @@ MatchQuery::matchAttributes(
     bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn;
     bool match = false;
 
-    if (isIP()) {
+    if (isKeyTypeIp()) {
         match = matchAttributesIp(values);
     } else if (isRegEx()) {
         match = matchAttributesRegEx(values, matched_override_keywords);
@@ -352,10 +355,18 @@ MatchQuery::matchAttributesString(const set<string> &values) const
 bool
 MatchQuery::matchAttributesIp(const set<string> &values) const
 {
-    for (const IPRange &rule_ip_range : ip_addr_value) {
     for (const string &requested_value : values) {
+        int left = 0;
+        int right = ip_addr_value.size() - 1;
         IpAddress ip_addr = IPUtilities::createIpFromString(requested_value);
-            if (IPUtilities::isIpAddrInRange(rule_ip_range, ip_addr)) return true;
+        while (left <= right) {
+            int mid = left + (right - left) / 2;
+            if (IPUtilities::isIpAddrInRange(ip_addr_value[mid], ip_addr)) return true;
+            if (ip_addr_value[mid].start < ip_addr) {
+                left = mid + 1;
+            } else {
+                right = mid - 1;
+            }
         }
     }
     return false;
@@ -367,8 +378,23 @@ MatchQuery::isRegEx() const
     return key != "protectionName";
 }
 
-bool
+void
-MatchQuery::isIP() const
+MatchQuery::sortAndMergeIpRangesValues()
 {
-    return key == "sourceIP" || key == "destinationIP";
+    if (ip_addr_value.empty()) return;
+
+    sort(ip_addr_value.begin(), ip_addr_value.end());
+    size_t mergedIndex = 0;
+    for (size_t i = 1; i < ip_addr_value.size(); ++i) {
+        if (ip_addr_value[i].start <= ip_addr_value[mergedIndex].end) {
+            if (ip_addr_value[mergedIndex].end <= ip_addr_value[i].end) {
+                ip_addr_value[mergedIndex].end = ip_addr_value[i].end;
+            }
+        } else {
+            ++mergedIndex;
+            ip_addr_value[mergedIndex] = ip_addr_value[i];
+        }
+    }
+
+    ip_addr_value.resize(mergedIndex + 1);
 }

components/utils/ip_utilities/ip_utilities.cc

@@ -22,7 +22,10 @@ bool
 operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
 {
     if (this_ip_addr.ip_type < other_ip_addr.ip_type) return true;
-    if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr < other_ip_addr.addr4_t.s_addr;
+    if (this_ip_addr.ip_type > other_ip_addr.ip_type) return false;
+    if (this_ip_addr.ip_type == IP_VERSION_4) {
+        return ntohl(this_ip_addr.addr4_t.s_addr) < ntohl(other_ip_addr.addr4_t.s_addr);
+    }
     return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) < 0;
 }
 
@@ -33,6 +36,19 @@ operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
     if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr == other_ip_addr.addr4_t.s_addr;
     return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) == 0;
 }
+
+bool
+operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr)
+{
+    if (this_ip_addr < other_ip_addr || this_ip_addr == other_ip_addr) return true;
+    return false;
+}
+
+bool
+operator<(const IPRange &range1, const IPRange &range2)
+{
+    return range1.start < range2.start || (range1.start == range2.start && range1.end < range2.end);
+}
 // LCOV_EXCL_STOP
 
 Maybe<pair<string, int>>

components/utils/keywords/keywords_rule.cc

@@ -5,6 +5,8 @@
 
 using namespace std;
 
+USE_DEBUG_FLAG(D_KEYWORD);
+
 static const string whitespaces = " \t";
 
 static string
@@ -14,6 +16,13 @@ getSubStrNoPadding(const string &str, uint start, uint end)
     auto r_end = str.find_last_not_of(whitespaces, end-1);
 
     if (r_end==string::npos || r_start==string::npos || r_start>r_end) {
+        dbgWarning(D_KEYWORD)
+            << "Can't extract substring from '"
+            << str
+            << "', padded start: "
+            << r_start
+            <<  ", padded end: "
+            << r_end;
         throw KeywordError("Found an empty section in the '"+ str + "'");
     }
 
@@ -45,13 +54,24 @@ split(const string &str, const string &delim, uint start = 0)
             }
             default:
                 if (!in_string && delim.find(str[index])!=string::npos) {
+                    if (part_start == index) {
+                        dbgTrace(D_KEYWORD) << "Encountered consecutive delimiter in: " << str;
+                    } else {
                         res.push_back(getSubStrNoPadding(str, part_start, index));
+                    }
                     part_start = index+1;
                 }
         }
     }
 
-    if (escape||in_string) throw KeywordError("Split has ended in the middle of the parsing");
+    if (escape||in_string) {
+        dbgWarning(D_KEYWORD)
+            << "Failed to split "
+            << str
+            << ". Split ended in "
+            << (escape ? "escape" : "middle of string");
+        throw KeywordError("Split has ended in the middle of the parsing");
+    }
 
     if (str.find_first_not_of(whitespaces, part_start)!=string::npos) {
         res.push_back(getSubStrNoPadding(str, part_start, str.size()));

components/utils/keywords/keywords_ut/keywords_ut.cc

@@ -55,6 +55,13 @@ TEST_F(KeywordsRuleTest, data_basic_test) {
     EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;"));
 }
 
+TEST_F(KeywordsRuleTest, consecutive_delimiter_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_TRUE(ruleRun("data: \"234\" , part  HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"75\", part  HTTP_RESPONSE_BODY;"));
+}
+
 TEST_F(KeywordsRuleTest, data_relative_test) {
     appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
 

core/agent_details/agent_details.cc

@@ -304,6 +304,12 @@ AgentDetails::getOrchestrationMode() const
     return orchestration_mode;
 }
 
+bool
+AgentDetails::isOpenAppsecAgent() const
+{
+    return (orchestration_mode == OrchestrationMode::HYBRID) || (tenant_id.rfind("org_", 0) == 0);
+}
+
 string
 AgentDetails::getAccessToken() const
 {

core/config/config.cc

@@ -315,22 +315,13 @@ void
 ConfigComponent::Impl::preload()
 {
     I_Environment *environment = Singleton::Consume<I_Environment>::by<ConfigComponent>();
-    auto executable = environment->get<string>("Executable Name");
+    auto executable = environment->get<string>("Base Executable Name");
     if (!executable.ok() || *executable == "") {
         dbgWarning(D_CONFIG)
             << "Could not load nano service's settings since \"Executable Name\" in not found in the environment";
         return;
     }
-
     executable_name = *executable;
-    auto file_path_end = executable_name.find_last_of("/");
-    if (file_path_end != string::npos) {
-        executable_name = executable_name.substr(file_path_end + 1);
-    }
-    auto file_sufix_start = executable_name.find_first_of(".");
-    if (file_sufix_start != string::npos) {
-        executable_name = executable_name.substr(0, file_sufix_start);
-    }
 
     config_file_paths.insert(executable_name + "-conf.json");
     config_file_paths.insert(executable_name + "-debug-conf.json");

core/debug_is/debug.cc

@@ -766,22 +766,8 @@ Debug::findDebugFilePrefix(const string &file_name)
 string
 Debug::getExecutableName()
 {
-    auto executable = env->get<string>("Executable Name");
+    auto executable = env->get<string>("Base Executable Name");
-    if (!executable.ok() || *executable == "") {
+    return executable.ok() ? *executable : "";
-        return "";
-    }
-
-    string executable_name = *executable;
-    auto file_path_end = executable_name.find_last_of("/");
-    if (file_path_end != string::npos) {
-        executable_name = executable_name.substr(file_path_end + 1);
-    }
-    auto file_sufix_start = executable_name.find_first_of(".");
-    if (file_sufix_start != string::npos) {
-        executable_name = executable_name.substr(0, file_sufix_start);
-    }
-
-    return executable_name;
 }
 
 void

core/debug_is/debug_is_ut/debug_ut.cc

@@ -836,7 +836,7 @@ TEST_F(DebugConfigTest, testSetConfig)
     EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
 
     env.preload();
-    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "debug-ut");
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "debug-ut");
     env.init();
 
     Debug::init();

core/environment/environment.cc

@@ -265,18 +265,9 @@ Environment::Impl::getCurrentHeadersMap()
         tracing_headers["X-Span-Id"] = span_id;
     }
 
-    auto exec_name = get<string>("Executable Name");
+    auto exec_name = get<string>("Base Executable Name");
     if (exec_name.ok() && *exec_name != "") {
-        string executable_name = *exec_name;
+        tracing_headers["X-Calling-Service"] = *exec_name;
-        auto file_path_end = executable_name.find_last_of("/");
-        if (file_path_end != string::npos) {
-            executable_name = executable_name.substr(file_path_end + 1);
-        }
-        auto file_sufix_start = executable_name.find_first_of(".");
-        if (file_sufix_start != string::npos) {
-            executable_name = executable_name.substr(0, file_sufix_start);
-        }
-        tracing_headers["X-Calling-Service"] = executable_name;
     }
 
     return tracing_headers;

core/environment/environment_ut/tracing_ut.cc

@@ -346,7 +346,7 @@ TEST_F(TracingCompRoutinesTest, 2SpansDifFlow)
 {
     I_MainLoop::Routine routine = [&] () {
         string service_name = "test-service-name";
-        i_env->registerValue("Executable Name", service_name);
+        i_env->registerValue("Base Executable Name", service_name);
 
         i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9");
         trace_id = i_env->getCurrentTrace();

core/include/services_sdk/interfaces/i_agent_details.h

@@ -46,6 +46,7 @@ public:
     virtual OrchestrationMode getOrchestrationMode()                  const = 0;
     virtual std::string getAccessToken()                              const = 0;
     virtual void loadAccessToken()                                          = 0;
+    virtual bool isOpenAppsecAgent()                                  const = 0;
 
     // OpenSSL
     virtual void setOpenSSLDir(const std::string &openssl_dir)  = 0;

core/include/services_sdk/interfaces/i_env_details.h

@@ -24,6 +24,7 @@ class I_EnvDetails
 public:
     virtual EnvType getEnvType() = 0;
     virtual std::string getToken() = 0;
+    virtual std::string getNameSpace() = 0;
 
 protected:
     virtual ~I_EnvDetails() {}

core/include/services_sdk/interfaces/messaging/messaging_metadata.h

@@ -215,6 +215,18 @@ public:
         return is_to_fog;
     }
 
+    void
+    setSniHostName(const std::string &_host_name)
+    {
+        sni_host_name = _host_name;
+    }
+
+    Maybe<std::string>
+    getSniHostName() const
+    {
+        return sni_host_name;
+    }
+
     template <class Archive>
     void
     serialize(Archive &ar)
@@ -237,6 +249,7 @@ public:
 
 private:
     std::string host_name = "";
+    Maybe<std::string> sni_host_name = genError("SNI host name not set");
     std::string ca_path = "";
     std::string client_cert_path = "";
     std::string client_key_path = "";

core/include/services_sdk/interfaces/mock/mock_agent_details.h

@@ -28,6 +28,7 @@ public:
     MOCK_CONST_METHOD0(getAgentId, std::string());
     MOCK_METHOD0(loadAccessToken, void());
     MOCK_CONST_METHOD0(getAccessToken, std::string());
+    MOCK_CONST_METHOD0(isOpenAppsecAgent, bool());
 
     // OpenSSL
     MOCK_METHOD1(setOpenSSLDir, void(const std::string&));

core/include/services_sdk/resources/agent_details.h

@@ -73,6 +73,7 @@ public:
     Maybe<std::string> getOpenSSLDir()       const;
     std::string getClusterId()               const;
     OrchestrationMode getOrchestrationMode() const;
+    bool isOpenAppsecAgent()                 const;
     std::string getAccessToken()             const;
     void loadAccessToken();
 

core/include/services_sdk/resources/component_is/components_list_impl.h

@@ -165,6 +165,14 @@ public:
         }
 
         registerGlobalValue<std::string>("Executable Name", arg_vec.front());
+
+        auto file_path_end = arg_vec.front().find_last_of("/");
+        auto executable_name = arg_vec.front().substr(file_path_end + 1);
+        auto file_sufix_start = executable_name.find_first_of(".");
+        if (file_sufix_start != std::string::npos) {
+            executable_name = executable_name.substr(0, file_sufix_start);
+        }
+        registerGlobalValue<std::string>("Base Executable Name", executable_name);
     }
 
     void

core/include/services_sdk/resources/debug_flags.h

@@ -76,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
             DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
             DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP)
+            DEFINE_FLAG(D_WAAP_MODEL_LOGGER, D_WAAP)
             DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP)
             DEFINE_FLAG(D_WAAP_BASE64, D_WAAP)
             DEFINE_FLAG(D_WAAP_JSON, D_WAAP)
@@ -190,6 +191,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
+    DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
 
 DEFINE_FLAG(D_FLOW, D_ALL)
     DEFINE_FLAG(D_DROP, D_FLOW)

core/include/services_sdk/resources/generic_metric.h

@@ -24,6 +24,7 @@
 #include "i_mainloop.h"
 #include "i_time_get.h"
 #include "i_agent_details.h"
+#include "i_instance_awareness.h"
 #include "i_environment.h"
 #include "i_messaging.h"
 #include "i_rest_api.h"
@@ -52,6 +53,7 @@ class GenericMetric
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_TimeGet>,
     Singleton::Consume<I_AgentDetails>,
+    Singleton::Consume<I_InstanceAwareness>,
     Singleton::Consume<I_Environment>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_RestApi>,
@@ -111,7 +113,9 @@ private:
 
     void handleMetricStreamSending();
     void generateLog();
+    void generatePrometheus();
     void generateDebug();
+    void generateAiopsLog();
 
     I_MainLoop *i_mainloop;
     I_TimeGet *i_time;

core/include/services_sdk/resources/log_generator.h

@@ -102,6 +102,8 @@ public:
 
     std::string getLogInsteadOfSending();
 
+    void addMarkerSuffix(const std::string &suffix);
+
 private:
     std::chrono::microseconds getCurrentTime() const;
     void loadBaseLogFields();

core/include/services_sdk/resources/metric/average.h

@@ -54,6 +54,12 @@ public:
         return (count > 0) ? double(sum)/count : 0;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(getAverage());
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/counter.h

@@ -44,6 +44,12 @@ public:
         return counter;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(counter);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/last_reported_value.h

@@ -44,6 +44,12 @@ public:
         return last_reported;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(last_reported);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/max.h

@@ -56,6 +56,12 @@ public:
         return max;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(max);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/metric_calc.h

@@ -18,21 +18,129 @@
 #error metric/metric_calc.h should not be included directly
 #endif // __GENERIC_METRIC_H_
 
+#include <cmath>
 #include <cereal/archives/json.hpp>
 
 #include "report/report.h"
+#include "customized_cereal_map.h"
 
 class GenericMetric;
 
 enum class MetricType { GAUGE, COUNTER };
 
+struct PrometheusData
+{
+    std::string name;
+    std::string type;
+    std::string desc;
+    std::string label;
+    std::string value;
+};
+
+class AiopsMetricData
+{
+public:
+    AiopsMetricData(
+        const std::string &_name,
+        const std::string &_type,
+        const std::string &_units,
+        const std::string &_description,
+        std::map<std::string, std::string> _resource_attributes,
+        float _value)
+            :
+        name(_name),
+        type(_type),
+        units(_units),
+        description(_description),
+        resource_attributes(_resource_attributes),
+        value(_value)
+    {
+        timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
+        asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
+    }
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        ar(cereal::make_nvp("Timestamp", timestamp));
+        ar(cereal::make_nvp("MetricName", name));
+        ar(cereal::make_nvp("MetricType", type));
+        ar(cereal::make_nvp("MetricUnit", units));
+        ar(cereal::make_nvp("MetricDescription", description));
+        ar(cereal::make_nvp("MetricValue", value));
+        ar(cereal::make_nvp("ResourceAttributes", resource_attributes));
+        ar(cereal::make_nvp("MetricAttributes", metric_attributes));
+        ar(cereal::make_nvp("AssetID", asset_id));
+    }
+
+    std::string
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        return ss.str();
+    }
+
+    void
+    addMetricAttribute(const std::string &label, const std::string &value)
+    {
+        metric_attributes[label] = value;
+    }
+
+private:
+    std::string timestamp = "";
+    std::string asset_id = "";
+    std::string name;
+    std::string type;
+    std::string units;
+    std::string description;
+    std::map<std::string, std::string> resource_attributes;
+    std::map<std::string, std::string> metric_attributes;
+    float value = 0;
+};
+
+class AiopsMetricList
+{
+public:
+    void
+    addMetrics(const std::vector<AiopsMetricData> &_metrics)
+    {
+        metrics.insert(metrics.end(), _metrics.begin(), _metrics.end());
+    }
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        ar(cereal::make_nvp("Metrics", metrics));
+    }
+
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+    std::string
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        return ss.str();
+    }
+// LCOV_EXCL_STOP
+
+private:
+    std::vector<AiopsMetricData> metrics;
+};
+
 class MetricCalc
 {
 public:
     template<typename ... Args>
     MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args)
     {
-        setMetadata("BaseName", calc_title);
+        setMetricName(calc_title);
         addMetric(metric);
         parseMetadata(args ...);
     }
@@ -47,7 +155,11 @@ public:
     std::string getMetircDescription() const { return getMetadata("Description"); }
     std::string getMetadata(const std::string &metadata) const;
     virtual MetricType getMetricType() const { return MetricType::GAUGE; }
+    virtual std::vector<PrometheusData> getPrometheusMetrics() const;
+    virtual float getValue() const = 0;
+    virtual std::vector<AiopsMetricData> getAiopsMetrics() const;
 
+    void setMetricName(const std::string &name) { setMetadata("BaseName", name); }
     void setMetricDotName(const std::string &name) { setMetadata("DotName", name); }
     void setMetircUnits(const std::string &units) { setMetadata("Units", units); }
     void setMetircDescription(const std::string &description) { setMetadata("Description", description); }
@@ -55,6 +167,7 @@ public:
 
 protected:
     void addMetric(GenericMetric *metric);
+    std::map<std::string, std::string> getBasicLabels() const;
 
     template <typename Metadata, typename ... OtherMetadata>
     void

core/include/services_sdk/resources/metric/metric_map.h

@@ -54,6 +54,37 @@ class MetricMap : public MetricCalc
             return first->second.getMetricType();
         }
 
+        std::vector<PrometheusData>
+        getPrometheusMetrics(const std::string &label, const std::string &name) const
+        {
+            std::vector<PrometheusData> res;
+
+            for (auto &metric : inner_map) {
+                auto sub_res =  metric.second.getPrometheusMetrics();
+                for (auto &sub_metric : sub_res) {
+                    sub_metric.label += "," + label + "=\"" + metric.first + "\"";
+                    sub_metric.name = name;
+                }
+                res.insert(res.end(), sub_res.begin(), sub_res.end());
+            }
+
+            return res;
+        }
+
+        std::vector<AiopsMetricData>
+        getAiopsMetrics(const std::string &label) const
+        {
+            std::vector<AiopsMetricData> aiops_metrics;
+            for (auto &metric : inner_map) {
+                auto metric_data = metric.second.getAiopsMetrics();
+                for (auto &sub_metric : metric_data) {
+                    sub_metric.addMetricAttribute(label,  metric.first);
+                }
+                aiops_metrics.insert(aiops_metrics.end(), metric_data.begin(), metric_data.end());
+            }
+            return aiops_metrics;
+        }
+
         typename std::map<std::string, Metric>::const_iterator begin() const { return inner_map.begin(); }
         typename std::map<std::string, Metric>::const_iterator end() const { return inner_map.end(); }
 
@@ -63,9 +94,17 @@ class MetricMap : public MetricCalc
 
 public:
     template <typename ... Args>
-    MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args)
+    MetricMap(
+        const Metric &sub_metric,
+        GenericMetric *metric,
+        const std::string &l,
+        const std::string &title,
+        const Args & ... args
+    )
             :
-        MetricCalc(metric, title, args ...)
+        MetricCalc(metric, title, args ...),
+        base_metric(sub_metric),
+        label(l)
     {
     }
 
@@ -75,6 +114,14 @@ public:
         if (getMetricType() == MetricType::GAUGE) metric_map.clear();
     }
 
+// LCOV_EXCL_START Reason: Covered by printPromeathusMultiMap unit-test, but misdetected by the coverage
+    float
+    getValue() const override
+    {
+        return std::nanf("");
+    }
+// LCOV_EXCL_STOP
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {
@@ -89,7 +136,9 @@ public:
     {
         std::stringstream string_key;
         string_key << key;
-        auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first;
+        auto new_metric = base_metric;
+        new_metric.setMetricName(string_key.str());
+        auto metric = metric_map.emplace(string_key.str(), std::move(new_metric)).first;
         metric->second.report(new_values...);
     }
 
@@ -105,8 +154,22 @@ public:
         return field;
     }
 
+    std::vector<PrometheusData>
+    getPrometheusMetrics() const override
+    {
+        return metric_map.getPrometheusMetrics(label, getMetricName());
+    }
+
+    std::vector<AiopsMetricData>
+    getAiopsMetrics() const
+    {
+        return metric_map.getAiopsMetrics(label);
+    }
+
 private:
     InnerMap metric_map;
+    Metric base_metric;
+    std::string label;
 };
 
 } // namespace MetricCalculations

core/include/services_sdk/resources/metric/min.h

@@ -56,6 +56,12 @@ public:
         return min;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(min);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/no_reset_counter.h

@@ -41,6 +41,12 @@ public:
         return counter;
     }
 
+    float
+    getValue() const override
+    {
+        return static_cast<float>(counter);
+    }
+
     void
     save(cereal::JSONOutputArchive &ar) const override
     {

core/include/services_sdk/resources/metric/top_values.h

@@ -59,6 +59,12 @@ public:
         values.clear();
     }
 
+    float
+    getValue() const override
+    {
+        return std::nanf("");
+    }
+
     std::vector<T>
     getTopValues() const
     {

core/intelligence_is_v2/intelligence_comp_v2.cc

@@ -485,6 +485,7 @@ private:
         MessageMetadata req_md(server, *port);
         req_md.insertHeaders(getHTTPHeaders());
         req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+        req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
         return sendIntelligenceRequestImpl(rest_req, req_md);
     }
 

core/logging/log_generator.cc

@@ -123,3 +123,12 @@ LogGen::loadBaseLogFields()
 
     log.getMarkers() = env->getAllStrings(EnvKeyAttr::LogSection::MARKER);
 }
+
+// Adding a suffix to log markers will allow for creating a unique log that won't be reduced
+void
+LogGen::addMarkerSuffix(const string &suffix)
+{
+    for (auto &marker : log.getMarkers()) {
+        marker.second += suffix;
+    }
+}

core/messaging/connection/connection.cc

@@ -96,6 +96,8 @@ public:
             client_key_path = metadata.getClientKeyPath();
             is_dual_auth = true;
         }
+
+        sni_hostname = metadata.getSniHostName();
     }
 
     void
@@ -329,6 +331,12 @@ private:
         if (!SSL_set1_host(ssl_socket, host)) {
             return genError("Failed to set host name verification. Host: " + string(host));
         }
+
+        if (sni_hostname.ok()) {
+            host = sni_hostname->c_str();
+        }
+
+        dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << host;
         if (!SSL_set_tlsext_host_name(ssl_socket, host)) {
             return genError("Failed to set TLS host name extension. Host: " + string(host));
         }
@@ -656,7 +664,7 @@ private:
     printOut(const string &data)
     {
         string type = getConfigurationWithDefault<string>("chopped", "message", "Data printout type");
-        uint length = getConfigurationWithDefault<uint>(10, "message", "Data printout length");
+        uint length = getConfigurationWithDefault<uint>(50, "message", "Data printout length");
         if (type == "full") return data;
         if (type == "size") return to_string(data.size()) + " bytes";
         if (type == "none") return "";
@@ -689,6 +697,7 @@ private:
     bool lock = false;
     bool should_close_connection = false;
     bool is_dual_auth = false;
+    Maybe<string> sni_hostname = genError<string>("Uninitialized");
 };
 
 Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)

core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc

@@ -112,9 +112,8 @@ MessagingBufferComponent::Impl::init()
     
     auto sub_path = getProfileAgentSettingWithDefault<string>("nano_agent/event_buffer/", "eventBuffer.baseFolder");
     buffer_root_path = getLogFilesPathConfig() + "/" + sub_path;
-    string full_executable_name =
+    string executable_name =
-        Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Executable Name").unpack();
+        Singleton::Consume<I_Environment>::by<Messaging>()->get<string>("Base Executable Name").unpack();
-    string executable_name = full_executable_name.substr(full_executable_name.find_last_of("/") + 1);
     removeLegacyBuffer(buffer_root_path, executable_name);
     mkdir(buffer_root_path.c_str(), 0644);
 

core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc

@@ -26,7 +26,7 @@ public:
     TestMessagingBuffer()
     {
         env.preload();
-        Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
+        Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
 
         config.preload();
         config.init();

core/metric/generic_metric.cc

@@ -17,6 +17,7 @@
 #include "debug.h"
 #include "report/log_rest.h"
 #include "config.h"
+#include "report_messaging.h"
 
 #include <fstream>
 #include <sstream>
@@ -30,6 +31,24 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric
 MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; }
 MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }
 
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); }
+// LCOV_EXCL_STOP
+
+vector<AiopsMetricData>
+MetricCalc::getAiopsMetrics() const
+{
+    float value = getValue();
+    if (isnan(value)) return {};
+
+    string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
+    string units = getMetircUnits();
+    string description = getMetircDescription();
+    string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
+    
+    return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
+}
+
 string
 MetricCalc::getMetadata(const string &key) const
 {
@@ -54,6 +73,54 @@ MetricCalc::addMetric(GenericMetric *metric)
     if (metric != nullptr) metric->addCalc(this);
 }
 
+vector<PrometheusData>
+MetricCalc::getPrometheusMetrics() const
+{
+    float value = getValue();
+    if (isnan(value)) return {};
+
+    PrometheusData res;
+
+    res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
+    res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
+    res.desc = getMetircDescription();
+
+    stringstream labels;
+    const auto &label_pairs = getBasicLabels();
+    bool first = true;
+    for (auto &pair : label_pairs) {
+        if (!first) labels << ',';
+        labels << pair.first << "=\"" << pair.second << '"';
+        first = false;
+    }
+    res.label = labels.str();
+
+    stringstream value_str;
+    value_str << value;
+    res.value = value_str.str();
+
+    return {res};
+}
+
+map<string, string>
+MetricCalc::getBasicLabels() const
+{
+    map<string, string> res;
+
+    auto i_instance = Singleton::Consume<I_InstanceAwareness>::by<GenericMetric>();
+    auto id = i_instance->getUniqueID();
+    if (id.ok()) res["id"] = *id;
+
+    auto details = Singleton::Consume<I_AgentDetails>::by<GenericMetric>();
+    res["agent"] = details->getAgentId();
+
+    auto env = Singleton::Consume<I_Environment>::by<GenericMetric>();
+    auto executable = env->get<string>("Base Executable Name");
+    if (executable.ok()) res["process"] = *executable;
+
+    return res;
+}
+
 static const string metric_file = "/tmp/metrics_output.txt";
 
 class GenericMetric::MetricsRest : public ServerRest
@@ -121,7 +188,9 @@ void
 GenericMetric::handleMetricStreamSending()
 {
     if (active_streams.isSet(Stream::DEBUG)) generateDebug();
+    if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus();
     if (active_streams.isSet(Stream::FOG)) generateLog();
+    if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog();
 
     if (reset) resetMetrics();
 }
@@ -244,6 +313,106 @@ GenericMetric::generateLog()
     sendLog(metric_client_rest);
 }
 
+class PrometheusRest : public ClientRest
+{
+    class Metric : public ClientRest
+    {
+    public:
+        Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
+                :
+            name(n),
+            type(t),
+            description(d),
+            labels(l),
+            value(v)
+        {}
+
+    private:
+        C2S_PARAM(string, name);
+        C2S_PARAM(string, type);
+        C2S_PARAM(string, description);
+        C2S_PARAM(string, labels);
+        C2S_PARAM(string, value);
+    };
+
+public:
+    PrometheusRest() : metrics(vector<Metric>()) {}
+
+    void
+    addMetric(const vector<PrometheusData> &vec)
+    {
+        auto &metric_vec = metrics.get();
+        metric_vec.reserve(vec.size());
+        for (auto &metric : vec) {
+            metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value);
+        }
+    }
+
+private:
+    C2S_PARAM(vector<Metric>, metrics);
+};
+
+void
+GenericMetric::generatePrometheus()
+{
+    if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
+
+    vector<PrometheusData> all_metrics;
+    for (auto &calc : calcs) {
+        const auto &cal_metrics = calc->getPrometheusMetrics();
+        all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end());
+    }
+
+    PrometheusRest rest;
+    rest.addMetric(all_metrics);
+
+    MessageMetadata new_config_req_md("127.0.0.1", 7465);
+    new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
+    new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+    Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
+        HTTPMethod::POST,
+        "/set-prometheus-data",
+        rest,
+        MessageCategory::GENERIC,
+        new_config_req_md
+    );
+}
+
+void
+GenericMetric::generateAiopsLog()
+{
+    if (!getConfigurationWithDefault<bool>(true, "metric", "aiopsMetricSendEnable")) return;
+
+    AiopsMetricList aiops_metrics;
+
+    for (auto &calc : calcs) {
+        aiops_metrics.addMetrics(calc->getAiopsMetrics());
+    }
+
+    set<ReportIS::Tags> tags;
+    Report metric_to_fog(
+        "AIOPS Metric Data",
+        Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltime(),
+        Type::PERIODIC,
+        Level::LOG,
+        LogLevel::INFO,
+        audience,
+        team,
+        Severity::INFO,
+        Priority::LOW,
+        report_interval,
+        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
+        tags,
+        Tags::INFORMATIONAL,
+        issuing_engine
+    );
+
+    metric_to_fog << LogField("eventObject", aiops_metrics);
+    
+    LogRest metric_client_rest(metric_to_fog);
+    sendLog(metric_client_rest);
+}
+
 void
 GenericMetric::generateDebug()
 {
@@ -270,6 +439,7 @@ GenericMetric::preload()
 {
     registerExpectedConfiguration<bool>("metric", "fogMetricSendEnable");
     registerExpectedConfiguration<bool>("metric", "debugMetricSendEnable");
+    registerExpectedConfiguration<bool>("metric", "aiopsMetricSendEnable");
     registerExpectedConfiguration<bool>("metric", "fogMetricUri");
     registerExpectedConfiguration<string>("metric", "metricsOutputTmpFile");
 }

core/metric/metric_ut/metric_ut.cc

@@ -10,6 +10,7 @@
 #include "mock/mock_rest_api.h"
 #include "agent_details.h"
 #include "mock/mock_messaging.h"
+#include "mock/mock_instance_awareness.h"
 #include "config.h"
 #include "config_component.h"
 
@@ -67,6 +68,14 @@ public:
         top_usage.report(event.getCPU());
     }
 
+    void
+    setAiopsMetric()
+    {
+        turnOnStream(Stream::AIOPS);
+        max.setMetricDotName("cpu.max");
+        max.setMetircUnits("percrnt");
+    }
+
     Max<double> max{this, "cpuMax"};
     Min<double> min{this, "cpuMin"};
     Average<double> avg{this, "cpuAvg"};
@@ -121,13 +130,15 @@ public:
 class HttpTransaction : public Event<HttpTransaction>
 {
 public:
-    HttpTransaction(const string &_url, uint _bytes) : url(_url), bytes(_bytes) {}
+    HttpTransaction(const string &_url, const string &m, uint _bytes) : url(_url), method(m), bytes(_bytes) {}
 
-    const string & getUrl() const { return url;}
+    const string & getUrl() const { return url; }
+    const string & getMethod() const { return method; }
     uint getBytes() const { return bytes; }
 
 private:
     string url;
+    string method;
     uint bytes;
 };
 
@@ -144,9 +155,33 @@ public:
         total.report(event.getUrl(), 1);
     }
 
+    void
+    setAiopsMetric()
+    {
+        turnOnStream(Stream::AIOPS);
+    }
+
 private:
-    MetricMap<string, Average<double>> avg{this, "PerUrlAvg"};
+    MetricMap<string, Average<double>> avg{Average<double>{nullptr, ""}, this, "url", "PerUrlAvg"};
-    MetricMap<string, NoResetCounter> total{this, "TotalRequests"};
+    MetricMap<string, NoResetCounter> total{NoResetCounter{nullptr, ""}, this, "url", "TotalRequests"};
+};
+
+class UrlMetric2 : public GenericMetric, public Listener<HttpTransaction>
+{
+public:
+    void
+    upon(const HttpTransaction &event) override
+    {
+        total.report(event.getUrl(), event.getMethod(), 1);
+    }
+
+private:
+    MetricMap<string, MetricMap<string, NoResetCounter>> total{
+        MetricMap<string, NoResetCounter>{NoResetCounter{nullptr, ""}, nullptr, "method", ""},
+        this,
+        "url",
+        "request.total"
+    };
 };
 
 class MetricTest : public Test
@@ -155,6 +190,8 @@ public:
     MetricTest()
     {
         EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true));
+        ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87")));
+        ON_CALL(instance, getFamilyID()).WillByDefault(Return(string("")));
         env.init();
         Debug::setNewDefaultStdout(&debug_output);
         Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
@@ -177,6 +214,7 @@ public:
 
     StrictMock<MockMainLoop> mock_ml;
     NiceMock<MockTimeGet> timer;
+    NiceMock<MockInstanceAwareness> instance;
     StrictMock<MockRestApi> rest;
     ::Environment env;
     ConfigComponent conf;
@@ -255,7 +293,9 @@ TEST_F(MetricTest, basicMetricTest)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 89,\n"
@@ -326,7 +366,9 @@ TEST_F(MetricTest, basicMetricTest)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 90,\n"
@@ -399,7 +441,9 @@ TEST_F(MetricTest, basicMetricTest)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 100,\n"
@@ -485,6 +529,150 @@ TEST_F(MetricTest, printMetricsTest)
     GenericMetric::fini();
 }
 
+TEST_F(MetricTest, printPromeathus)
+{
+    conf.preload();
+
+    stringstream configuration;
+    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
+
+    EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
+
+    CPUMetric cpu_mt;
+    cpu_mt.init(
+        "CPU usage",
+        ReportIS::AudienceTeam::AGENT_CORE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        seconds(5),
+        false
+    );
+    cpu_mt.turnOffStream(GenericMetric::Stream::FOG);
+    cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG);
+    cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS);
+    cpu_mt.registerListener();
+
+    CPUEvent cpu_event;
+    cpu_event.setProcessCPU(89);
+    cpu_event.notify();
+
+    string message_body;
+    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
+        .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
+    routine();
+
+    string res =
+        "{\n"
+        "    \"metrics\": [\n"
+        "        {\n"
+        "            \"name\": \"cpuMax\",\n"
+        "            \"type\": \"gauge\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"89\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"cpuMin\",\n"
+        "            \"type\": \"gauge\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"89\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"cpuAvg\",\n"
+        "            \"type\": \"gauge\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"89\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"cpuCurrent\",\n"
+        "            \"type\": \"gauge\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"89\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"cpuCounter\",\n"
+        "            \"type\": \"gauge\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"1\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"cpuTotalCounter\",\n"
+        "            \"type\": \"counter\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
+        "            \"value\": \"1\"\n"
+        "        }\n"
+        "    ]\n"
+        "}";
+
+    EXPECT_EQ(message_body, res);
+}
+
+TEST_F(MetricTest, printPromeathusMultiMap)
+{
+    conf.preload();
+
+    stringstream configuration;
+    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
+
+    EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));
+
+    UrlMetric2 metric;
+    metric.init(
+        "Bytes per URL",
+        ReportIS::AudienceTeam::AGENT_CORE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        seconds(5),
+        true
+    );
+    metric.turnOnStream(GenericMetric::Stream::PROMETHEUS);
+    metric.registerListener();
+
+    HttpTransaction("/index.html", "GET", 10).notify();
+    HttpTransaction("/index2.html", "GET", 20).notify();
+    HttpTransaction("/index.html", "POST", 40).notify();
+
+    string message_body;
+    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
+        .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
+    routine();
+
+    string res =
+        "{\n"
+        "    \"metrics\": [\n"
+        "        {\n"
+        "            \"name\": \"request.total\",\n"
+        "            \"type\": \"counter\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
+                                "method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
+        "            \"value\": \"1\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"request.total\",\n"
+        "            \"type\": \"counter\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
+                                "method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
+        "            \"value\": \"1\"\n"
+        "        },\n"
+        "        {\n"
+        "            \"name\": \"request.total\",\n"
+        "            \"type\": \"counter\",\n"
+        "            \"description\": \"\",\n"
+        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
+                                "method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
+        "            \"value\": \"1\"\n"
+        "        }\n"
+        "    ]\n"
+        "}";
+
+    EXPECT_EQ(message_body, res);
+}
+
 TEST_F(MetricTest, metricTestWithReset)
 {
     CPUMetric cpu_mt;
@@ -554,7 +742,9 @@ TEST_F(MetricTest, metricTestWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 89,\n"
@@ -624,7 +814,9 @@ TEST_F(MetricTest, metricTestWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 90,\n"
@@ -694,7 +886,9 @@ TEST_F(MetricTest, metricTestWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 100,\n"
@@ -796,7 +990,9 @@ TEST_F(MetricTest, generateReportWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 89,\n"
@@ -888,7 +1084,9 @@ TEST_F(MetricTest, generateReportWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"Unnamed Nano Service\"\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 90,\n"
@@ -959,7 +1157,9 @@ TEST_F(MetricTest, generateReportWithReset)
         "            \"eventTraceId\": \"\",\n"
         "            \"eventSpanId\": \"\",\n"
         "            \"issuingEngineVersion\": \"\",\n"
-        "            \"serviceName\": \"My named nano service\"\n"
+        "            \"serviceName\": \"My named nano service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
         "        },\n"
         "        \"eventData\": {\n"
         "            \"cpuMax\": 100,\n"
@@ -1015,6 +1215,7 @@ TEST_F(MetricTest, allMetricTest)
         seconds(5),
         false
     );
+
     msg_size_mt.registerListener();
 
     EXPECT_EQ(msg_size_mt.getMetricName(), "Message size");
@@ -1074,9 +1275,9 @@ TEST_F(MetricTest, testMapMetric)
     );
     url_mt.registerListener();
 
-    HttpTransaction("/index.html", 10).notify();
+    HttpTransaction("/index.html", "GET", 10).notify();
-    HttpTransaction("/index2.html", 20).notify();
+    HttpTransaction("/index2.html", "GET", 20).notify();
-    HttpTransaction("/index.html", 40).notify();
+    HttpTransaction("/index.html", "POST", 40).notify();
 
     string message_body;
 
@@ -1191,3 +1392,309 @@ TEST_F(MetricTest, testManyValuesOutOfOrder)
 
     EXPECT_THAT(AllMetricEvent().query(), ElementsAre(HasSubstr(cpu_str)));
 }
+
+TEST_F(MetricTest, basicAIOPSMetricTest)
+{
+    EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+    CPUMetric cpu_mt;
+    cpu_mt.init(
+        "CPU usage",
+        ReportIS::AudienceTeam::AGENT_CORE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        seconds(5),
+        false
+    );
+    cpu_mt.setAiopsMetric();
+    cpu_mt.registerListener();
+
+    EXPECT_EQ(cpu_mt.getMetricName(), "CPU usage");
+    EXPECT_EQ(cpu_mt.getReportInterval().count(), 5);
+
+    routine();
+
+    CPUEvent cpu_event;
+    cpu_event.setProcessCPU(89);
+    cpu_event.notify();
+
+    string metric_str =
+        "{\n"
+        "    \"Metric\": \"CPU usage\",\n"
+        "    \"Reporting interval\": 5,\n"
+        "    \"cpuMax\": 89.0,\n"
+        "    \"cpuMin\": 89.0,\n"
+        "    \"cpuAvg\": 89.0,\n"
+        "    \"cpuCurrent\": 89.0,\n"
+        "    \"cpuCounter\": 1,\n"
+        "    \"cpuTotalCounter\": 1,\n"
+        "    \"cpuTops\": [\n"
+        "        89.0\n"
+        "    ]\n"
+        "}";
+
+    string message_body;
+    EXPECT_CALL(messaging_mock, sendAsyncMessage(
+        _,
+        "/api/v1/agents/events",
+        _,
+        MessageCategory::METRIC,
+        _,
+        _
+    )).WillRepeatedly(SaveArg<2>(&message_body));
+
+    string expected_message =
+        "{\n"
+        "    \"log\": {\n"
+        "        \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
+        "        \"eventName\": \"AIOPS Metric Data\",\n"
+        "        \"eventSeverity\": \"Info\",\n"
+        "        \"eventPriority\": \"Low\",\n"
+        "        \"eventType\": \"Periodic\",\n"
+        "        \"eventLevel\": \"Log\",\n"
+        "        \"eventLogLevel\": \"info\",\n"
+        "        \"eventAudience\": \"Internal\",\n"
+        "        \"eventAudienceTeam\": \"Agent Core\",\n"
+        "        \"eventFrequency\": 5,\n"
+        "        \"eventTags\": [\n"
+        "            \"Informational\"\n"
+        "        ],\n"
+        "        \"eventSource\": {\n"
+        "            \"agentId\": \"Unknown\",\n"
+        "            \"issuingEngine\": \"Agent Core\",\n"
+        "            \"eventTraceId\": \"\",\n"
+        "            \"eventSpanId\": \"\",\n"
+        "            \"issuingEngineVersion\": \"\",\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
+        "        },\n"
+        "        \"eventData\": {\n"
+        "            \"eventObject\": {\n"
+        "                \"Metrics\": [\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpu.max\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"percrnt\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 89.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpuMin\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 89.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpuAvg\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 89.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpuCurrent\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 89.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpuCounter\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 1.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"cpuTotalCounter\",\n"
+        "                        \"MetricType\": \"counter\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 1.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {},\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    }\n"
+        "                ]\n"
+        "            }\n"
+        "        }\n"
+        "    }\n"
+        "}";
+
+    routine();
+    EXPECT_THAT(debug_output.str(), HasSubstr(metric_str));
+    EXPECT_EQ(message_body, expected_message);
+    debug_output.str("");
+}
+
+TEST_F(MetricTest, testAIOPSMapMetric)
+{
+    EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+    UrlMetric url_mt;
+    url_mt.init(
+        "Bytes per URL",
+        ReportIS::AudienceTeam::AGENT_CORE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        seconds(5),
+        true
+    );
+    url_mt.registerListener();
+
+    url_mt.setAiopsMetric();
+
+    HttpTransaction("/index.html", "GET", 10).notify();
+    HttpTransaction("/index2.html", "GET", 20).notify();
+    HttpTransaction("/index.html", "POST", 40).notify();
+
+    string message_body;
+
+    EXPECT_CALL(messaging_mock, sendAsyncMessage(
+        _,
+        "/api/v1/agents/events",
+        _,
+        MessageCategory::METRIC,
+        _,
+        _
+    )).WillRepeatedly(SaveArg<2>(&message_body));
+    routine();
+
+    string expected_message =
+        "{\n"
+        "    \"log\": {\n"
+        "        \"eventTime\": \"2016-11-13T17:31:24.087\",\n"
+        "        \"eventName\": \"AIOPS Metric Data\",\n"
+        "        \"eventSeverity\": \"Info\",\n"
+        "        \"eventPriority\": \"Low\",\n"
+        "        \"eventType\": \"Periodic\",\n"
+        "        \"eventLevel\": \"Log\",\n"
+        "        \"eventLogLevel\": \"info\",\n"
+        "        \"eventAudience\": \"Internal\",\n"
+        "        \"eventAudienceTeam\": \"Agent Core\",\n"
+        "        \"eventFrequency\": 5,\n"
+        "        \"eventTags\": [\n"
+        "            \"Informational\"\n"
+        "        ],\n"
+        "        \"eventSource\": {\n"
+        "            \"agentId\": \"Unknown\",\n"
+        "            \"issuingEngine\": \"Agent Core\",\n"
+        "            \"eventTraceId\": \"\",\n"
+        "            \"eventSpanId\": \"\",\n"
+        "            \"issuingEngineVersion\": \"\",\n"
+        "            \"serviceName\": \"Unnamed Nano Service\",\n"
+        "            \"serviceId\": \"87\",\n"
+        "            \"serviceFamilyId\": \"\"\n"
+        "        },\n"
+        "        \"eventData\": {\n"
+        "            \"eventObject\": {\n"
+        "                \"Metrics\": [\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"/index.html\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 25.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {\n"
+        "                            \"url\": \"/index.html\"\n"
+        "                        },\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"/index2.html\",\n"
+        "                        \"MetricType\": \"gauge\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 20.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {\n"
+        "                            \"url\": \"/index2.html\"\n"
+        "                        },\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"/index.html\",\n"
+        "                        \"MetricType\": \"counter\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 2.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {\n"
+        "                            \"url\": \"/index.html\"\n"
+        "                        },\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
+        "                        \"MetricName\": \"/index2.html\",\n"
+        "                        \"MetricType\": \"counter\",\n"
+        "                        \"MetricUnit\": \"\",\n"
+        "                        \"MetricDescription\": \"\",\n"
+        "                        \"MetricValue\": 1.0,\n"
+        "                        \"ResourceAttributes\": {\n"
+        "                            \"agent\": \"Unknown\",\n"
+        "                            \"id\": \"87\"\n"
+        "                        },\n"
+        "                        \"MetricAttributes\": {\n"
+        "                            \"url\": \"/index2.html\"\n"
+        "                        },\n"
+        "                        \"AssetID\": \"Unknown\"\n"
+        "                    }\n"
+        "                ]\n"
+        "            }\n"
+        "        }\n"
+        "    }\n"
+        "}";
+
+    EXPECT_EQ(message_body, expected_message);
+}

core/rest/rest_conn.cc

@@ -23,11 +23,12 @@ using namespace std;
 
 USE_DEBUG_FLAG(D_API);
 
-RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke)
+RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external)
         :
         fd(_fd),
         mainloop(_mainloop),
-    invoke(_invoke)
+        invoke(_invoke),
+        is_external_ip(is_external)
 {}
 
 RestConn::~RestConn()
@@ -101,6 +102,12 @@ RestConn::parseConn() const
         return sendResponse("200 OK", invoke->invokeGet(identifier), false);
     }
 
+    if (is_external_ip) {
+        dbgWarning(D_API) << "External IP tried to POST";
+        sendResponse("500 Internal Server Error", "", false);
+        stop();
+    }
+
     stringstream body;
     body.str(readSize(len));
 

core/rest/rest_conn.h

@@ -21,7 +21,7 @@
 class RestConn
 {
 public:
-    RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke);
+    RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external = false);
     ~RestConn();
 
     void parseConn() const;
@@ -35,6 +35,7 @@ private:
     int fd;
     I_MainLoop *mainloop;
     const I_RestInvoke *invoke;
+    bool is_external_ip = false;
 };
 
 #endif // __REST_CONN_H__

core/rest/rest_server.cc

@@ -47,6 +47,7 @@ public:
     void startNewConnection() const;
 
     bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
+    bool bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range);
     bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
     bool addGetCall(const string &uri, const function<string()> &cb) override;
     uint16_t getListeningPort() const override { return listening_port; }
@@ -73,10 +74,36 @@ private:
 bool
 RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range)
 {
+    dbgFlow(D_API) << "Binding IPv4 socket";
     for (uint16_t port : port_range) {
         addr.sin_port = htons(port);
 
         if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true;
+
+        if (errno == EADDRINUSE) {
+            dbgDebug(D_API) << "Port " << port << " is already in use";
+        } else {
+            dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+        }
+    }
+
+    return false;
+}
+
+bool
+RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range)
+{
+    dbgFlow(D_API) << "Binding IPv6 socket";
+    for (uint16_t port : port_range) {
+        addr.sin6_port = htons(port);
+
+        if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true;
+
+        if (errno == EADDRINUSE) {
+            dbgDebug(D_API) << "Port " << port << " is already in use";
+        } else {
+            dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+        }
     }
 
     return false;
@@ -119,22 +146,56 @@ RestServer::Impl::init()
     mainloop = Singleton::Consume<I_MainLoop>::by<RestServer>();
 
     auto init_connection = [this] () {
+        auto allow_external_conn = "Nano service API Allow Get From External IP";
+        auto conf_value = getConfiguration<bool>("connection", allow_external_conn);
+        bool accept_get_from_external_ip = false;
+        if (conf_value.ok()) {
+            accept_get_from_external_ip = *conf_value;
+        } else {
+            auto env_value = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>(allow_external_conn);
+            if (env_value.ok()) {
+                accept_get_from_external_ip = *env_value;
+            }
+        }
+
+        if (accept_get_from_external_ip) {
+            fd = socket(AF_INET6, SOCK_STREAM, 0);
+        } else {
             fd = socket(AF_INET, SOCK_STREAM, 0);
+        }
         dbgAssert(fd >= 0) << alert << "Failed to open a socket";
+
         int socket_enable = 1;
         if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
             dbgWarning(D_API) << "Could not set the socket options";
         }
 
+        if (accept_get_from_external_ip) {
+            int option = 0;
+            if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) {
+                dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option";
+            }
+
+            struct sockaddr_in6 addr6;
+            bzero(&addr6, sizeof(addr6));
+            addr6.sin6_family = AF_INET6;
+            addr6.sin6_addr = in6addr_any;
+
+            while (!bindRestServerSocket(addr6, port_range)) {
+                mainloop->yield(bind_retry_interval_msec);
+            }
+            listening_port = ntohs(addr6.sin6_port);
+        } else {
             struct sockaddr_in addr;
             bzero(&addr, sizeof(addr));
-
             addr.sin_family = AF_INET;
             addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 
             while (!bindRestServerSocket(addr, port_range)) {
                 mainloop->yield(bind_retry_interval_msec);
             }
+            listening_port = ntohs(addr.sin_port);
+        }
 
         listen(fd, listen_limit);
 
@@ -146,9 +207,12 @@ RestServer::Impl::init()
             "REST server listener",
             is_primary.ok() && *is_primary
         );
-
+        dbgInfo(D_API)
-        listening_port = ntohs(addr.sin_port);
+            << "REST server started: "
-        dbgInfo(D_API) << "REST server started: " << listening_port;
+            << listening_port
+            << ". Accepting: "
+            << (accept_get_from_external_ip ? "external" : "loopback")
+            << " connections";
         Singleton::Consume<I_Environment>::by<RestServer>()->registerValue<int>("Listening Port", listening_port);
     };
 
@@ -172,14 +236,30 @@ void
 RestServer::Impl::startNewConnection() const
 {
     dbgFlow(D_API) << "Starting a new connection";
-    int new_socket = accept(fd, nullptr, nullptr);
+    struct sockaddr_storage addr;
+    socklen_t addr_len = sizeof(addr);
+    int new_socket = accept(fd, (struct sockaddr *)&addr, &addr_len);
     if (new_socket < 0) {
-        dbgWarning(D_API) << "Failed to accept a new socket";
+        dbgWarning(D_API) << "Failed to accept a new socket: " << strerror(errno);
         return;
     }
-    dbgDebug(D_API) << "Starting a new socket: " << new_socket;
 
-    RestConn conn(new_socket, mainloop, this);
+    dbgDebug(D_API) << "Starting a new socket: " << new_socket;
+    bool is_external = false;
+    if (addr.ss_family == AF_INET6) {
+        struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *) &addr;
+        if (IN6_IS_ADDR_V4MAPPED(&addr_in6->sin6_addr)) {
+            struct in_addr ipv4_addr;
+            memcpy(&ipv4_addr, &addr_in6->sin6_addr.s6_addr[12], sizeof(ipv4_addr));
+            is_external = ipv4_addr.s_addr != htonl(INADDR_LOOPBACK);
+        } else {
+            is_external = memcmp(&addr_in6->sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)) != 0;
+        }
+    } else {
+        struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr;
+        is_external = addr_in->sin_addr.s_addr != htonl(INADDR_LOOPBACK);
+    }
+    RestConn conn(new_socket, mainloop, this, is_external);
     mainloop->addFileRoutine(
         I_MainLoop::RoutineType::Offline,
         new_socket,
@@ -283,4 +363,5 @@ RestServer::preload()
     registerExpectedConfiguration<uint>("connection", "Nano service API Port Alternative");
     registerExpectedConfiguration<uint>("connection", "Nano service API Port Range start");
     registerExpectedConfiguration<uint>("connection", "Nano service API Port Range end");
+    registerExpectedConfiguration<bool>("connection", "Nano service API Allow Get From External IP");
 }

core/rest/rest_ut/rest_config_ut.cc

@@ -14,24 +14,34 @@
 #include "agent_details.h"
 #include "mock/mock_messaging.h"
 #include "tenant_manager.h"
+#include <netdb.h>
+#include <arpa/inet.h>
 
 using namespace std;
 using namespace testing;
 
-USE_DEBUG_FLAG(D_API);
+static const string config_json_allow_external =
-USE_DEBUG_FLAG(D_MAINLOOP);
+        "{\n"
+        "    \"connection\": {\n"
+        "        \"Nano service API Port Primary\": [\n"
+        "            {\n"
+        "                \"value\": 9777\n"
+        "            }\n"
+        "        ],\n"
+        "        \"Nano service API Port Alternative\": [\n"
+        "            {\n"
+        "                \"value\": 9778\n"
+        "            }\n"
+        "        ],\n"
+        "        \"Nano service API Allow Get From External IP\": [\n"
+        "            {\n"
+        "                \"value\": true\n"
+        "            }\n"
+        "        ]\n"
+        "    }\n"
+        "}\n";
 
-class RestConfigTest : public Test
+static const string config_json =
-{
-public:
-    RestConfigTest()
-    {
-        rest_server.preload();
-
-        time_proxy.init();
-        mainloop_comp.init();
-
-        string config_json =
         "{\n"
         "    \"connection\": {\n"
         "        \"Nano service API Port Primary\": [\n"
@@ -47,6 +57,19 @@ public:
         "    }\n"
         "}\n";
 
+USE_DEBUG_FLAG(D_API);
+USE_DEBUG_FLAG(D_MAINLOOP);
+
+class RestConfigTest : public Test
+{
+public:
+    RestConfigTest()
+    {
+        rest_server.preload();
+
+        time_proxy.init();
+        mainloop_comp.init();
+
         istringstream ss(config_json);
         Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
 
@@ -58,6 +81,9 @@ public:
     ~RestConfigTest()
     {
         Debug::setNewDefaultStdout(&cout);
+        auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
+        mainloop->stopAll();
+        rest_server.fini();
         time_proxy.fini();
         mainloop_comp.fini();
     }
@@ -133,7 +159,7 @@ int TestServer::g_num = 0;
 TEST_F(RestConfigTest, basic_flow)
 {
     env.preload();
-    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
 
     config.preload();
     config.init();
@@ -165,11 +191,15 @@ TEST_F(RestConfigTest, basic_flow)
 
     auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
     I_MainLoop::Routine stop_routine = [&] () {
-        EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
+        EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
+            << "file_descriptor1 Error: "
+            << strerror(errno);
         string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
         EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
 
-        EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0);
+        EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
+            << "file_descriptor2 Error: "
+            << strerror(errno);
         string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
         EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
 
@@ -204,3 +234,159 @@ TEST_F(RestConfigTest, basic_flow)
         "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
     );
 }
+
+string
+getLocalIPAddress() {
+    char hostname[1024];
+    hostname[1024 - 1] = '\0';
+
+    // Get the hostname
+    if (gethostname(hostname, sizeof(hostname)) == -1) {
+        return "";
+    }
+
+    struct addrinfo hints, *info, *p;
+    int gai_result;
+
+    memset(&hints, 0, sizeof(hints));
+    hints.ai_family = AF_INET; // Use AF_INET for IPv4
+    hints.ai_socktype = SOCK_STREAM;
+
+    // Get the address info
+    if ((gai_result = getaddrinfo(hostname, nullptr, &hints, &info)) != 0) {
+        return "";
+    }
+
+    std::string ip_address;
+    for (p = info; p != nullptr; p = p->ai_next) {
+        void *addr;
+        char ipstr[INET_ADDRSTRLEN];
+
+        struct sockaddr_in *ipv4 = (struct sockaddr_in *)p->ai_addr;
+        addr = &(ipv4->sin_addr);
+
+        // Convert the IP to a string and print it
+        inet_ntop(p->ai_family, addr, ipstr, sizeof(ipstr));
+        if (std::string(ipstr) != "127.0.0.1") {
+            ip_address = ipstr;
+            break;
+        }
+    }
+
+    freeaddrinfo(info); // Free the linked list
+
+    return ip_address;
+}
+
+
+TEST_F(RestConfigTest, not_loopback_flow)
+{
+    env.preload();
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
+
+
+    istringstream ss(config_json_allow_external);
+    Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
+
+    config.preload();
+    config.init();
+
+    rest_server.init();
+    time_proxy.init();
+    mainloop_comp.init();
+
+    auto i_rest = Singleton::Consume<I_RestApi>::from(rest_server);
+    ASSERT_TRUE(i_rest->addRestCall<TestServer>(RestAction::ADD, "test"));
+    ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); }));
+
+    int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0);
+    EXPECT_NE(file_descriptor1, -1);
+    int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0);
+    EXPECT_NE(file_descriptor2, -1);
+
+    auto primary_port = getConfiguration<uint>("connection", "Nano service API Port Primary");
+    auto second_port = getConfiguration<uint>("connection", "Nano service API Port Alternative");
+    auto local_ip = getLocalIPAddress();
+    struct sockaddr_in sa_primary;
+    sa_primary.sin_family = AF_INET;
+    sa_primary.sin_port = htons(primary_port.unpack());
+    sa_primary.sin_addr.s_addr = inet_addr(local_ip.c_str());
+    struct sockaddr_in sa_second;
+    sa_second.sin_family = AF_INET;
+    sa_second.sin_port = htons(second_port.unpack());
+    sa_second.sin_addr.s_addr = inet_addr(local_ip.c_str());
+
+    int socket_enable = 1;
+    EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
+    EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
+
+    EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _))
+            .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
+    Debug::setNewDefaultStdout(&cout);
+    auto mainloop = Singleton::Consume<I_MainLoop>::from(mainloop_comp);
+    Debug::setNewDefaultStdout(&cout);
+    I_MainLoop::Routine stop_routine = [&] () {
+        int socket_client_2 = -1;
+        auto socket_client_1 = connect(file_descriptor1, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr));
+        dbgDebug(D_API) <<  "socket_client_1: " << socket_client_1;
+        if (socket_client_1 == -1) {
+            dbgDebug(D_API) << "Error: " << strerror(errno);
+            socket_client_2 = connect(file_descriptor1, (struct sockaddr*)&sa_second, sizeof(struct sockaddr));
+            dbgDebug(D_API) << "socket_client_2: " << socket_client_2;
+            if (socket_client_2 == -1) {
+                dbgDebug(D_API) << "Error: " << strerror(errno) << endl;
+            } else {
+                EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_second, sizeof(struct sockaddr)), 0);
+                string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
+                EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
+            }
+        } else {
+            EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr)), 0);
+            string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
+            EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
+        }
+        EXPECT_TRUE(socket_client_1 != -1 || socket_client_2 != -1);
+        string msg1 = "GET /stuff HTTP/1.1\r\n\r\n";
+        EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast<int>(msg1.size()));
+
+        mainloop->yield(true);
+
+        struct pollfd s_poll;
+        s_poll.fd = file_descriptor1;
+        s_poll.events = POLLIN;
+        s_poll.revents = 0;
+        while(poll(&s_poll, 1, 0) <= 0) {
+            mainloop->yield(true);
+        }
+
+        struct pollfd s_poll2;
+        s_poll2.fd = file_descriptor2;
+        s_poll2.events = POLLIN;
+        s_poll2.revents = 0;
+        while(poll(&s_poll2, 1, 0) <= 0) {
+            mainloop->yield(true);
+        }
+
+        mainloop->stopAll();
+    };
+    mainloop->addOneTimeRoutine(
+            I_MainLoop::RoutineType::RealTime,
+            stop_routine,
+            "RestConfigTest-alternative_port_used stop routine",
+            true
+    );
+    mainloop->run();
+
+    char respose[1000];
+    EXPECT_EQ(read(file_descriptor1, respose, 1000), 76);
+    EXPECT_EQ(
+            string(respose, 76),
+            "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
+    );
+
+    EXPECT_EQ(read(file_descriptor2, respose, 1000), 89);
+    EXPECT_EQ(
+            string(respose, 89),
+            "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n"
+    );
+}

core/rest/rest_ut/rest_schema_ut.cc

@@ -437,7 +437,7 @@ TEST(RestSchema, server_schema)
 
 
     env.preload();
-    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Executable Name", "tmp_test_file");
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "tmp_test_file");
 
     config.preload();
     config.init();

core/version/version_ut/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_unit_test(
     version_ut
     "version_ut.cc"
-    "version;rest;singleton;environment;event_is;metric;"
+    "version;rest;singleton;environment;event_is;metric;messaging"
 )

nodes/orchestration/package/open-appsec-ctl.sh

@@ -1452,12 +1452,12 @@ upload_ai() # Initials - uai
         if [ "$AI_VERBOSE" = "true" ]; then
             echo "Uploading file $file"
         fi
-        if [ -z "${is_gaia}" -o "$is_smb_release" = "1" ]; then
+        if [ -n "${is_gaia}" -o "$is_smb_release" = "1" ]; then
-            uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
+            uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
         elif [ "${remove_curl_ld_path}" = "true" ]; then
-            uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
+            uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
         else
-            uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1)
+            uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1)
         fi
         if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "false" ] && [ "$uai_curl_output" = "413" ]; then
             AI_UPLOAD_TOO_LARGE_FLAG=true