Merge pull request #207 from openappsec/Nov_28_2024-Dev

Nov 28 2024 dev
2025-06-28 16:41:02 +03:00 · 2024-12-01 11:53:26 +02:00 · 2024-12-01 11:53:26 +02:00 · 0663f20691
commit 0663f20691
parent 6255e1f30d 2dda6231f6
63 changed files with 1272 additions and 707 deletions
--- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc
+++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc
@ -1135,7 +1135,11 @@ private:
            "webUserResponse"
        );

+        bool remove_event_id_param =
+            getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
+
        string uuid;
+        string redirectUrl;
        if (i_transaction_table->hasState<NginxAttachmentOpaque>()) {
            NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
            uuid = opaque.getSessionUUID();
@ -1145,7 +1149,12 @@ private:
        if (web_trigger_conf.getDetailsLevel() == "Redirect") {
            web_response_data.response_data.redirect_data.redirect_location_size =
                web_trigger_conf.getRedirectURL().size();
-            web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0;
+            bool add_event = web_trigger_conf.getAddEventId();
+            if (add_event && !remove_event_id_param) {
+                web_response_data.response_data.redirect_data.redirect_location_size +=
+                    strlen("?event_id=") + uuid.size();
+            }
+            web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
            web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
        } else {
            web_response_data.response_data.custom_response_data.title_size =
@ -1159,8 +1168,13 @@ private:
        verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));

        if (web_trigger_conf.getDetailsLevel() == "Redirect") {
-            verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getRedirectURL().data()));
-            verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size());
+            redirectUrl = web_trigger_conf.getRedirectURL();
+            if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
+                redirectUrl += "?event-id=" + uuid;
+            }
+
+            verdict_data.push_back(reinterpret_cast<const char *>(redirectUrl.data()));
+            verdict_data_sizes.push_back(redirectUrl.size());
        } else {
            verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getResponseTitle().data()));
            verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size());
--- a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc
+++ b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc
@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector<CIDRSData> &cidr_values)
 }

 Maybe<string>
-UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
+UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const
 {
    vector<string> header_values = split(str);

@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
    vector<string> xff_values = getHeaderValuesFromConfig("x-forwarded-for");
    vector<CIDRSData> cidr_values(xff_values.begin(), xff_values.end());

-    for (const string &value : header_values) {
-        if (!IPAddr::createIPAddr(value).ok()) {
-            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value;
+    for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) {
+        if (!IPAddr::createIPAddr(*it).ok()) {
+            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it;
            return genError("Invalid IP address");
        }
-        if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found");
+        if (type == ExtractType::PROXYIP) continue;
+        if (!isIpTrusted(*it, cidr_values)) {
+            dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it;
+            return *it;
+        }
+    }
+
+    if (!IPAddr::createIPAddr(header_values[0]).ok()) {
+        dbgWarning(D_NGINX_ATTACHMENT_PARSER)
+            << "Invalid IP address found in the xff header IPs list: "
+            << header_values[0];
+        return genError("Invalid IP address");
    }

    return header_values[0];
@ -312,7 +323,7 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
        return;
    }
    NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
-    auto value = parseXForwardedFor(header.getValue());
+    auto value = parseXForwardedFor(header.getValue(), type);
    if (!value.ok()) {
        dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
        return;
@ -321,12 +332,13 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
    if (type == ExtractType::SOURCEIDENTIFIER) {
        opaque.setSourceIdentifier(header.getKey(), value.unpack());
        dbgDebug(D_NGINX_ATTACHMENT_PARSER)
-            << "Added source identifir to XFF "
+            << "Added source identifier from XFF header"
            <<  value.unpack();
        opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
+        opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack());
        dbgTrace(D_NGINX_ATTACHMENT_PARSER)
-        << "XFF found, set ctx with value from header: "
-        << static_cast<string>(header.getValue());
+            << "XFF found, set ctx with value from header: "
+            << static_cast<string>(header.getValue());
    } else {
        opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack());
    }
--- a/components/include/reverse_proxy_defaults.h
+++ b/components/include/reverse_proxy_defaults.h
@ -28,7 +28,7 @@ static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/ngi
 static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf";
 static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
 static const std::string default_nginx_config_include_file =
-    "/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf";
+    "/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf";
 static const std::string default_global_conf_include_template =
    "/etc/cp/conf/rpmanager/nginx-conf-include-template";
 static const std::string default_global_conf_include_template_no_responses =
--- a/components/include/user_identifiers_config.h
+++ b/components/include/user_identifiers_config.h
@ -58,7 +58,7 @@ private:
        const std::string::const_iterator &end,
        const std::string &key) const;
    Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const;
-    Maybe<std::string> parseXForwardedFor(const std::string &str) const;
+    Maybe<std::string> parseXForwardedFor(const std::string &str, ExtractType type) const;

    std::vector<UsersIdentifiersConfig> user_identifiers;
 };
--- a/components/include/waap.h
+++ b/components/include/waap.h
@ -33,7 +33,6 @@ class I_WaapAssetStatesManager;
 class I_Messaging;
 class I_AgentDetails;
 class I_Encryptor;
-class I_WaapModelResultLogger;

 const std::string WAAP_APPLICATION_NAME = "waap application";

@ -51,8 +50,7 @@ class WaapComponent
    Singleton::Consume<I_AgentDetails>,
    Singleton::Consume<I_Messaging>,
    Singleton::Consume<I_Encryptor>,
-    Singleton::Consume<I_Environment>,
-    Singleton::Consume<I_WaapModelResultLogger>
+    Singleton::Consume<I_Environment>
 {
 public:
    WaapComponent();
--- a/components/security_apps/orchestration/details_resolver/details_resolver.cc
+++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc
@ -350,7 +350,7 @@ DetailsResolver::Impl::readCloudMetadata()
    }

    if (!cloud_metadata.ok()) {
-        dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr();
+        dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
        return genError("Failed to fetch cloud metadata");
    }

--- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
+++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
@ -18,6 +18,8 @@
 #include <regex>
 #include <boost/regex.hpp>
 #include <boost/algorithm/string.hpp>
+#include <cereal/external/rapidjson/document.h>
+#include <cereal/external/rapidjson/filereadstream.h>

 #if defined(gaia)

@ -100,6 +102,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
    return command_output;
 }

+Maybe<string>
+getOtlpAgentGaiaOsRole(const string &command_output)
+{
+    if (command_output == "" ) return string("-1");
+
+    return command_output;
+}
+
 Maybe<string>
 getQUID(const string &command_output)
 {
@ -111,6 +121,13 @@ getQUID(const string &command_output)
    return command_output;
 }

+Maybe<string>
+getIsAiopsRunning(const string &command_output)
+{
+    if (command_output == "" ) return string("false");
+    
+    return command_output;
+}

 Maybe<string>
 checkHasSDWan(const string &command_output)
@ -186,6 +203,24 @@ getMgmtObjAttr(shared_ptr<istream> file_stream, const string &attr)
    return genError("Object attribute was not found. Attr: " + attr);
 }

+Maybe<string>
+getAttrFromCpsdwanGetDataJson(const string &attr)
+{
+    static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json";
+    std::ifstream ifs(get_data_json_path);
+    if (ifs.is_open()) {
+        rapidjson::IStreamWrapper isw(ifs);
+        rapidjson::Document document;
+        document.ParseStream(isw);
+
+        if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) {
+            return string(document[attr.c_str()].GetString());
+        }
+    }
+
+    return genError("Attribute " + attr + " was not found in " + get_data_json_path);
+}
+
 Maybe<string>
 getMgmtObjUid(const string &command_output)
 {
@ -193,6 +228,11 @@ getMgmtObjUid(const string &command_output)
        return command_output;
    }

+    Maybe<string> obj_uuid = getAttrFromCpsdwanGetDataJson("uuid");
+    if (obj_uuid.ok()) {
+        return obj_uuid.unpack();
+    }
+
    static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
    auto file_stream = std::make_shared<std::ifstream>(obj_path);
    if (!file_stream->is_open()) {
@ -310,7 +350,12 @@ getSmbObjectName(const string &command_output)
    if (command_output.empty() || command_output[0] != centrally_managed_comd_output) {
        return genError("Object name was not found");
    }
-    
+
+    Maybe<string> obj_name = getAttrFromCpsdwanGetDataJson("name");
+    if (obj_name.ok()) {
+        return obj_name.unpack();
+    }
+
    static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
    auto ifs = std::make_shared<std::ifstream>(obj_path);
    if (!ifs->is_open()) {
--- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
+++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
@ -73,6 +73,15 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
    "&& python3 /opt/CPquid/Quid_Api.py -i "
    "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
    getQUID)
+SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
+    "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
+    getOtlpAgentGaiaOsRole)
+SHELL_CMD_HANDLER(
+    "IS_AIOPS_RUNNING",
+    "FS_PATH=<FILESYSTEM-PREFIX>; "
+    "PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
+    "[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
+    getIsAiopsRunning)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(
    "canUpdateSDWanData",
@ -180,8 +189,7 @@ SHELL_CMD_HANDLER(
 )
 SHELL_CMD_HANDLER(
    "managements",
-    "sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |"
-    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    "echo 1",
    extractManagements
 )
 #endif //gaia
@ -237,8 +245,7 @@ SHELL_CMD_HANDLER(

 SHELL_CMD_HANDLER(
    "managements",
-    "sed -n '/:masters (/,$p' /tmp/local.cfg |"
-    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    "echo 1",
    extractManagements
 )
 #endif//smb
--- a/components/security_apps/orchestration/hybrid_mode_telemetry.cc
+++ b/components/security_apps/orchestration/hybrid_mode_telemetry.cc
@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &)
 {
    auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
    auto maybe_cmd_output = shell_cmd->getExecOutput(
-        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count"
+        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count",
+        1000,
+        false
    );

    // get wd process restart count
--- a/components/security_apps/orchestration/include/declarative_policy_utils.h
+++ b/components/security_apps/orchestration/include/declarative_policy_utils.h
@ -79,8 +79,8 @@ public:
    ) override;
    std::string getUpdate(CheckUpdateRequest &request) override;
    bool shouldApplyPolicy() override;
-    void turnOffApplyPolicyFlag() override;
-    void turnOnApplyPolicyFlag() override;
+    void turnOffApplyLocalPolicyFlag() override;
+    void turnOnApplyLocalPolicyFlag() override;

    std::string getCurrPolicy() override { return curr_policy; }

@ -94,7 +94,7 @@ private:
    std::string curr_version;
    std::string curr_policy;
    std::string curr_checksum;
-    bool should_apply_policy;
+    bool should_apply_local_policy;
 };

 #endif // __DECLARATIVE_POLICY_UTILS_H__
--- a/components/security_apps/orchestration/include/i_declarative_policy.h
+++ b/components/security_apps/orchestration/include/i_declarative_policy.h
@ -22,8 +22,8 @@ public:

    virtual std::string getCurrPolicy() = 0;

-    virtual void turnOffApplyPolicyFlag() = 0;
-    virtual void turnOnApplyPolicyFlag() = 0;
+    virtual void turnOffApplyLocalPolicyFlag() = 0;
+    virtual void turnOnApplyLocalPolicyFlag() = 0;

 protected:
    virtual ~I_DeclarativePolicy() {}
--- a/components/security_apps/orchestration/orchestration_comp.cc
+++ b/components/security_apps/orchestration/orchestration_comp.cc
@ -2033,7 +2033,7 @@ private:
        }
        auto policy_mgmt_mode = getSettingWithDefault<string>("management", "profileManagedMode");
        if (getOrchestrationMode() == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative") {
-            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyPolicyFlag();
+            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyLocalPolicyFlag();
        }

        auto policy_version = i_service_controller->getPolicyVersion();
--- a/components/security_apps/orchestration/service_controller/service_controller.cc
+++ b/components/security_apps/orchestration/service_controller/service_controller.cc
@ -793,7 +793,7 @@ ServiceController::Impl::updateServiceConfiguration(
            << "Policy file was not updated. Sending reload command regarding settings and data";
        auto signal_services = sendSignalForServices(nano_services_to_update, "");
        if (!signal_services.ok()) return signal_services.passErr();
-        Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+        Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
        return Maybe<void>();
    }

@ -940,7 +940,7 @@ ServiceController::Impl::updateServiceConfiguration(
        if (new_policy_path.compare(config_file_path) == 0) {
            dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file";
            policy_version = version_value;
-            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+            Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
            return Maybe<void>();
        }

@ -959,7 +959,7 @@ ServiceController::Impl::updateServiceConfiguration(
    }

    if (!was_policy_updated && !send_signal_for_services_err.empty()) return genError(send_signal_for_services_err);
-    Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
+    Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
    return Maybe<void>();
 }

--- a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
+++ b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
@ -17,7 +17,7 @@ void
 DeclarativePolicyUtils::init()
 {
    local_policy_path = getFilesystemPathConfig() + "/conf/local_policy.yaml";
-    should_apply_policy = true;
+    should_apply_local_policy = true;
    Singleton::Consume<I_RestApi>::by<DeclarativePolicyUtils>()->addRestCall<ApplyPolicyRest>(
        RestAction::SET, "apply-policy"
    );
@ -40,7 +40,7 @@ DeclarativePolicyUtils::upon(const ApplyPolicyEvent &event)
 {
    dbgTrace(D_ORCHESTRATOR) << "Apply policy event";
    local_policy_path = event.getPolicyPath();
-    should_apply_policy = true;
+    should_apply_local_policy = true;
 }
 // LCOV_EXCL_STOP

@ -48,19 +48,24 @@ bool
 DeclarativePolicyUtils::shouldApplyPolicy()
 {
    auto env_type = Singleton::Consume<I_EnvDetails>::by<DeclarativePolicyUtils>()->getEnvType();
-    return env_type == EnvType::K8S ? true : should_apply_policy;
+    if (env_type == EnvType::K8S) {
+        I_OrchestrationTools *orch_tools = Singleton::Consume<I_OrchestrationTools>::by<DeclarativePolicyUtils>();
+        auto maybe_new_version = orch_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger");
+        return maybe_new_version != curr_version;
+    }
+    return should_apply_local_policy;
 }

 void
-DeclarativePolicyUtils::turnOffApplyPolicyFlag()
+DeclarativePolicyUtils::turnOffApplyLocalPolicyFlag()
 {
-    should_apply_policy = false;
+    should_apply_local_policy = false;
 }

 void
-DeclarativePolicyUtils::turnOnApplyPolicyFlag()
+DeclarativePolicyUtils::turnOnApplyLocalPolicyFlag()
 {
-    should_apply_policy = true;
+    should_apply_local_policy = true;
 }

 Maybe<string>
@ -211,6 +216,6 @@ DeclarativePolicyUtils::periodicPolicyLoad()

    if (*new_checksum == curr_checksum) return;

-    should_apply_policy = true;
+    should_apply_local_policy = true;
    curr_checksum = *new_checksum;
 }
--- a/components/security_apps/orchestration/update_communication/fog_communication.cc
+++ b/components/security_apps/orchestration/update_communication/fog_communication.cc
@ -74,7 +74,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
                << " to: "
                << policy_mgmt_mode;
            profile_mode = policy_mgmt_mode;
-            i_declarative_policy->turnOnApplyPolicyFlag();
+            i_declarative_policy->turnOnApplyLocalPolicyFlag();
        }

        if (i_declarative_policy->shouldApplyPolicy()) {
--- a/components/security_apps/rate_limit/rate_limit.cc
+++ b/components/security_apps/rate_limit/rate_limit.cc
@ -293,8 +293,13 @@ public:
            return ACCEPT;
        }

-        burst = rule.getRateLimit();
-        limit = calcRuleLimit(rule);
+        auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1;
+        if (replicas == 0) {
+            dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1";
+            replicas = 1;
+        }
+        burst = static_cast<float>(rule.getRateLimit()) / replicas;
+        limit = static_cast<float>(calcRuleLimit(rule)) / replicas;

        dbgTrace(D_RATE_LIMIT)
            << "found rate limit rule with: "
--- a/components/security_apps/waap/include/i_waap_model_result_logger.h
+++ b/components/security_apps/waap/include/i_waap_model_result_logger.h
@ -1,37 +0,0 @@
-// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-class IWaf2Transaction;
-struct Waf2ScanResult;
-namespace Waap {
-namespace Scores {
-struct ModelLoggingSettings;
-}
-}
-
-class I_WaapModelResultLogger {
-public:
-    virtual ~I_WaapModelResultLogger() {}
-
-    virtual void
-    logModelResult(
-        Waap::Scores::ModelLoggingSettings &settings,
-        IWaf2Transaction* transaction,
-        Waf2ScanResult &res,
-        std::string modelName,
-        std::string otherModelName,
-        double newScore,
-        double baseScore) = 0;
-};
--- a/components/security_apps/waap/waap_clib/CMakeLists.txt
+++ b/components/security_apps/waap/waap_clib/CMakeLists.txt
@ -87,9 +87,10 @@ add_library(waap_clib
    ParserPairs.cc
    Waf2Util2.cc
    ParserPDF.cc
+    ParserKnownBenignSkipper.cc
+    ParserScreenedJson.cc
    ParserBinaryFile.cc
    RegexComparator.cc
-    WaapModelResultLogger.cc
 )

 add_definitions("-Wno-unused-function")
--- a/components/security_apps/waap/waap_clib/DeepParser.cc
+++ b/components/security_apps/waap/waap_clib/DeepParser.cc
@ -28,6 +28,8 @@
 #include "ParserDelimiter.h"
 #include "ParserPDF.h"
 #include "ParserBinaryFile.h"
+#include "ParserKnownBenignSkipper.h"
+#include "ParserScreenedJson.h"
 #include "WaapAssetState.h"
 #include "Waf2Regex.h"
 #include "Waf2Util.h"
@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
            isRefererParamPayload,
            isUrlPayload,
            isUrlParamPayload,
+            isCookiePayload,
            flags,
            parser_depth,
            base64BinaryFileType
@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
            isRefererParamPayload,
            isUrlPayload,
            isUrlParamPayload,
+            isCookiePayload,
            flags,
            parser_depth,
            base64BinaryFileType
@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
                isRefererParamPayload,
                isUrlPayload,
                isUrlParamPayload,
+                isCookiePayload,
                flags,
                parser_depth,
                base64ParamFound,
@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
    bool isRefererParamPayload,
    bool isUrlPayload,
    bool isUrlParamPayload,
+    bool isCookiePayload,
    int flags,
    size_t parser_depth,
    bool base64ParamFound,
@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
            isRefererParamPayload,
            isUrlPayload,
            isUrlParamPayload,
+            isCookiePayload,
            flags,
            parser_depth,
            b64FileType
@ -918,6 +925,7 @@ bool isRefererPayload,
 bool isRefererParamPayload,
 bool isUrlPayload,
 bool isUrlParamPayload,
+bool isCookiePayload,
 int flags,
 size_t parser_depth
 ) {
@ -959,6 +967,7 @@ DeepParser::createInternalParser(
    bool isRefererParamPayload,
    bool isUrlPayload,
    bool isUrlParamPayload,
+    bool isCookiePayload,
    int flags,
    size_t parser_depth,
    Waap::Util::BinaryFileType b64FileType
@ -978,7 +987,19 @@ DeepParser::createInternalParser(
        << "\n\tflags: "
        << flags
        << "\n\tparser_depth: "
-        << parser_depth;
+        << parser_depth
+        << "\n\tisBodyPayload: "
+        << isBodyPayload
+        << "\n\tisRefererPayload: "
+        << isRefererPayload
+        << "\n\tisRefererParamPayload: "
+        << isRefererParamPayload
+        << "\n\tisUrlPayload: "
+        << isUrlPayload
+        << "\n\tisUrlParamPayload: "
+        << isUrlParamPayload
+        << "\n\tisCookiePayload: "
+        << isCookiePayload;
    bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false,
        isAmperType = false;
    bool isKeyValDelimited = false;
@ -1045,6 +1066,53 @@ DeepParser::createInternalParser(
        }
    }

+    if (Waap::Util::isScreenedJson(cur_val)) {
+        dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON";
+        m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserScreenedJson>>(*this, parser_depth + 1));
+        offset = 0;
+        return offset;
+    }
+
+    dbgTrace(D_WAAP_DEEP_PARSER)
+        << "Offset = "
+        << offset
+        << " depth = "
+        << m_depth
+        << " isBodyPayload = "
+        << isBodyPayload;
+    //Detect sensor_data format in body and just use dedicated filter for it
+    if (m_depth == 1
+        && isBodyPayload
+        && Waap::Util::detectKnownSource(cur_val) ==  Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
+        m_parsersDeque.push_back(
+            std::make_shared<BufferedParser<ParserKnownBenignSkipper>>(
+                *this,
+                parser_depth + 1,
+                Waap::Util::SOURCE_TYPE_SENSOR_DATA
+            )
+        );
+        offset = 0;
+        dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it";
+        return offset;
+    }
+    // Detect cookie parameter sensorsdata2015jssdkcross
+    // and causes false positives due to malformed JSON. Make preprocessing to parse it correctly
+    if (m_depth == 2
+        && isCookiePayload) {
+        offset = Waap::Util::definePrefixedJson(cur_val);
+        if (offset >= 0) {
+            m_parsersDeque.push_back(
+                std::make_shared<BufferedParser<ParserJson>>(
+                    *this,
+                    parser_depth + 1,
+                    m_pTransaction
+                )
+            );
+            dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data";
+            return offset;
+        }
+    }
+
    // Detect wbxml (binary XML) data type
    if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) {
        m_is_wbxml = true;
@ -1374,6 +1442,7 @@ DeepParser::createInternalParser(
            isRefererParamPayload,
            isUrlPayload,
            isUrlParamPayload,
+            isCookiePayload,
            flags,
            parser_depth
            );
--- a/components/security_apps/waap/waap_clib/DeepParser.h
+++ b/components/security_apps/waap/waap_clib/DeepParser.h
@ -129,6 +129,7 @@ private:
        bool isRefererParamPayload,
        bool isUrlPayload,
        bool isUrlParamPayload,
+        bool isCookiePayload,
        int flags,
        size_t parser_depth,
        Waap::Util::BinaryFileType b64FileType
@ -144,6 +145,7 @@ private:
        bool isRefererParamPayload,
        bool isUrlPayload,
        bool isUrlParamPayload,
+        bool isCookiePayload,
        int flags,
        size_t parser_depth
    );
@ -160,6 +162,7 @@ private:
        bool isRefererParamPayload,
        bool isUrlPayload,
        bool isUrlParamPayload,
+        bool isCookiePayload,
        int flags,
        size_t parser_depth,
        bool base64ParamFound,
--- a/components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.cc
+++ b/components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.cc
@ -0,0 +1,139 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ParserKnownBenignSkipper.h"
+#include "Waf2Util.h"
+#include "debug.h"
+#include <string.h>
+USE_DEBUG_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER);
+USE_DEBUG_FLAG(D_WAAP);
+
+const std::string ParserKnownBenignSkipper::m_parserName = "ParserKnownBenignSkipper";
+const char* DATA_SENSOR_TAIL = "\"}";
+
+ParserKnownBenignSkipper::ParserKnownBenignSkipper(
+    IParserStreamReceiver &receiver,
+    size_t parser_depth,
+    Waap::Util::KnownSourceType source_type
+) :
+    m_receiver(receiver),
+    m_state(s_start),
+    m_parser_depth(parser_depth),
+    m_source_type(source_type)
+{}
+
+ParserKnownBenignSkipper::~ParserKnownBenignSkipper()
+{}
+
+size_t
+ParserKnownBenignSkipper::push(const char *buf, size_t len)
+{
+    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+        << "buf='"
+        << std::string(buf, std::min((size_t)200, len))
+        << (len > 200 ? "..." : "")
+        << "' len="
+        << len
+        << " depth="
+        << depth();
+
+    const char *c;
+
+    if (m_state == s_error) {
+        return 0;
+    }
+    if (len == 0)
+    {
+        dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+            << "ParserKnownBenignSkipper::push(): end of stream. m_state="
+            << m_state;
+
+        if (m_state == s_end) {
+            m_receiver.onKvDone();
+        } else {
+            m_state = s_error;
+        }
+        return 0;
+    }
+
+    size_t tail_lookup_offset = 0;
+
+    switch (m_state) {
+            case s_start:
+                m_state = s_body;
+                CP_FALL_THROUGH;
+            case s_body:
+                {
+                    if (m_source_type == Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
+                        tail_lookup_offset =
+                            (len > MAX_DATA_SENSOR_TAIL_LOOKUP) ? len - MAX_DATA_SENSOR_TAIL_LOOKUP : 0;
+                        c = strstr(buf + tail_lookup_offset, DATA_SENSOR_TAIL);
+                        if (c) {
+                            dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                                << "ParserKnownBenignSkipper::push(): found end of sensor data";
+                            m_state = s_end;
+                            CP_FALL_THROUGH;
+                        } else {
+                            break;
+                        }
+                    } else {
+                        dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                            << "ParserKnownBenignSkipper::push(): unknown source type";
+                        m_state = s_error;
+                        break;
+                    }
+                }
+            case s_end:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state = end";
+                if (m_receiver.onKey("SENSOR_DATA", 11) != 0) {
+                    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state moving to error onKey";
+                    m_state = s_error;
+                    return 0;
+                }
+                if (m_receiver.onValue("", 0) != 0) {
+                    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state moving to error onValue";
+                    m_state = s_error;
+                    return 0;
+                }
+                break;
+            case s_error:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER) << "state = error";
+                break;
+            default:
+                dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+                    << "ParserKnownBenignSkipper::push(): unknown state: "
+                    << m_state;
+                m_state = s_error;
+                return 0;
+    }
+    dbgTrace(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER)
+        << "ParserKnownBenignSkipper::push(): final state: "
+        << m_state;
+    return len;
+}
+
+
+void ParserKnownBenignSkipper::finish()
+{
+    push(NULL, 0);
+}
+
+const std::string& ParserKnownBenignSkipper::name() const
+{
+    return m_parserName;
+}
+
+bool ParserKnownBenignSkipper::error() const
+{
+    return m_state == s_error;
+}
--- a/components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.h
+++ b/components/security_apps/waap/waap_clib/ParserKnownBenignSkipper.h
@ -0,0 +1,52 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_BENIGN_SKIPPER_H__
+#define __PARSER_BENIGN_SKIPPER_H__
+
+#include "ParserBase.h"
+#include "Waf2Util.h"
+#include <string.h>
+#include "Waf2Util.h"
+
+#define MAX_DATA_SENSOR_TAIL_LOOKUP 5
+
+class ParserKnownBenignSkipper : public ParserBase {
+public:
+    ParserKnownBenignSkipper(
+        IParserStreamReceiver &receiver,
+        size_t parser_depth,
+        Waap::Util::KnownSourceType source_type=Waap::Util::SOURCE_TYPE_UNKNOWN);
+    virtual ~ParserKnownBenignSkipper();
+    virtual size_t push(const char *buf, size_t len);
+    virtual void finish();
+    virtual const std::string &name() const;
+    virtual bool error() const;
+    virtual size_t depth() { return 1; }
+
+private:
+    enum state {
+        s_start,
+        s_body,
+        s_end,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    enum state m_state;
+    static const std::string m_parserName;
+    size_t m_parser_depth;
+    Waap::Util::KnownSourceType m_source_type;
+};
+
+#endif // __PARSER_BENIGN_SKIPPER_H__
--- a/components/security_apps/waap/waap_clib/ParserScreenedJson.cc
+++ b/components/security_apps/waap/waap_clib/ParserScreenedJson.cc
@ -0,0 +1,187 @@
+#include "ParserScreenedJson.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_WAAP_PARSER_SCREENED_JSON);
+
+const std::string ParserScreenedJson::m_parserName = "ParserScreenedJson";
+
+ParserScreenedJson::ParserScreenedJson(IParserStreamReceiver &receiver, size_t parser_depth) :
+    m_receiver(receiver),
+    m_state(s_start),
+    m_unscreenedLen(0),
+    m_leftoverLen(0),
+    m_parser_depth(parser_depth)
+{
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+        << "parser_depth="
+        << parser_depth;
+
+    memset(m_unscreened, 0, sizeof(m_unscreened));
+}
+
+ParserScreenedJson::~ParserScreenedJson()
+{}
+
+size_t
+ParserScreenedJson::push(const char *buf, size_t len)
+{
+    size_t i = 0;
+    char c;
+
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON) << "ParserScreenedJson::push(): starting (len=" << len << ")";
+
+    if (len == 0) {
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON) << "ParserScreenedJson::push(): end of data signal! m_state=" << m_state;
+        // flush unescaped data collected (if any)
+        if (m_leftoverLen > 0) {
+            // No need any processing for leftover data - last char must be doublequote, else - error
+            m_state = s_error;
+            dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                << "ParserScreenedJson::push(): end of data and leftover detected m_state="
+                << m_state;
+            return i;
+        }
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+            << "ParserScreenedJson::push(): s_value, pushing m_unscreened = "
+            << m_unscreened
+            << ", m_leftoverLen = "
+            << m_leftoverLen
+            << ", m_unscreenedLen = "
+            << m_unscreenedLen;
+
+        if (m_receiver.onKey("json_unscreened", 15) != 0) {
+            m_state = s_error;
+            return i;
+        }
+
+        if (m_receiver.onValue(m_unscreened, m_unscreenedLen) != 0) {
+            m_state = s_error;
+            return i;
+        }
+
+        if (m_receiver.onKvDone() != 0)
+        {
+            m_state = s_error;
+            return i;
+        }
+        return 0;
+    }
+
+    while (i < len)
+    {
+        c = buf[i];
+
+        dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+            << "ParserScreenedJson::push(): state="
+            << m_state
+            << "; c='"
+            << c
+            << "'"
+            << "; i="
+            << i
+            << ", m_leftoverLen = "
+            << m_leftoverLen
+            << ", m_unscreenedLen = "
+            << m_unscreenedLen
+            << ", m_unscreened = "
+            << m_unscreened;
+
+        switch (m_state)
+        {
+            case s_start:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_start";
+                m_state = s_value;
+
+                // fallthrough  not required, removing 1st doublequote, it denoted by regex //
+                //CP_FALL_THROUGH;
+                break;
+            }
+            case s_value:
+            {
+                if (c == '\\') {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\\';
+                        m_leftoverLen = 0;
+                        m_unscreenedLen++;
+                    } else {
+                        m_leftoverLen++;
+                    }
+                } else  if (c =='\"') {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\"';
+                        m_unscreenedLen++;
+                        if (m_leftoverLen > 0) {
+                            m_leftoverLen = 0;
+                        }
+                    }
+                } else {
+                    if (m_leftoverLen > 0) {
+                        m_unscreened[m_unscreenedLen] = '\\';
+                        m_unscreenedLen++;
+                        m_leftoverLen = 0;
+                    }
+                    m_unscreened[m_unscreenedLen] = c;
+                    m_unscreenedLen++;
+                }
+                if (m_unscreenedLen >= MAX_UNSCREENED_JSON_SIZE) {
+                    if (m_receiver.onKey("json_unscreened", 15) != 0) {
+                        m_state = s_error;
+                        return i;
+                    }
+                    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_value, pushing m_unscreened = "
+                    << m_unscreened
+                    << ", m_leftoverLen = "
+                    << m_leftoverLen
+                    << ", m_unscreenedLen = "
+                    << m_unscreenedLen;
+                    if (m_receiver.onValue(m_unscreened, m_unscreenedLen) != 0) {
+                        m_state = s_error;
+                        return i;
+                    }
+                    m_unscreenedLen = 0;
+                }
+            break;
+            }
+            case s_error:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): s_error";
+                return 0;
+            }
+            default:
+            {
+                dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+                    << "ParserScreenedJson::push(): JSON parser unrecoverable error";
+                m_state = s_error;
+                return 0;
+            }
+        }
+        ++i;
+    }
+
+    dbgTrace(D_WAAP_PARSER_SCREENED_JSON)
+        << "ParserScreenedJson::push(): finished: len="
+        << len;
+    return len;
+}
+
+void
+ParserScreenedJson::finish()
+{
+    push(NULL, 0);
+}
+
+const std::string &
+ParserScreenedJson::name() const
+{
+    return m_parserName;
+}
+
+bool
+ParserScreenedJson::error() const
+{
+    return m_state == s_error;
+}
--- a/components/security_apps/waap/waap_clib/ParserScreenedJson.h
+++ b/components/security_apps/waap/waap_clib/ParserScreenedJson.h
@ -0,0 +1,52 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_SCREENED_JSON_H_
+#define __PARSER_SCREENED_JSON_H_
+
+#include "ParserBase.h"
+#include <string.h>
+
+#define MAX_UNSCREENED_JSON_SIZE 4095
+
+class ParserScreenedJson : public ParserBase {
+public:
+    ParserScreenedJson(IParserStreamReceiver &receiver, size_t parser_depth);
+    virtual ~ParserScreenedJson();
+    size_t push(const char *data, size_t data_len);
+    void finish();
+    virtual const std::string &name() const;
+    bool error() const;
+    // LCOV_EXCL_START Reason: The function not in use, compliance with the interface
+    virtual size_t depth() { return 1; }
+    // LCOV_EXCL_STOP
+
+private:
+    enum state
+    {
+        s_start,
+        s_value,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    enum state m_state;
+    size_t m_unscreenedLen;
+    char m_unscreened[MAX_UNSCREENED_JSON_SIZE];
+    size_t m_leftoverLen;
+    static const std::string m_parserName;
+    size_t m_parser_depth;
+};
+
+#endif
+
--- a/components/security_apps/waap/waap_clib/WaapAssetState.cc
+++ b/components/security_apps/waap/waap_clib/WaapAssetState.cc
@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
 #endif
 }

+// std::string nicePrint() - is a function used to create std::string that will represent all data that is
+// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
+// into the object easier.
+
+std::string
+WaapAssetState::nicePrint(Waf2ScanResult &res) const
+{
+    std::string result = "Waf2ScanResult:\n";
+    result += "keyword_matches:\n";
+    for (const auto &keyword : res.keyword_matches) {
+        result += keyword + "\n";
+    }
+    result += "regex_matches:\n";
+    for (const auto &regex : res.regex_matches) {
+        result += regex + "\n";
+    }
+    result += "filtered_keywords:\n";
+    for (const auto &filtered : res.filtered_keywords) {
+        result += filtered + "\n";
+    }
+    result += "found_patterns:\n";
+    for (const auto &pattern : res.found_patterns) {
+        result += pattern.first + ":\n";
+        for (const auto &value : pattern.second) {
+            result += value + "\n";
+        }
+    }
+    result += "unescaped_line: " + res.unescaped_line + "\n";
+    result += "param_name: " + res.param_name + "\n";
+    result += "location: " + res.location + "\n";
+    result += "score: " + std::to_string(res.score) + "\n";
+    result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n";
+    result += "scoreArray:\n";
+    for (const auto &score : res.scoreArray) {
+        result += std::to_string(score) + "\n";
+    }
+    result += "keywordCombinations:\n";
+    for (const auto &combination : res.keywordCombinations) {
+        result += combination + "\n";
+    }
+    result += "attack_types:\n";
+    for (const auto &attack : res.attack_types) {
+        result += attack + "\n";
+    }
+    result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n";
+    return result;
+}
+
+
 bool
 checkBinaryData(const std::string &line, bool binaryDataFound)
 {
@ -1033,7 +1082,7 @@ WaapAssetState::apply(
    // Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
    // This is done to improve performance of regex matching.
    SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
    checkRegex(
        unescapedLineSample,
        m_Signatures->specific_acuracy_keywords_regex,
@ -1111,7 +1160,7 @@ WaapAssetState::apply(
    }

    bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns);
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res);
    if (os_cmd_ev) {
        dbgTrace(D_WAAP_EVASIONS) << "os command evasion found";

@ -1295,6 +1344,47 @@ WaapAssetState::apply(
        }
    }

+    bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns);
+    dbgTrace(D_WAAP_EVASIONS)
+        << "path_traversal_ev = " << path_traversal_ev
+        << " sample = " << res.unescaped_line
+        << " res.unescaped_line.find(2f) =  " << res.unescaped_line.find("2f");
+    if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) {
+        // Possible path traversal evasion .2f. detected: - clean up and scan with regexes again.
+        dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line
+            << "Status beroe evasion checking " << nicePrint(res);
+
+        std::string unescaped = line;
+        replaceAll(unescaped, "2f", "/");
+        size_t kwCount = res.keyword_matches.size();
+
+        if (res.unescaped_line != unescaped) {
+            SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
+            checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
+                res.found_patterns, longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+        }
+
+        if (kwCount == res.keyword_matches.size()) {
+            // Remove the evasion keyword if no real evasion found
+            keywordsToRemove.push_back("path_traversal");
+            path_traversal_ev = false;
+        }
+        else if (!binaryDataFound) {
+            // Recalculate repetition and/or probing indicators
+            unsigned int newWordsCount = 0;
+            calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
+                newWordsCount);
+            // Take minimal words count because empirically it means evasion was probably succesfully decoded
+            wordsCount = std::min(wordsCount, newWordsCount);
+        }
+        dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
+    }
+
+
    bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
        "quotes_space_ev_fast_reg",
        res.found_patterns
@ -1726,7 +1816,7 @@ WaapAssetState::apply(
            wordsCount = std::min(wordsCount, newWordsCount);
        }
    }
-
+    dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
    // Remove evasion keywords that should not be reported because there's no real evasion found
    if (!keywordsToRemove.empty()) {
        dbgTrace(D_WAAP_SAMPLE_SCAN)
--- a/components/security_apps/waap/waap_clib/WaapAssetState.h
+++ b/components/security_apps/waap/waap_clib/WaapAssetState.h
@ -49,6 +49,7 @@ private: //ugly but needed for build
        Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;

    void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
+    std::string nicePrint(Waf2ScanResult &res) const;

 public:
    // Load and compile signatures from file
--- a/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc
+++ b/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc
@ -1,250 +0,0 @@
-// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "WaapModelResultLogger.h"
-#include "Waf2Engine.h"
-#include "i_time_get.h"
-#include "i_messaging.h"
-#include "i_instance_awareness.h"
-#include "http_manager.h"
-#include "LogGenWrapper.h"
-#include "rest.h"
-#include "debug.h"
-
-USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
-
-using namespace std;
-
-static const unsigned int MAX_FILES_PER_WINDOW = 5;
-static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
-static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
-
-class WaapModelReport : public RestGetFile
-{
-public:
-    WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
-
-private:
-    C2S_PARAM(vector<WaapModelResult>, data);
-};
-
-class WaapModelResultLogger::Impl
-        :
-    Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
-{
-public:
-    Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
-        last_sent_s3(std::chrono::minutes::zero()),
-        last_kusto_log_window(std::chrono::minutes::zero()) {}
-    virtual ~Impl();
-    void
-    logModelResult(
-        Waap::Scores::ModelLoggingSettings &settings,
-        IWaf2Transaction* transaction,
-        Waf2ScanResult &res,
-        string modelName,
-        string otherModelName,
-        double score,
-        double otherScore) override;
-
-private:
-    void logToStream(WaapModelResult &result, chrono::minutes now);
-    void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
-    bool shouldSendLogsToS3(chrono::minutes now);
-    void sendLogsToS3();
-    size_t max_logs;
-    unsigned int sent_files_count;
-    unsigned int sent_logs_count;
-    std::chrono::minutes last_sent_s3;
-    std::chrono::minutes last_kusto_log_window;
-    std::map<std::string, vector<WaapModelResult>> logs;
-};
-
-WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
-{
-}
-
-WaapModelResultLogger::~WaapModelResultLogger()
-{
-}
-
-void
-WaapModelResultLogger::logModelResult(
-    Waap::Scores::ModelLoggingSettings &settings,
-    IWaf2Transaction* transaction,
-    Waf2ScanResult &res,
-    std::string modelName,
-    std::string otherModelName,
-    double score,
-    double otherScore
-)
-{
-    pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
-}
-
-void
-WaapModelResultLogger::Impl::logModelResult(
-    Waap::Scores::ModelLoggingSettings &settings,
-    IWaf2Transaction* transaction,
-    Waf2ScanResult &res,
-    string modelName,
-    string otherModelName,
-    double score,
-    double otherScore)
-{
-    if (transaction == NULL) return;
-    if (!Singleton::exists<I_Messaging>()) {
-        dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
-        return;
-    }
-
-    double score_diff = score - otherScore;
-    if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
-        ! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
-        (score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
-        return;
-    }
-
-    auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
-    auto now = chrono::duration_cast<chrono::minutes>(current_time);
-
-    WaapModelResult result = WaapModelResult(
-        *transaction,
-        res,
-        modelName,
-        otherModelName,
-        score,
-        otherScore,
-        now.count()
-        );
-
-    if (settings.logToStream) logToStream(result, now);
-    if (settings.logToS3) logToS3(result, transaction, now);
-}
-
-void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
-{
-    auto asset_state = transaction->getAssetState();
-    string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
-    auto asset_logs = logs.find(asset_id);
-    if (asset_logs == logs.end()) {
-        logs.emplace(asset_id, vector<WaapModelResult>());
-    }
-    logs.at(asset_id).push_back(result);
-    if (shouldSendLogsToS3(now)) {
-        sendLogsToS3();
-    }
-}
-
-void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
-{
-    if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
-        last_kusto_log_window = now;
-        sent_logs_count = 0;
-    }
-    else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
-        return;
-    }
-    sent_logs_count++;
-    dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
-
-    auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
-    LogGenWrapper logGenWrapper(
-        maybeLogTriggerConf,
-        "WAAP Model Telemetry",
-        ReportIS::Audience::SECURITY,
-        LogTriggerConf::SecurityType::ThreatPrevention,
-        ReportIS::Severity::CRITICAL,
-        ReportIS::Priority::HIGH,
-        false);
-
-    LogGen& waap_log = logGenWrapper.getLogGen();
-    waap_log.addMarkerSuffix(result.location);
-    waap_log << LogField("httpuripath", result.uri);
-    waap_log << LogField("matchedlocation", result.location);
-    waap_log << LogField("matchedparameter", result.param);
-    waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
-    waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
-    waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
-    waap_log << LogField("waapfinalscore", (int)(result.score * 100));
-    waap_log << LogField("indicatorssource", result.modelName);
-    waap_log << LogField("indicatorsversion", result.otherModelName);
-}
-
-bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
-{
-    if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
-    for (const auto &asset_logs : logs) {
-        if (asset_logs.second.size() >= max_logs) return true;
-    }
-    return false;
-}
-
-void WaapModelResultLogger::Impl::sendLogsToS3()
-{
-    dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
-
-    I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
-
-    for (auto &asset_logs : logs) {
-        if (asset_logs.second.empty()) {
-            continue;
-        }
-        if (sent_files_count >= MAX_FILES_PER_WINDOW) {
-            dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
-            asset_logs.second.clear();
-            continue;
-        }
-        I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
-        string tenant_id = agentDetails->getTenantId();
-        string agent_id = agentDetails->getAgentId();
-        string asset_id = asset_logs.first;
-        if (Singleton::exists<I_InstanceAwareness>()) {
-            I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
-            Maybe<string> uniqueId = instance->getUniqueID();
-            if (uniqueId.ok())
-            {
-                agent_id += "/" + uniqueId.unpack();
-            }
-        }
-        string uri = "/storage/waap/" +
-                    tenant_id + "/" + asset_id + "/waap_model_results/window_" +
-                    to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
-                    "/" + agent_id + "/data.data";
-        WaapModelReport report = WaapModelReport(asset_logs.second);
-
-        dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
-            ", length " << asset_logs.second.size() <<
-            ", uri " << uri;
-        msg->sendAsyncMessage(
-            HTTPMethod::PUT,
-            uri,
-            report,
-            MessageCategory::LOG
-        );
-
-        asset_logs.second.clear();
-    }
-
-    auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
-    auto now = chrono::duration_cast<chrono::minutes>(current_time);
-    if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
-        last_sent_s3 = now;
-        sent_files_count = 0;
-    } else {
-        sent_files_count++;
-    }
-}
-
-WaapModelResultLogger::Impl::~Impl()
-{}
--- a/components/security_apps/waap/waap_clib/WaapModelResultLogger.h
+++ b/components/security_apps/waap/waap_clib/WaapModelResultLogger.h
@ -1,109 +0,0 @@
-// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#pragma once
-
-#include <memory>
-#include <string>
-#include <map>
-#include <chrono>
-#include <vector>
-#include <ostream>
-#include "cereal/archives/json.hpp"
-
-#include "i_waap_model_result_logger.h"
-#include "DeepAnalyzer.h"
-#include "i_transaction.h"
-#include "ScanResult.h"
-#include "WaapAssetState.h"
-#include "WaapScores.h"
-
-class WaapModelResultLogger
-        :
-    Singleton::Provide<I_WaapModelResultLogger>
-{
-public:
-    WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
-    virtual ~WaapModelResultLogger();
-    virtual void logModelResult(
-        Waap::Scores::ModelLoggingSettings &settings,
-        IWaf2Transaction* transaction,
-        Waf2ScanResult &res,
-        std::string modelName,
-        std::string otherModelName,
-        double score,
-        double otherScore
-    );
-    class Impl;
-
-protected:
-    std::unique_ptr<Impl> pimpl;
-    static const size_t MAX_WAAP_MODEL_LOGS = 20000;
-};
-
-class WaapModelResult
-{
-public:
-    WaapModelResult(
-        IWaf2Transaction &transaction,
-        Waf2ScanResult &res,
-        const std::string &modelName,
-        const std::string &otherModelName,
-        double score,
-        double otherScore,
-        uint64_t time
-    ) : uri(transaction.getUri()), location(res.location), param(res.param_name),
-        modelName(modelName), otherModelName(otherModelName),
-        score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
-        sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
-    {
-    }
-
-    template<class Archive>
-    void serialize(Archive &ar) const
-    {
-        ar(cereal::make_nvp("uri", uri));
-        ar(cereal::make_nvp("location", location));
-        ar(cereal::make_nvp("param", param));
-        ar(cereal::make_nvp("modelName", modelName));
-        ar(cereal::make_nvp("otherModelName", otherModelName));
-        ar(cereal::make_nvp("score", score));
-        ar(cereal::make_nvp("otherScore", otherScore));
-        ar(cereal::make_nvp("keywords", keywords));
-        ar(cereal::make_nvp("sample", sample));
-        ar(cereal::make_nvp("id", id));
-        ar(cereal::make_nvp("time", time));
-    }
-
-    std::string toString() const
-    {
-        std::stringstream message_stream;
-        {
-            cereal::JSONOutputArchive ar(message_stream);
-            serialize(ar);
-        }
-        return message_stream.str();
-    }
-
-    std::string uri;
-    std::string location;
-    std::string param;
-    std::string modelName;
-    std::string otherModelName;
-    double score;
-    double otherScore;
-    std::vector<std::string> keywords;
-    std::string sample;
-    uint64_t id;
-    uint64_t time;
-};
--- a/components/security_apps/waap/waap_clib/WaapOverride.h
+++ b/components/security_apps/waap/waap_clib/WaapOverride.h
@ -48,8 +48,9 @@ public:
            m_tag = to_lower_copy(m_tag);

            if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
-                m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
-                m_tag != "responsebody" &&  m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") {
+                m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" &&
+                m_tag != "paramlocation" && m_tag != "responsebody" &&  m_tag != "headername" &&
+                m_tag != "headervalue" && m_tag != "method") {
                m_isValid = false;
                dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
            }
--- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
+++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
@ -105,7 +105,7 @@ bool WaapOverrideFunctor::operator()(
            }
            return false;
        }
-        else if (tagLower == "keyword") {
+        else if (tagLower == "keyword" || tagLower == "indicator") {
            for (const auto &rx : rxes) {
                for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
                    if (REGX_MATCH(keywordStr)) {
--- a/components/security_apps/waap/waap_clib/WaapScanner.cc
+++ b/components/security_apps/waap/waap_clib/WaapScanner.cc
@ -15,7 +15,6 @@
 #include "WaapScores.h"
 #include "Waf2Engine.h"
 #include "i_transaction.h"
-#include "WaapModelResultLogger.h"
 #include <string>
 #include "debug.h"
 #include "reputation_features_events.h"
@ -111,10 +110,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
    for (auto keyword : newKeywords) {
        res.keywordsAfterFilter.push_back(keyword);
    }
-    res.scoreArray.clear();
-    res.coefArray.clear();
-    res.keywordCombinations.clear();
-
    double res_score = getScoreFromPool(res, newKeywords, poolName);

    std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
@ -123,14 +118,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
    if (applyLearning && poolName != other_pool_name &&
        modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
        double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
-
        dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
                                << ", vs. pool " << other_pool_name << ": " << other_score
                                << ", score difference: " << res_score - other_score
                                << ", sample: " << res.unescaped_line;
-        Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
-            modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
-        );
        res.other_model_score = other_score;
    } else {
        res.other_model_score = res_score;
@ -142,6 +133,9 @@ double Waap::Scanner::getScoreFromPool(
    Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
 )
 {
+    res.scoreArray.clear();
+    res.coefArray.clear();
+    res.keywordCombinations.clear();
    KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);

    if (!newKeywords.empty()) {
--- a/components/security_apps/waap/waap_clib/Waf2Engine.cc
+++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc
@ -1358,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
        dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
        return false;
    }
-    std::string body(data);
+    std::string body(data, data_len);
    if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
    {
        dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
@ -1661,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
    waapLog << LogField("sourcePort", m_remote_port);
    waapLog << LogField("httpHostName", m_hostStr);
    waapLog << LogField("httpMethod", m_methodStr);
+    if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
+    if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
+
    const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
        m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
    bool send_extended_log = shouldSendExtendedLog(triggerLog);
@ -2343,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
        exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
        exceptions_dict["url"].insert(getUriStr());
        exceptions_dict["hostName"].insert(m_hostStr);
+        exceptions_dict["method"].insert(m_methodStr);

        for (auto &keyword : res.keyword_matches) {
            exceptions_dict["indicator"].insert(keyword);
@ -2355,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
        auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
                getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
        for (const auto &behavior : behaviors) {
+            dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
            if (!res.filtered_keywords.empty() || res.score > 0) {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
+                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
                std::string overrideId = behavior.getId();
                if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
                    m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
@ -2375,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
            }
            if (behavior == action_ignore)
            {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
+                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
                std::string overrideId = behavior.getId();
                if (!overrideId.empty()) {
                    m_matchedOverrideIds.insert(overrideId);
--- a/components/security_apps/waap/waap_clib/Waf2Engine.h
+++ b/components/security_apps/waap/waap_clib/Waf2Engine.h
@ -41,7 +41,6 @@
 #include "i_waap_telemetry.h"
 #include "i_deepAnalyzer.h"
 #include "i_time_get.h"
-#include "i_waap_model_result_logger.h"
 #include "table_opaque.h"
 #include "WaapResponseInspectReasons.h"
 #include "WaapResponseInjectReasons.h"
--- a/components/security_apps/waap/waap_clib/Waf2Util.cc
+++ b/components/security_apps/waap/waap_clib/Waf2Util.cc
@ -35,6 +35,7 @@
 #include "user_identifiers_config.h"
 #include "Waf2Regex.h"
 #include "ParserBinaryFile.h"
+#include "ParserKnownBenignSkipper.h"

 using boost::algorithm::to_lower_copy;
 using namespace std;
@ -1218,21 +1219,21 @@ static const SingleRegex csp_report_policy_re(
    "csp_report_policy"
 );
 static const SingleRegex base64_key_value_detector_re(
-        "^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
-        err,
-        "base64_key_value");
+    "^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
+    err,
+    "base64_key_value");
 static const SingleRegex json_key_value_detector_re(
    "\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
-        err,
-        "json_key_value");
+    err,
+    "json_key_value");
 static const SingleRegex base64_key_detector_re(
-        "^[^<>{};,&\\?|=\\s]+={1}",
-        err,
-        "base64_key");
+    "^[^<>{};,&\\?|=\\s]+={1}",
+    err,
+    "base64_key");
 static const SingleRegex base64_prefix_detector_re(
-        "data:\\S*;base64,\\S+|base64,\\S+",
-        err,
-        "base64_prefix");
+    "data:\\S*;base64,\\S+|base64,\\S+",
+    err,
+    "base64_prefix");

 // looks for combination <param>={<some text>*:<some text>*}
 //used to allow parsing param=JSON to reduce false positives
--- a/components/security_apps/waap/waap_clib/Waf2Util.h
+++ b/components/security_apps/waap/waap_clib/Waf2Util.h
@ -894,6 +894,16 @@ namespace Util {

    bool isValidJson(const std::string &input);

+    enum KnownSourceType {
+        SOURCE_TYPE_UNKNOWN = 0,
+        SOURCE_TYPE_SENSOR_DATA = 1
+    };
+
+    KnownSourceType detectKnownSource(const std::string &input);
+    bool isScreenedJson(const std::string &input);
+
+    int definePrefixedJson(const std::string &input);
+
    bool detectJSONasParameter(const std::string &s,
            std::string &key,
            std::string &value);
--- a/components/security_apps/waap/waap_clib/Waf2Util2.cc
+++ b/components/security_apps/waap/waap_clib/Waf2Util2.cc
@ -1,5 +1,7 @@
 #include "Waf2Util.h"
+#include "Waf2Regex.h"
 #include <string>
+#include "debug.h"

 namespace Waap {
 namespace Util {
@ -628,5 +630,52 @@ isValidJson(const std::string &input)
    return false;
 }

+KnownSourceType
+detectKnownSource(const std::string &input)
+{
+    static bool err = false;
+    static const SingleRegex known_source_sensor_data_re(
+        "^\\{\\\"sensor_data\\\":\\\"",
+        err,
+        "known_source_sensor_data"
+    );
+    if (known_source_sensor_data_re.hasMatch(input)) {
+        return SOURCE_TYPE_SENSOR_DATA;
+    }
+    return SOURCE_TYPE_UNKNOWN;
+}
+
+int
+definePrefixedJson(const std::string &input)
+{
+    static const size_t MAX_JSON_PREFIX_LEN = 32;
+    static const size_t MIN_PARAMETER_LEN = 4;
+    if (input.size() < MIN_PARAMETER_LEN) {
+        return -1;
+    }
+
+    for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) {
+        if (input[i] == '-' && input[i+1] == '{') return i + 1;
+    }
+
+    return -1;
+}
+
+bool
+isScreenedJson(const std::string &input)
+{
+    static bool err = false;
+    static const SingleRegex screened_json_re(
+        R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])",
+        err,
+        "screened_json"
+    );
+
+    if (screened_json_re.hasMatch(input)) {
+        return true;
+    }
+    return false;
+}
+
 } // namespace Util
 } // namespace Waap
--- a/components/security_apps/waap/waap_component_impl.cc
+++ b/components/security_apps/waap/waap_component_impl.cc
@ -50,8 +50,7 @@ WaapComponent::Impl::Impl() :
    drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
    waapStateTable(NULL),
    transactionsCount(0),
-    deepAnalyzer(),
-    waapModelResultLogger()
+    deepAnalyzer()
 {
 }

@ -536,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
        verdict = drop_response.getVerdict();
    }

+    bool sould_inject_response = waf2Transaction.shouldInjectResponse();
+    // in Chunked transfer encoding the last chunk is always empty - and we leave it empty
+    bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0;
+    dbgTrace(D_WAAP)
+        << (sould_inject_response ? "should Inject Response" : "should not Inject Response")
+        << (should_stay_empty_chunk ? " empty last chunk will stay empty" : "");
    if (verdict == pending_response.getVerdict() &&
-        waf2Transaction.shouldInjectResponse() &&
-        !event.isLastChunk()
+        sould_inject_response &&
+        !should_stay_empty_chunk
    ) {
        // Inject if needed. Note that this is only reasonable to do if there was no DROP decision above

--- a/components/security_apps/waap/waap_component_impl.h
+++ b/components/security_apps/waap/waap_component_impl.h
@ -19,7 +19,6 @@
 #include "table_opaque.h"
 #include "i_transaction.h"
 #include "waap_clib/DeepAnalyzer.h"
-#include "waap_clib/WaapModelResultLogger.h"
 #include "waap_clib/WaapAssetState.h"
 #include "waap_clib/WaapAssetStatesManager.h"
 #include "reputation_features_agg.h"
@ -81,7 +80,6 @@ private:
    uint64_t transactionsCount;
    // instance of singleton classes
    DeepAnalyzer deepAnalyzer;
-    WaapModelResultLogger waapModelResultLogger;
    WaapAssetStatesManager waapAssetStatesManager;
    std::unordered_set<std::string> m_seen_assets_id;
 };
--- a/components/utils/generic_rulebase/match_query.cc
+++ b/components/utils/generic_rulebase/match_query.cc
@ -111,7 +111,7 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
                    is_specific_label = false;
                }
            }
-            is_ignore_keyword = (key == "indicator");
+            is_ignore_keyword = (key == "indicator" || key == "keyword");

            if (condition_type != Conditions::Exist) {
                archive_in(cereal::make_nvp("value", value));
@ -244,9 +244,10 @@ MatchQuery::getAllKeys() const
 bool
 MatchQuery::matchAttributes(
        const unordered_map<string, set<string>> &key_value_pairs,
-        set<string> &matched_override_keywords ) const
+        set<string> &matched_override_keywords) const
 {

+    dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes";
    if (type == MatchType::Condition) {
        auto key_value_pair = key_value_pairs.find(key);
        if (key_value_pair == key_value_pairs.end()) {
@ -257,9 +258,11 @@ MatchQuery::matchAttributes(
    } else if (type == MatchType::Operator && operator_type == Operators::And) {
        for (const MatchQuery &inner_match: items) {
            if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator";
                return false;
            }
        }
+        dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator";
        return true;
    } else if (type == MatchType::Operator && operator_type == Operators::Or) {
        // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
@ -272,6 +275,7 @@ MatchQuery::matchAttributes(
                res = true;
            }
        }
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res;
        return res;
    } else {
        dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type";
@ -285,6 +289,7 @@ MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs)
    MatchQuery::MatchResult matches;
    matches.matched_keywords = make_shared<set<string>>();
    matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
+    dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match;
    return matches;
 }

@ -306,10 +311,13 @@ MatchQuery::matchAttributes(

    if (isKeyTypeIp()) {
        match = matchAttributesIp(values);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for IP address: " << match;
    } else if (isRegEx()) {
        match = matchAttributesRegEx(values, matched_override_keywords);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for regex: " << match;
    } else {
        match = matchAttributesString(values);
+        dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match;
    }

    return negate ? !match : match;
@ -324,6 +332,8 @@ MatchQuery::matchAttributesRegEx(
    boost::cmatch value_matcher;
    for (const boost::regex &val_regex : regex_values) {
        for (const string &requested_match_value : values) {
+            dbgTrace(D_RULEBASE_CONFIG) << "Matching value: '" << requested_match_value
+                                    << "' with regex: '" << val_regex << "'";
            if (NGEN::Regex::regexMatch(
                __FILE__,
                __LINE__,
--- a/components/utils/generic_rulebase/parameters_config.cc
+++ b/components/utils/generic_rulebase/parameters_config.cc
@ -119,10 +119,13 @@ ParameterException::getBehavior(
    for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
        MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
        if (match_res.is_match) {
-            dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches.";
+            dbgTrace(D_RULEBASE_CONFIG)
+                << "Successfully matched an exception from a list of matches, behavior: "
+                << match_behavior_pair.behavior.getId();
            // When matching indicators with action=ignore, we expect no behavior override.
            // Instead, a matched keywords list should be returned which will be later removed from score calculation
            if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore";
                matched_override_keywords.insert(match_res.matched_keywords->begin(),
                        match_res.matched_keywords->end());
            } else {
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@ -37,7 +37,7 @@ target_link_libraries(
    ngen_core
    -Wl,-whole-archive
    "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest"
-    "config;intelligence_is_v2;event_is;memory_consumption;connkey"
+    "compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey"
    "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities"
    "report_messaging"
    -Wl,-no-whole-archive
--- a/core/agent_details_reporter/agent_details_reporter.cc
+++ b/core/agent_details_reporter/agent_details_reporter.cc
@ -117,7 +117,7 @@ private:
                return;
            }

-            dbgInfo(D_AGENT_DETAILS)
+            dbgDebug(D_AGENT_DETAILS)
                << "Successfully handled attributes persistence. Operation: "
                << operation
                << ", Path "
--- a/core/include/services_sdk/interfaces/messaging/http_response.h
+++ b/core/include/services_sdk/interfaces/messaging/http_response.h
@ -31,15 +31,26 @@ public:
    HTTPResponse() = default;
 // LCOV_EXCL_STOP

-    HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {}
+    HTTPResponse(
+        HTTPStatusCode _status_code,
+        const std::string &_body,
+        std::unordered_map<std::string, std::string> _headers = std::unordered_map<std::string, std::string>()
+    )
+            :
+        status_code(_status_code),
+        body(_body),
+        headers(_headers)
+    {}

    HTTPStatusCode getHTTPStatusCode() const;
    const std::string & getBody() const;
    std::string toString() const;
+    Maybe<std::string> getHeaderVal(const std::string &header_key);

 private:
    HTTPStatusCode status_code;
    std::string body;
+    std::unordered_map<std::string, std::string> headers;
 };

 #endif // __HTTP_RESPONSE_H__
--- a/core/include/services_sdk/interfaces/messaging/messaging_enums.h
+++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h
@ -63,6 +63,7 @@ enum class HTTPStatusCode
    HTTP_PROXY_AUTHENTICATION_REQUIRED = 407,
    HTTP_REQUEST_TIME_OUT = 408,
    HTTP_PAYLOAD_TOO_LARGE = 413,
+    HTTP_TOO_MANY_REQUESTS = 429,
    // 5xx - Server error responses.
    HTTP_INTERNAL_SERVER_ERROR = 500,
    HTTP_NOT_IMPLEMENTED = 501,
--- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h
+++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h
@ -8,6 +8,7 @@
 #include "config.h"
 #include "singleton.h"
 #include "i_agent_details.h"
+#include "i_time_get.h"

 class MessageProxySettings
 {
@ -54,7 +55,7 @@ private:
    uint16_t proxy_port = 0;
 };

-class MessageMetadata
+class MessageMetadata : Singleton::Consume<I_TimeGet>
 {
 public:
    inline MessageMetadata();
@ -227,6 +228,26 @@ public:
        return sni_host_name;
    }

+    void
+    setRateLimitBlock(uint block_time)
+    {
+        is_rate_limit_block = true;
+        auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+        auto current_timeout = timer->getMonotonicTime() + std::chrono::seconds(block_time);
+        rate_limit_block_time = current_timeout.count();
+    }
+
+    bool
+    isRateLimitBlock() const
+    {
+        if (is_rate_limit_block) {
+            auto timer = Singleton::Consume<I_TimeGet>::by<MessageMetadata>();
+            uint current_time = timer->getMonotonicTime().count();
+            if (current_time < rate_limit_block_time) return true;
+        }
+        return false;
+    }
+
    template <class Archive>
    void
    serialize(Archive &ar)
@ -243,7 +264,9 @@ public:
            cereal::make_nvp("is_to_fog", is_to_fog),
            cereal::make_nvp("ca_path", ca_path),
            cereal::make_nvp("client_cert_path", client_cert_path),
-            cereal::make_nvp("client_key_path", client_key_path)
+            cereal::make_nvp("client_key_path", client_key_path),
+            cereal::make_nvp("is_rate_limit_block", is_rate_limit_block),
+            cereal::make_nvp("rate_limit_block_time", rate_limit_block_time)
        );
    }

@ -262,6 +285,8 @@ private:
    std::string external_certificate = "";
    bool should_buffer = false;
    bool is_to_fog = false;
+    bool is_rate_limit_block = false;
+    uint rate_limit_block_time = 0;
 };

 #endif // __MESSAGING_METADATA_H__
--- a/core/include/services_sdk/resources/debug_flags.h
+++ b/core/include/services_sdk/resources/debug_flags.h
@ -107,6 +107,8 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
            DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER)
            DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER)
            DEFINE_FLAG(D_WAAP_PARSER_BINARY_FILE, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER, D_WAAP_PARSER)
+            DEFINE_FLAG(D_WAAP_PARSER_SCREENED_JSON, D_WAAP_PARSER)

    DEFINE_FLAG(D_IPS, D_COMPONENT)
    DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT)
--- a/core/include/services_sdk/resources/generic_metric.h
+++ b/core/include/services_sdk/resources/generic_metric.h
@ -24,6 +24,7 @@
 #include "i_mainloop.h"
 #include "i_time_get.h"
 #include "i_agent_details.h"
+#include "i_encryptor.h"
 #include "i_instance_awareness.h"
 #include "i_environment.h"
 #include "i_messaging.h"
@ -57,6 +58,7 @@ class GenericMetric
    Singleton::Consume<I_Environment>,
    Singleton::Consume<I_Messaging>,
    Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Encryptor>,
    public Listener<AllMetricEvent>
 {
 public:
--- a/core/include/services_sdk/resources/metric/metric_calc.h
+++ b/core/include/services_sdk/resources/metric/metric_calc.h
@ -23,6 +23,8 @@

 #include "report/report.h"
 #include "customized_cereal_map.h"
+#include "compression_utils.h"
+#include "i_encryptor.h"

 class GenericMetric;

@ -56,6 +58,11 @@ public:
        value(_value)
    {
        timestamp = Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltimeStr();
+        // convert timestamp to RFC 3339 format
+        std::size_t pos = timestamp.find('.');
+        if (pos != std::string::npos) {
+            timestamp = timestamp.substr(0, pos) + "Z";
+        }
        asset_id = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId();
    }

@ -118,7 +125,73 @@ public:
    }

 // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
-    std::string
+    Maybe<std::string>
+    toString() const
+    {
+        std::stringstream ss;
+        {
+            cereal::JSONOutputArchive ar(ss);
+            serialize(ar);
+        }
+        auto res = compressAndEncodeData(ss.str());
+        if (!res.ok()) {
+            return genError("Failed to compress and encode the data");
+        }
+        return res.unpack();
+    }
+// LCOV_EXCL_STOP
+
+private:
+    Maybe<std::string>
+    compressAndEncodeData(const std::string &unhandled_data) const
+    {
+        std::string data_holder = unhandled_data;
+        auto compression_stream = initCompressionStream();
+        CompressionResult compression_response = compressData(
+            compression_stream,
+            CompressionType::GZIP,
+            data_holder.size(),
+            reinterpret_cast<const unsigned char *>(data_holder.c_str()),
+            true
+        );
+        finiCompressionStream(compression_stream);
+        if (!compression_response.ok) {
+            // send log to Kibana
+            return genError("Failed to compress(gzip) data");
+        }
+
+        std::string compressed_data =
+            std::string((const char *)compression_response.output, compression_response.num_output_bytes);
+
+        auto encryptor = Singleton::Consume<I_Encryptor>::by<GenericMetric>();
+        Maybe<std::string> handled_data = encryptor->base64Encode(compressed_data);
+
+        if (compression_response.output) free(compression_response.output);
+        compression_response.output = nullptr;
+        compression_response.num_output_bytes = 0;
+        return handled_data;
+    }
+
+    std::vector<AiopsMetricData> metrics;
+};
+
+class CompressAndEncodeAIOPSMetrics
+{
+public:
+    CompressAndEncodeAIOPSMetrics(const AiopsMetricList &_aiops_metrics) : aiops_metrics(_aiops_metrics) {}
+
+    void
+    serialize(cereal::JSONOutputArchive &ar) const
+    {
+        auto metric_str = aiops_metrics.toString();
+        if (!metric_str.ok()) {
+            return;
+        }
+        ar(cereal::make_nvp("records", metric_str.unpack()));
+    }
+
+// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
+    Maybe<std::string>
    toString() const
    {
        std::stringstream ss;
@ -131,7 +204,7 @@ public:
 // LCOV_EXCL_STOP

 private:
-    std::vector<AiopsMetricData> metrics;
+    AiopsMetricList aiops_metrics;
 };

 class MetricCalc
--- a/core/include/services_sdk/resources/rest.h
+++ b/core/include/services_sdk/resources/rest.h
@ -33,6 +33,8 @@
 #include "maybe_res.h"
 #include "rest/schema_printer.h"

+static const std::string BULK_ARRAY_NAME = "bulkArray";
+
 /// @class JsonError
 /// @brief Class representing JSON parsing errors.
 ///
--- a/core/intelligence_is_v2/intelligence_comp_v2.cc
+++ b/core/intelligence_is_v2/intelligence_comp_v2.cc
@ -93,10 +93,13 @@ public:
        res << "\"name\": \"" << details->getAgentId() << "\", ";
        auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
        res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
+        res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, ";
        res << "\"dataMap\": [";
        res << stream.str();
        res << " ] }";

+        dbgTrace(D_INTELLIGENCE) << res.str();
+
        return res;
    }

@ -148,13 +151,19 @@ public:
    void
    performCallBacks(const Invalidation &invalidation, const string &registration_id) const override
    {
-        dbgDebug(D_INTELLIGENCE) << "Looking for callbacks for invalidation " << invalidation.genObject();
+        dbgTrace(D_INTELLIGENCE)
+            << "Looking for callbacks for invalidation "
+            << invalidation.genObject()
+            << " registration id: "
+            << registration_id;
        if (registration_id != "") {
            auto invalidation_cb = registration_id_to_cb.find(registration_id);
            if (invalidation_cb != registration_id_to_cb.end()) return invalidation_cb->second(invalidation);
        }
+        dbgDebug(D_INTELLIGENCE) << "Have not found callback per registration id";
+
        for (auto &registed_invalidation : callbacks) {
-            dbgTrace(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject();
+            dbgDebug(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject();
            performCallBacksImpl(invalidation, registed_invalidation.second);
        }
    }
@ -169,6 +178,7 @@ private:
        auto &registereed_invalidation = invalidation_and_cb.first;
        auto &cb = invalidation_and_cb.second;
        if (!registereed_invalidation.matches(actual_invalidation)) return;
+        dbgTrace(D_INTELLIGENCE) << "Found a matching invalidation registration, should callback";
        cb(actual_invalidation);
    }

@ -177,7 +187,7 @@ private:
    uint running_id = 0;
 };

-class ReceiveInvalidation : public ServerRest
+class SingleReceivedInvalidation : public ServerRest
 {
 public:
    void
@ -248,6 +258,29 @@ private:
    C2S_OPTIONAL_PARAM(string, invalidationType);
 };

+
+class ReceiveInvalidation : public ServerRest
+{
+public:
+
+    void
+    doCall() override
+    {
+        dbgTrace(D_INTELLIGENCE)
+            << (bulkArray.isActive() ?
+            "BULK invalidations, receiving invalidations in bulks"
+            : "error in format, expected bulk invalidations, not single");
+
+        for (SingleReceivedInvalidation &r : bulkArray.get()) {
+            r.doCall();
+        }
+        return;
+    }
+
+private:
+    C2S_LABEL_PARAM(vector<SingleReceivedInvalidation>, bulkArray, BULK_ARRAY_NAME);
+};
+
 class PagingController
 {
 public:
--- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
+++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc
@ -486,6 +486,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
    EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
    EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": \"3\" } ]"));
    EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
+    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));
+
 }

 TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
@ -529,6 +531,7 @@ TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
    ));

    EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
+    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));

    EXPECT_THAT(
        body,
@ -606,7 +609,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback)
        .setObjectType(Intelligence::ObjectType::ASSET);

    stringstream json;
-    json << invalidation2.genObject();
+    json << "[" << invalidation2.genObject() << "]";
    mock_invalidation->performRestCall(json);

    EXPECT_EQ(recieved_invalidations.size(), 1u);
@ -650,7 +653,7 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback)
        .setObjectType(Intelligence::ObjectType::ASSET);

    stringstream json;
-    json << invalidation2.genObject();
+    json << "[" << invalidation2.genObject() << "]";
    mock_invalidation->performRestCall(json);

    EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -694,7 +697,7 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling)
        .setObjectType(Intelligence::ObjectType::ASSET);

    stringstream json;
-    json << invalidation2.genObject();
+    json <<  "[" << invalidation2.genObject() << "]";
    mock_invalidation->performRestCall(json);

    EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -789,7 +792,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets)
        .setObjectType(Intelligence::ObjectType::ASSET);

    stringstream json1;
-    json1 << not_matching_invalidation.genObject();
+    json1 << "[" << not_matching_invalidation.genObject() << "]";
    mock_invalidation->performRestCall(json1);

    EXPECT_EQ(recieved_invalidations.size(), 0u);
@ -805,7 +808,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets)
        .setObjectType(Intelligence::ObjectType::ASSET);

    stringstream json2;
-    json2 << matching_invalidation.genObject();
+    json2 << "[" << matching_invalidation.genObject() << "]";
    mock_invalidation->performRestCall(json2);

    EXPECT_EQ(recieved_invalidations.size(), 1u);
@ -865,7 +868,7 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_2_registred_assets)
    auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); });

    stringstream json;
-    json << matching_invalidation.genObject();
+    json << "[" << matching_invalidation.genObject() << "]";
    mock_invalidation->performRestCall(json);

    EXPECT_EQ(recieved_invalidations.size(), 2u);
@ -927,9 +930,43 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_by_registration_id)

    string modifiedJsonString = matching_invalidation.genObject().substr(2);
    stringstream json;
-    json << "{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString;
+    json << "[{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString << "]";
    cout << json.str() << endl;
    mock_invalidation->performRestCall(json);

    EXPECT_EQ(recieved_invalidations.size(), 1u);
 }
+
+TEST_F(IntelligenceInvalidation, bulk_invalidation_callback)
+{
+    stringstream configuration;
+    configuration << "{";
+    configuration << "  \"agentSettings\":[";
+    configuration << "    {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
+    configuration << "  ],";
+    configuration << "  \"intelligence\":{";
+    configuration << "    \"local intelligence server ip\":\"127.0.0.1\",";
+    configuration << "    \"local intelligence server primary port\":9090";
+    configuration << "  }";
+    configuration << "}";
+    Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
+
+    EXPECT_CALL(
+        messaging_mock,
+        sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _)
+    ).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
+    
+    auto invalidation = Invalidation("agent_y");
+    EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
+    auto invalidation2 = Invalidation("agent2");
+    EXPECT_NE(i_intelligence->registerInvalidation(invalidation2, callback), 0);
+    auto invalidation3 = Invalidation("agent3");
+    EXPECT_NE(i_intelligence->registerInvalidation(invalidation3, callback), 0);
+    
+    dbgTrace(D_INTELLIGENCE) << "2 callbacks: ";
+
+    stringstream json3;
+    json3 << "[{\"class\":\"agent3\"},{\"class\":\"agent2\"}]";
+    mock_invalidation->performRestCall(json3);
+    EXPECT_EQ(recieved_invalidations.size(), 2u);
+}
--- a/core/messaging/include/response_parser.h
+++ b/core/messaging/include/response_parser.h
@ -25,7 +25,7 @@ private:
    bool isLegalChunkedResponse(const std::string &res);

    Maybe<HTTPStatusCode> status_code = genError("Not received");
-    Maybe<std::map<std::string, std::string>> headers = genError("Not received");
+    Maybe<std::unordered_map<std::string, std::string>> headers = genError("Not received");
    std::string body;
    std::string raw_response;
    bool error = false;
--- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
+++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc
@ -109,7 +109,7 @@ MessagingBufferComponent::Impl::init()
    encryptor = Singleton::Consume<I_Encryptor>::by<Messaging>();
    mainloop = Singleton::Consume<I_MainLoop>::by<Messaging>();
    messaging = Singleton::Consume<I_Messaging>::from<Messaging>();
-    
+
    auto sub_path = getProfileAgentSettingWithDefault<string>("nano_agent/event_buffer/", "eventBuffer.baseFolder");
    buffer_root_path = getLogFilesPathConfig() + "/" + sub_path;
    string executable_name =
@ -276,11 +276,18 @@ MessagingBufferComponent::Impl::sendMessage()
    }

    if (res == HTTPStatusCode::HTTP_SUSPEND) {
-        dbgDebug(D_MESSAGING) << "Suspended connection - sleeping for a while";
+        dbgDebug(D_MESSAGING) << "Message is Suspended - sleeping for a while";
        mainloop->yield(chrono::seconds(1));
        return true;
    }

+    if (res == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
+        dbgDebug(D_MESSAGING) << "Suspended message due to rate limit block - sleeping for a while";
+        mainloop->yield(chrono::seconds(1));
+        popMessage();
+        return true;
+    }
+
    ++curr_no_retries;
    if (curr_no_retries >= getProfileAgentSettingWithDefault<uint>(10, "eventBuffer.maxNumOfSendigRetries")) {
        dbgWarning(D_MESSAGING) << "Reached maximum number of retries - poping message";
@ -295,6 +302,12 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons
 {
    MessageMetadata message_metadata = message.getMessageMetadata();
    message_metadata.setShouldBufferMessage(false);
+
+    if (message_metadata.isRateLimitBlock()) {
+        mainloop->yield(chrono::seconds(1));
+        return HTTPStatusCode::HTTP_SUSPEND;
+    }
+
    auto res = messaging->sendSyncMessage(
        message.getMethod(),
        message.getURI(),
@ -305,6 +318,9 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons

    if (res.ok()) return HTTPStatusCode::HTTP_OK;
    if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_SUSPEND) return HTTPStatusCode::HTTP_SUSPEND;
+    if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
+        return HTTPStatusCode::HTTP_TOO_MANY_REQUESTS;
+    }
    return HTTPStatusCode::HTTP_UNKNOWN;
 }

--- a/core/messaging/messaging_comp/http_response.cc
+++ b/core/messaging/messaging_comp/http_response.cc
@ -36,6 +36,7 @@ static const map<HTTPStatusCode, string> status_code_to_string = {
    { HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED,   "407 - HTTP_PROXY_AUTHENTICATION_REQUIRED"  },
    { HTTPStatusCode::HTTP_REQUEST_TIME_OUT,                "408 - HTTP_REQUEST_TIME_OUT"               },
    { HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE,               "413 - HTTP_PAYLOAD_TOO_LARGE"              },
+    { HTTPStatusCode::HTTP_TOO_MANY_REQUESTS,               "429 - HTTP_TOO_MANY_REQUESTS"              },
    { HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR,           "500 - HTTP_INTERNAL_SERVER_ERROR"          },
    { HTTPStatusCode::HTTP_NOT_IMPLEMENTED,                 "501 - HTTP_NOT_IMPLEMENTED"                },
    { HTTPStatusCode::HTTP_BAD_GATEWAY,                     "502 - HTTP_BAD_GATEWAY"                    },
@ -63,6 +64,7 @@ static const map<int, HTTPStatusCode> num_to_status_code = {
    { 407, HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED  },
    { 408, HTTPStatusCode::HTTP_REQUEST_TIME_OUT               },
    { 413, HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE              },
+    { 429, HTTPStatusCode::HTTP_TOO_MANY_REQUESTS              },
    { 500, HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR          },
    { 501, HTTPStatusCode::HTTP_NOT_IMPLEMENTED                },
    { 502, HTTPStatusCode::HTTP_BAD_GATEWAY                    },
@ -100,6 +102,16 @@ HTTPResponse::toString() const
    return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body);
 }

+Maybe<string>
+HTTPResponse::getHeaderVal(const string &header_key)
+{
+    auto header = headers.find(header_key);
+    if (header == headers.end()) {
+        return genError("Header \'" + header_key + "\' not found.");
+    }
+    return header->second;
+}
+
 Maybe<HTTPResponse>
 HTTPResponseParser::parseData(const string &data, bool is_connect)
 {
@ -116,7 +128,7 @@ HTTPResponseParser::parseData(const string &data, bool is_connect)

    if (!handleBody(is_connect)) return genError("Response not ready!");

-    return HTTPResponse(status_code.unpack(), body);
+    return HTTPResponse(status_code.unpack(), body, headers.unpack());
 }

 static string
@ -133,7 +145,7 @@ bool
 HTTPResponseParser::handleHeaders()
 {
    stringstream ss(raw_response);
-    map<string, string> header_map;
+    unordered_map<string, string> header_map;

    while (true) {
        string header;
@ -171,7 +183,7 @@ HTTPResponseParser::getHeaderVal(const string &header_key)
    auto headers_map = headers.unpack();
    auto header = headers_map.find(header_key);
    if (header == headers_map.end()) {
-        return genError("Header\'" + header_key + "\' not found.");
+        return genError("Header \'" + header_key + "\' not found.");
    }
    return header->second;
 }
--- a/core/messaging/messaging_comp/messaging_comp.cc
+++ b/core/messaging/messaging_comp/messaging_comp.cc
@ -148,6 +148,21 @@ MessagingComp::sendMessage(
    auto response = i_conn->sendRequest(conn, *req);
    if (!response.ok()) return response.passErr();

+    auto response_data = response.unpack();
+
+    if (response_data.getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) {
+        dbgDebug(D_MESSAGING) << "Too many requests. Suspend the message";
+        auto rate_limit_metadata = message_metadata;
+        uint retry_after_sec = 60;
+        auto retry_after_header = response_data.getHeaderVal("retry-after");
+        if (retry_after_header.ok()) {
+            retry_after_sec = stoi(*retry_after_header);
+        }
+        rate_limit_metadata.setShouldBufferMessage(true);
+        rate_limit_metadata.setRateLimitBlock(retry_after_sec);
+        return suspendMessage(body, method, uri, category, rate_limit_metadata);
+    }
+
    if (is_to_fog && method == HTTPMethod::GET) fog_get_requests_cache.emplaceEntry(uri, *response);
    return response;
 }
@ -355,6 +370,15 @@ MessagingComp::suspendMessage(
    const MessageMetadata &message_metadata
 ) const
 {
+    if (message_metadata.isRateLimitBlock()) {
+        dbgInfo(D_MESSAGING) << "Rate limit block is active, message is suspended, message is buffered.";
+        i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
+        return genError<HTTPResponse>(
+            HTTPStatusCode::HTTP_TOO_MANY_REQUESTS,
+            "The connection is suspended due to rate limit block, message is buffered."
+        );
+    }
+
    if (message_metadata.shouldBufferMessage()) {
        dbgWarning(D_MESSAGING) << "Buffering message due to connection suspended";
        i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false);
--- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
+++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc
@ -276,3 +276,30 @@ TEST_F(TestMessagingComp, testSetFogConnection)
    EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn));
    EXPECT_TRUE(messaging_comp.setFogConnection(category));
 }
+
+TEST_F(TestMessagingComp, testRateLimitBlock)
+{
+    setAgentDetails();
+    string body = "test body";
+    HTTPMethod method = HTTPMethod::POST;
+    string uri = "/test-uri";
+    MessageCategory category = MessageCategory::GENERIC;
+
+    MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC);
+    Flags<MessageConnectionConfig> conn_flags;
+    conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
+    MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true);
+    Connection conn(conn_key, conn_metadata);
+
+    EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC))
+        .WillOnce(Return(conn));
+
+    unordered_map<string, string> res_headers = {{"Retry-After", "10"}};
+    HTTPResponse res(HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, "response!!", res_headers);
+    EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res));
+    auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata);
+    ASSERT_FALSE(sending_res.ok());
+    HTTPResponse http_res = sending_res.getErr();
+    EXPECT_EQ(http_res.getBody(), "The connection is suspended due to rate limit block, message is buffered.");
+    EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_TOO_MANY_REQUESTS);
+}
--- a/core/metric/generic_metric.cc
+++ b/core/metric/generic_metric.cc
@ -32,7 +32,10 @@ MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricM
 MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }

 // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
-static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); }
+static ostream & operator<<(ostream &os, const CompressAndEncodeAIOPSMetrics &metrics)
+{
+    return os << metrics.toString();
+}
 // LCOV_EXCL_STOP

 vector<AiopsMetricData>
@ -44,8 +47,8 @@ MetricCalc::getAiopsMetrics() const
    string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName();
    string units = getMetircUnits();
    string description = getMetircDescription();
-    string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter";
-    
+    string type = getMetricType() == MetricType::GAUGE ? "Gauge" : "Counter";
+
    return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) };
 }

@ -320,17 +323,17 @@ class PrometheusRest : public ClientRest
    public:
        Metric(const string &n, const string &t, const string &d, const string &l, const string &v)
                :
-            name(n),
-            type(t),
-            description(d),
+            metric_name(n),
+            metric_type(t),
+            metric_description(d),
            labels(l),
            value(v)
        {}

    private:
-        C2S_PARAM(string, name);
-        C2S_PARAM(string, type);
-        C2S_PARAM(string, description);
+        C2S_PARAM(string, metric_name);
+        C2S_PARAM(string, metric_type);
+        C2S_PARAM(string, metric_description);
        C2S_PARAM(string, labels);
        C2S_PARAM(string, value);
    };
@ -356,6 +359,7 @@ void
 GenericMetric::generatePrometheus()
 {
    if (!getProfileAgentSettingWithDefault(false, "prometheus")) return;
+    dbgTrace(D_METRICS) << "Generate prometheus metric";

    vector<PrometheusData> all_metrics;
    for (auto &calc : calcs) {
@ -371,7 +375,7 @@ GenericMetric::generatePrometheus()
    new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
    Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessage(
        HTTPMethod::POST,
-        "/set-prometheus-data",
+        "/add-metrics",
        rest,
        MessageCategory::GENERIC,
        new_config_req_md
@ -382,6 +386,7 @@ void
 GenericMetric::generateAiopsLog()
 {
    if (!getConfigurationWithDefault<bool>(true, "metric", "aiopsMetricSendEnable")) return;
+    dbgTrace(D_METRICS) << "Generate AIOPS metric";

    AiopsMetricList aiops_metrics;

@ -397,18 +402,17 @@ GenericMetric::generateAiopsLog()
        Level::LOG,
        LogLevel::INFO,
        audience,
-        team,
+        ReportIS::AudienceTeam::HORIZON_TELEMETRY,
        Severity::INFO,
        Priority::LOW,
        report_interval,
        LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
        tags,
        Tags::INFORMATIONAL,
-        issuing_engine
+        ReportIS::IssuingEngine::HORIZON_TELEMETRY_METRICS
    );

-    metric_to_fog << LogField("eventObject", aiops_metrics);
-    
+    metric_to_fog << LogField("eventObject", CompressAndEncodeAIOPSMetrics(aiops_metrics));
    LogRest metric_client_rest(metric_to_fog);
    sendLog(metric_client_rest);
 }
--- a/core/metric/metric_ut/metric_ut.cc
+++ b/core/metric/metric_ut/metric_ut.cc
@ -9,6 +9,7 @@
 #include "mock/mock_time_get.h"
 #include "mock/mock_rest_api.h"
 #include "agent_details.h"
+#include "mock/mock_encryptor.h"
 #include "mock/mock_messaging.h"
 #include "mock/mock_instance_awareness.h"
 #include "config.h"
@ -219,6 +220,7 @@ public:
    ::Environment env;
    ConfigComponent conf;
    AgentDetails agent_details;
+    StrictMock<MockEncryptor> mock_encryptor;
    NiceMock<MockMessaging> messaging_mock;
    stringstream debug_output;
    I_MainLoop::Routine routine;
@ -556,7 +558,7 @@ TEST_F(MetricTest, printPromeathus)
    cpu_event.notify();

    string message_body;
-    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
+    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
        .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
    routine();

@ -564,44 +566,44 @@ TEST_F(MetricTest, printPromeathus)
        "{\n"
        "    \"metrics\": [\n"
        "        {\n"
-        "            \"name\": \"cpuMax\",\n"
-        "            \"type\": \"gauge\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuMax\",\n"
+        "            \"metric_type\": \"gauge\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"89\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"cpuMin\",\n"
-        "            \"type\": \"gauge\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuMin\",\n"
+        "            \"metric_type\": \"gauge\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"89\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"cpuAvg\",\n"
-        "            \"type\": \"gauge\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuAvg\",\n"
+        "            \"metric_type\": \"gauge\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"89\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"cpuCurrent\",\n"
-        "            \"type\": \"gauge\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuCurrent\",\n"
+        "            \"metric_type\": \"gauge\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"89\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"cpuCounter\",\n"
-        "            \"type\": \"gauge\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuCounter\",\n"
+        "            \"metric_type\": \"gauge\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"1\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"cpuTotalCounter\",\n"
-        "            \"type\": \"counter\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"cpuTotalCounter\",\n"
+        "            \"metric_type\": \"counter\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n"
        "            \"value\": \"1\"\n"
        "        }\n"
@ -636,7 +638,7 @@ TEST_F(MetricTest, printPromeathusMultiMap)
    HttpTransaction("/index.html", "POST", 40).notify();

    string message_body;
-    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _))
+    EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _))
        .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse())));
    routine();

@ -644,25 +646,25 @@ TEST_F(MetricTest, printPromeathusMultiMap)
        "{\n"
        "    \"metrics\": [\n"
        "        {\n"
-        "            \"name\": \"request.total\",\n"
-        "            \"type\": \"counter\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"request.total\",\n"
+        "            \"metric_type\": \"counter\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
                                "method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n"
        "            \"value\": \"1\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"request.total\",\n"
-        "            \"type\": \"counter\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"request.total\",\n"
+        "            \"metric_type\": \"counter\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
                                "method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n"
        "            \"value\": \"1\"\n"
        "        },\n"
        "        {\n"
-        "            \"name\": \"request.total\",\n"
-        "            \"type\": \"counter\",\n"
-        "            \"description\": \"\",\n"
+        "            \"metric_name\": \"request.total\",\n"
+        "            \"metric_type\": \"counter\",\n"
+        "            \"metric_description\": \"\",\n"
        "            \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\","
                                "method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n"
        "            \"value\": \"1\"\n"
@ -1396,6 +1398,8 @@ TEST_F(MetricTest, testManyValuesOutOfOrder)
 TEST_F(MetricTest, basicAIOPSMetricTest)
 {
    EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087")));
+    EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload"));
+
    CPUMetric cpu_mt;
    cpu_mt.init(
        "CPU usage",
@ -1452,14 +1456,14 @@ TEST_F(MetricTest, basicAIOPSMetricTest)
        "        \"eventLevel\": \"Log\",\n"
        "        \"eventLogLevel\": \"info\",\n"
        "        \"eventAudience\": \"Internal\",\n"
-        "        \"eventAudienceTeam\": \"Agent Core\",\n"
+        "        \"eventAudienceTeam\": \"\",\n"
        "        \"eventFrequency\": 5,\n"
        "        \"eventTags\": [\n"
        "            \"Informational\"\n"
        "        ],\n"
        "        \"eventSource\": {\n"
        "            \"agentId\": \"Unknown\",\n"
-        "            \"issuingEngine\": \"Agent Core\",\n"
+        "            \"issuingEngine\": \"horizonTelemetryMetrics\",\n"
        "            \"eventTraceId\": \"\",\n"
        "            \"eventSpanId\": \"\",\n"
        "            \"issuingEngineVersion\": \"\",\n"
@ -1469,92 +1473,7 @@ TEST_F(MetricTest, basicAIOPSMetricTest)
        "        },\n"
        "        \"eventData\": {\n"
        "            \"eventObject\": {\n"
-        "                \"Metrics\": [\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpu.max\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"percrnt\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 89.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpuMin\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 89.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpuAvg\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 89.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpuCurrent\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 89.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpuCounter\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 1.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"cpuTotalCounter\",\n"
-        "                        \"MetricType\": \"counter\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 1.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {},\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    }\n"
-        "                ]\n"
+        "                \"records\": \"compress and encode metric payload\"\n"
        "            }\n"
        "        }\n"
        "    }\n"
@ -1595,8 +1514,68 @@ TEST_F(MetricTest, testAIOPSMapMetric)
        _,
        _
    )).WillRepeatedly(SaveArg<2>(&message_body));
+    EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload"));
    routine();

+    // aiops data example
+    //     "                \"Metrics\": [\n"
+    //     "                    {\n"
+    //     "                        \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
+    //     "                        \"MetricName\": \"/index.html\",\n"
+    //     "                        \"MetricType\": \"Gauge\",\n"
+    //     "                        \"MetricUnit\": \"\",\n"
+    //     "                        \"MetricDescription\": \"\",\n"
+    //     "                        \"MetricValue\": 0.0,\n"
+    //     "                        \"ResourceAttributes\": {},\n"
+    //     "                        \"MetricAttributes\": {\n"
+    //     "                            \"key1\": \"value1\",\n"
+    //     "                            \"key2\": \"value2\"\n"
+    //     "                        },\n"
+    //     "                        \"AssetID\": \"Unknown\"\n"
+    //     "                    },\n"
+    //     "                    {\n"
+    //     "                        \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
+    //     "                        \"MetricName\": \"/index2.html\",\n"
+    //     "                        \"MetricType\": \"Gauge\",\n"
+    //     "                        \"MetricUnit\": \"\",\n"
+    //     "                        \"MetricDescription\": \"\",\n"
+    //     "                        \"MetricValue\": 0.0,\n"
+    //     "                        \"ResourceAttributes\": {},\n"
+    //     "                        \"MetricAttributes\": {\n"
+    //     "                            \"key1\": \"value1\",\n"
+    //     "                            \"key2\": \"value2\"\n"
+    //     "                        },\n"
+    //     "                        \"AssetID\": \"Unknown\"\n"
+    //     "                    },\n"
+    //     "                    {\n"
+    //     "                        \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
+    //     "                        \"MetricName\": \"/index.html\",\n"
+    //     "                        \"MetricType\": \"Counter\",\n"
+    //     "                        \"MetricUnit\": \"\",\n"
+    //     "                        \"MetricDescription\": \"\",\n"
+    //     "                        \"MetricValue\": 0.0,\n"
+    //     "                        \"ResourceAttributes\": {},\n"
+    //     "                        \"MetricAttributes\": {},\n"
+    //     "                        \"AssetID\": \"Unknown\"\n"
+    //     "                    },\n"
+    //     "                    {\n"
+    //     "                        \"Timestamp\": \"2016-11-13T17:31:24Z\",\n"
+    //     "                        \"MetricName\": \"/index2.html\",\n"
+    //     "                        \"MetricType\": \"Counter\",\n"
+    //     "                        \"MetricUnit\": \"\",\n"
+    //     "                        \"MetricDescription\": \"\",\n"
+    //     "                        \"MetricValue\": 0.0,\n"
+    //     "                        \"ResourceAttributes\": {},\n"
+    //     "                        \"MetricAttributes\": {},\n"
+    //     "                        \"AssetID\": \"Unknown\"\n"
+    //     "                    }\n"
+    //     "                ]\n"
+    //     "            }\n"
+    //     "        }\n"
+    //     "    }\n"
+    //     "}";
+
+
    string expected_message =
        "{\n"
        "    \"log\": {\n"
@ -1608,14 +1587,14 @@ TEST_F(MetricTest, testAIOPSMapMetric)
        "        \"eventLevel\": \"Log\",\n"
        "        \"eventLogLevel\": \"info\",\n"
        "        \"eventAudience\": \"Internal\",\n"
-        "        \"eventAudienceTeam\": \"Agent Core\",\n"
+        "        \"eventAudienceTeam\": \"\",\n"
        "        \"eventFrequency\": 5,\n"
        "        \"eventTags\": [\n"
        "            \"Informational\"\n"
        "        ],\n"
        "        \"eventSource\": {\n"
        "            \"agentId\": \"Unknown\",\n"
-        "            \"issuingEngine\": \"Agent Core\",\n"
+        "            \"issuingEngine\": \"horizonTelemetryMetrics\",\n"
        "            \"eventTraceId\": \"\",\n"
        "            \"eventSpanId\": \"\",\n"
        "            \"issuingEngineVersion\": \"\",\n"
@ -1625,72 +1604,7 @@ TEST_F(MetricTest, testAIOPSMapMetric)
        "        },\n"
        "        \"eventData\": {\n"
        "            \"eventObject\": {\n"
-        "                \"Metrics\": [\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"/index.html\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 25.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {\n"
-        "                            \"url\": \"/index.html\"\n"
-        "                        },\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"/index2.html\",\n"
-        "                        \"MetricType\": \"gauge\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 20.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {\n"
-        "                            \"url\": \"/index2.html\"\n"
-        "                        },\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"/index.html\",\n"
-        "                        \"MetricType\": \"counter\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 2.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {\n"
-        "                            \"url\": \"/index.html\"\n"
-        "                        },\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    },\n"
-        "                    {\n"
-        "                        \"Timestamp\": \"2016-11-13T17:31:24.087\",\n"
-        "                        \"MetricName\": \"/index2.html\",\n"
-        "                        \"MetricType\": \"counter\",\n"
-        "                        \"MetricUnit\": \"\",\n"
-        "                        \"MetricDescription\": \"\",\n"
-        "                        \"MetricValue\": 1.0,\n"
-        "                        \"ResourceAttributes\": {\n"
-        "                            \"agent\": \"Unknown\",\n"
-        "                            \"id\": \"87\"\n"
-        "                        },\n"
-        "                        \"MetricAttributes\": {\n"
-        "                            \"url\": \"/index2.html\"\n"
-        "                        },\n"
-        "                        \"AssetID\": \"Unknown\"\n"
-        "                    }\n"
-        "                ]\n"
+        "                \"records\": \"compress and encode metric payload\"\n"
        "            }\n"
        "        }\n"
        "    }\n"
--- a/core/rest/rest.cc
+++ b/core/rest/rest.cc
@ -33,6 +33,18 @@ ServerRest::performRestCall(istream &in)
 {
    try {
        try {
+            int firstChar = in.peek();
+            if (firstChar != EOF) {
+                // array as root is not supported in JSONInputArchive format
+                // but this struct is in bulk invalidations requirements
+                // we will change the format here
+                if (firstChar == '[') {
+                    std::string content((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
+                    std::string modifiedContent = "{\"" + BULK_ARRAY_NAME + "\": " + content + "}";
+                    std::istringstream* modifiedStream = new std::istringstream(modifiedContent);
+                    in.rdbuf(modifiedStream->rdbuf());
+                }
+            }
            cereal::JSONInputArchive in_ar(in);
            load(in_ar);
        } catch (cereal::Exception &e) {
--- a/nodes/orchestration/package/open-appsec-cloud-mgmt
+++ b/nodes/orchestration/package/open-appsec-cloud-mgmt
@ -58,6 +58,7 @@ send_notification_to_the_fog()
        "$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \
        --header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \
        --header "Content-Type: application/json" \
+        --header "x-rate-limit-product-type: openappsec" \
        --data "{\"logs\": [{\"log\": {\"eventTime\": \
        \"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\",\"eventSeverity\": \
        \"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",
@ -73,7 +74,9 @@ send_notification_to_the_fog()
        upload_res=$(curl -s -o /dev/null -s -w "%{http_code}\n" \
            --request POST "$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \
            --header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \
-            --header "Content-Type: application/json" --data "{\"logs\": \
+            --header "Content-Type: application/json" \
+            --header "x-rate-limit-product-type: openappsec" \
+            --data "{\"logs\": \
            [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\",
            \"eventSeverity\": \"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",
            \"eventLevel\": \"Log\",\"eventAudience\": \"Internal\",\"eventAudienceTeam\": \"Agent Core\",\"eventFrequency\": 0,
--- a/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s
+++ b/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s
@ -157,7 +157,9 @@ send_notification_to_the_fog()
    DATE=$(date "+%FT%T.000")
    upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \
        -H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \
-        --header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \
+        --header "user-agent: Infinity Next (a7030abf93a4c13)" \
+        --header "Content-Type: application/json" \
+        --header "x-rate-limit-product-type: openappsec" \
        --data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \
        \"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \
        \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\"eventAudience\": \
@ -171,7 +173,9 @@ send_notification_to_the_fog()
        sleep 5
        upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \
            -H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \
-            --header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \
+            --header "user-agent: Infinity Next (a7030abf93a4c13)" \
+            --header "Content-Type: application/json" \
+            --header "x-rate-limit-product-type: openappsec" \
            --data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \
            \"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \
            \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\
--- a/nodes/orchestration/package/orchestration_package.sh
+++ b/nodes/orchestration/package/orchestration_package.sh
@ -924,6 +924,8 @@ install_orchestration()
        cp_exec "ln -sf /storage/nano_agent${USR_LIB_PATH}/cpnano ${USR_LIB_PATH}/cpnano"
        cp_exec "mkdir -p /storage/nano_agent/${FILESYSTEM_PATH}"
        cp_exec "ln -sf /storage/nano_agent/${FILESYSTEM_PATH} ${FILESYSTEM_PATH}"
+        cp_copy smb_egg/nano-egg-internal /opt/fw1/bin/nano-egg-internal
+        chmod +x  /opt/fw1/bin/nano-egg-internal
    fi
    ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/
    ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/
--- a/unit_test.cmake
+++ b/unit_test.cmake
@ -2,7 +2,7 @@ enable_testing()

 function(add_unit_test ut_name ut_sources use_libs)
    add_executable(${ut_name} ${ut_sources})
-    target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)
+    target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)

    add_test(NAME ${ut_name}
        COMMAND ${ut_name}