Update docker-compose.yaml

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>

components/attachment-intakers/nginx_attachment/nginx_attachment.cc

@@ -36,6 +36,7 @@
 
 #include "nginx_attachment_config.h"
 #include "nginx_attachment_opaque.h"
+#include "generic_rulebase/evaluators/trigger_eval.h"
 #include "nginx_parser.h"
 #include "i_instance_awareness.h"
 #include "common.h"
@@ -130,6 +131,7 @@ class NginxAttachment::Impl
     Singleton::Provide<I_StaticResourcesHandler>::From<NginxAttachment>
 {
     static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
+    static constexpr auto LIMIT_RESPONSE_HEADERS = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS;
     static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
     static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
     static constexpr auto INJECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
@@ -1146,10 +1148,18 @@ private:
     handleCustomWebResponse(
         SharedMemoryIPC *ipc,
         vector<const char *> &verdict_data,
-        vector<uint16_t> &verdict_data_sizes)
+        vector<uint16_t> &verdict_data_sizes,
+        string web_user_response_id)
     {
         ngx_http_cp_web_response_data_t web_response_data;
-
+        ScopedContext ctx;
+        if (web_user_response_id != "") {
+            dbgTrace(D_NGINX_ATTACHMENT)
+            << "web user response ID registered in contex: "
+            << web_user_response_id;
+            set<string> triggers_set{web_user_response_id};
+            ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
+        }
         WebTriggerConf web_trigger_conf = getConfigurationWithDefault<WebTriggerConf>(
             WebTriggerConf::default_trigger_conf,
             "rulebase",
@@ -1271,7 +1281,7 @@ private:
         if (verdict.getVerdict() == DROP) {
             nginx_attachment_event.addTrafficVerdictCounter(nginxAttachmentEvent::trafficVerdict::DROP);
             verdict_to_send.modification_count = 1;
-            return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes);
+            return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, verdict.getWebUserResponseID());
         }
 
         if (verdict.getVerdict() == ACCEPT) {
@@ -1497,11 +1507,17 @@ private:
         opaque.activateContext();
 
         FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data, opaque);
-
         bool is_header =
             *chunked_data_type == ChunkType::REQUEST_HEADER  ||
             *chunked_data_type == ChunkType::RESPONSE_HEADER ||
             *chunked_data_type == ChunkType::CONTENT_LENGTH;
+
+        if (verdict.getVerdict() == LIMIT_RESPONSE_HEADERS) {
+            handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header);
+            popData(attachment_ipc);
+            verdict = FilterVerdict(INSPECT);
+        }
+
         handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header);
 
         bool is_final_verdict = verdict.getVerdict() == ACCEPT ||
@@ -1614,6 +1630,8 @@ private:
                 return "INJECT";
             case INSPECT:
                 return "INSPECT";
+            case LIMIT_RESPONSE_HEADERS:
+                return "LIMIT_RESPONSE_HEADERS";
             case IRRELEVANT:
                 return "IRRELEVANT";
             case RECONF:

components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc

@@ -70,6 +70,12 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da
         ctx.registerValue(HttpTransactionData::uri_query_decoded, decoded_url.substr(question_mark_location + 1));
     }
     ctx.registerValue(HttpTransactionData::uri_path_decoded, decoded_url);
+
+    // Register waf_tag from transaction data if available
+    const std::string& waf_tag = transaction_data.getWafTag();
+    if (!waf_tag.empty()) {
+        ctx.registerValue(HttpTransactionData::waf_tag_ctx, waf_tag);
+    }
 }
 
 NginxAttachmentOpaque::~NginxAttachmentOpaque()

components/attachment-intakers/nginx_attachment/nginx_parser.cc

@@ -28,7 +28,6 @@ USE_DEBUG_FLAG(D_NGINX_ATTACHMENT_PARSER);
 
 Buffer NginxParser::tenant_header_key = Buffer();
 static const Buffer proxy_ip_header_key("X-Forwarded-For", 15, Buffer::MemoryType::STATIC);
-static const Buffer waf_tag_key("x-waf-tag", 9, Buffer::MemoryType::STATIC);
 static const Buffer source_ip("sourceip", 8, Buffer::MemoryType::STATIC);
 bool is_keep_alive_ctx = getenv("SAAS_KEEP_ALIVE_HDR_NAME") != nullptr;
 
@@ -244,8 +243,6 @@ NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set<string>
             opaque.setSessionTenantAndProfile(active_tenant_and_profile[0], active_tenant_and_profile[1]);
         } else if (proxy_ip_header_key == header_key) {
             source_identifiers.setXFFValuesToOpaqueCtx(header, UsersAllIdentifiersConfig::ExtractType::PROXYIP);
-        } else if (waf_tag_key == header_key) {
-            source_identifiers.setWafTagValuesToOpaqueCtx(header);
         }
     }
 
@@ -382,12 +379,15 @@ NginxParser::parseResponseBody(const Buffer &raw_response_body, CompressionStrea
 Maybe<CompressionType>
 NginxParser::parseContentEncoding(const vector<HttpHeader> &headers)
 {
-    static const Buffer content_encoding_header_key("Content-Encoding");
+    dbgFlow(D_NGINX_ATTACHMENT_PARSER) << "Parsing \"Content-Encoding\" header";
+    static const Buffer content_encoding_header_key("content-encoding");
 
     auto it = find_if(
         headers.begin(),
         headers.end(),
-        [&] (const HttpHeader &http_header) { return http_header.getKey() == content_encoding_header_key; }
+        [&] (const HttpHeader &http_header) {
+            return http_header.getKey().isEqualLowerCase(content_encoding_header_key);
+        }
     );
     if (it == headers.end()) {
         dbgTrace(D_NGINX_ATTACHMENT_PARSER)

components/gradual_deployment/gradual_deployment.cc

@@ -142,7 +142,7 @@ private:
         if (temp_params_list.size() == 1) {
             Maybe<IPAddr> maybe_ip = IPAddr::createIPAddr(temp_params_list[0]);
             if (!maybe_ip.ok()) return genError("Could not create IP address, " + maybe_ip.getErr());
-            IpAddress addr = move(ConvertToIpAddress(maybe_ip.unpackMove()));
+            IpAddress addr = ConvertToIpAddress(maybe_ip.unpackMove());
 
             return move(IPRange{.start = addr, .end = addr});
         }
@@ -157,11 +157,11 @@ private:
             IPAddr max_addr = maybe_ip_max.unpackMove();
             if (min_addr > max_addr) return genError("Could not create ip range - start greater then end");
 
-            IpAddress addr_min = move(ConvertToIpAddress(move(min_addr)));
-            IpAddress addr_max = move(ConvertToIpAddress(move(max_addr)));
+            IpAddress addr_min = ConvertToIpAddress(move(min_addr));
+            IpAddress addr_max = ConvertToIpAddress(move(max_addr));
             if (addr_max.ip_type != addr_min.ip_type) return genError("Range IP's type does not match");
 
-            return move(IPRange{.start = move(addr_min), .end = move(addr_max)});
+            return IPRange{.start = move(addr_min), .end = move(addr_max)};
         }
 
         return genError("Illegal range received: " + range);

components/http_manager/http_manager.cc

@@ -37,6 +37,7 @@ operator<<(ostream &os, const EventVerdict &event)
 {
     switch (event.getVerdict()) {
         case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: return os << "Inspect";
+        case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers";
         case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: return os << "Accept";
         case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: return os << "Drop";
         case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: return os << "Inject";
@@ -93,13 +94,14 @@ public:
         ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER);
 
         HttpManagerOpaque &state = i_transaction_table->getState<HttpManagerOpaque>();
-        string event_key = static_cast<string>(event.getKey());
-
-        if (event_key == getProfileAgentSettingWithDefault<string>("", "agent.customHeaderValueLogging")) {
+        
+        const auto &custom_header = getProfileAgentSettingWithDefault<string>("", "agent.customHeaderValueLogging");
+        
+        if (event.getKey().isEqualLowerCase(custom_header)) {
             string event_value = static_cast<string>(event.getValue());
             dbgTrace(D_HTTP_MANAGER)
                 << "Found header key and value - ("
-                << event_key
+                << custom_header
                 << ": "
                 << event_value
                 << ") that matched agent settings";
@@ -195,7 +197,6 @@ public:
         if (state.getUserDefinedValue().ok()) {
             ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA);
         }
-
         return handleEvent(EndRequestEvent().performNamedQuery());
     }
 
@@ -323,8 +324,9 @@ private:
                 << respond.second.getVerdict();
 
             state.setApplicationVerdict(respond.first, respond.second.getVerdict());
+            state.setApplicationWebResponse(respond.first, respond.second.getWebUserResponseByPractice());
         }
-        FilterVerdict aggregated_verdict = state.getCurrVerdict();
+        FilterVerdict aggregated_verdict(state.getCurrVerdict(), state.getCurrWebUserResponse());
         if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
             SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify();
         }

components/http_manager/http_manager_opaque.cc

@@ -32,6 +32,13 @@ HttpManagerOpaque::setApplicationVerdict(const string &app_name, ngx_http_cp_ver
     applications_verdicts[app_name] = verdict;
 }
 
+void
+HttpManagerOpaque::setApplicationWebResponse(const string &app_name, string web_user_response_id)
+{
+    dbgTrace(D_HTTP_MANAGER) << "Security app: " << app_name << ", has web user response: " << web_user_response_id;
+    applications_web_user_response[app_name] = web_user_response_id;
+}
+
 ngx_http_cp_verdict_e
 HttpManagerOpaque::getApplicationsVerdict(const string &app_name) const
 {
@@ -51,8 +58,12 @@ HttpManagerOpaque::getCurrVerdict() const
     for (const auto &app_verdic_pair : applications_verdicts) {
         switch (app_verdic_pair.second) {
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP:
+                dbgTrace(D_HTTP_MANAGER) << "Verdict DROP for app: " << app_verdic_pair.first;
+                current_web_user_response = applications_web_user_response.at(app_verdic_pair.first);
+                dbgTrace(D_HTTP_MANAGER) << "current_web_user_response=" << current_web_user_response;
                 return app_verdic_pair.second;
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT:
+                // Sent in ResponseHeaders and ResponseBody.
                 verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
                 break;
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT:
@@ -60,11 +71,16 @@ HttpManagerOpaque::getCurrVerdict() const
                 break;
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT:
                 break;
+            case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS:
+                // Sent in End Request.
+                verdict = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS;
+                break;
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT:
                 dbgTrace(D_HTTP_MANAGER) << "Verdict 'Irrelevant' is not yet supported. Returning Accept";
                 accepted_apps++;
                 break;
             case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT:
+                // Sent in Request Headers and Request Body.
                 verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
                 break;
             default:

components/http_manager/http_manager_opaque.h

@@ -28,10 +28,12 @@ public:
     HttpManagerOpaque();
 
     void setApplicationVerdict(const std::string &app_name, ngx_http_cp_verdict_e verdict);
+    void setApplicationWebResponse(const std::string &app_name, std::string web_user_response_id);
     ngx_http_cp_verdict_e getApplicationsVerdict(const std::string &app_name) const;
     void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; }
     ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; }
     ngx_http_cp_verdict_e getCurrVerdict() const;
+    const std::string & getCurrWebUserResponse() const { return current_web_user_response; };
     std::set<std::string> getCurrentDropVerdictCausers() const;
     void saveCurrentDataToCache(const Buffer &full_data);
     void setUserDefinedValue(const std::string &value) { user_defined_value = value; }
@@ -52,6 +54,8 @@ public:
 
 private:
     std::unordered_map<std::string, ngx_http_cp_verdict_e> applications_verdicts;
+    std::unordered_map<std::string, std::string> applications_web_user_response;
+    mutable std::string current_web_user_response;
     ngx_http_cp_verdict_e manager_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
     Buffer prev_data_cache;
     uint aggregated_payload_size = 0;

components/include/generic_rulebase/triggers_config.h

@@ -317,12 +317,12 @@ public:
     {
         return url_for_cef;
     }
+    Flags<ReportIS::StreamType> getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const;
+    Flags<ReportIS::Enreachments> getEnrechments(SecurityType security_type) const;
 
 private:
     ReportIS::Severity getSeverity(bool is_action_drop_or_prevent) const;
     ReportIS::Priority getPriority(bool is_action_drop_or_prevent) const;
-    Flags<ReportIS::StreamType> getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const;
-    Flags<ReportIS::Enreachments> getEnrechments(SecurityType security_type) const;
 
     std::string name;
     std::string verbosity;
@@ -339,4 +339,32 @@ private:
     bool should_format_output = false;
 };
 
+class ReportTriggerConf
+{
+public:
+    /// \brief Default constructor for ReportTriggerConf.
+    ReportTriggerConf() {}
+
+    /// \brief Preload function to register expected configuration.
+    static void
+    preload()
+    {
+        registerExpectedConfiguration<ReportTriggerConf>("rulebase", "report");
+    }
+
+    /// \brief Load function to deserialize configuration from JSONInputArchive.
+    /// \param archive_in The JSON input archive.
+    void load(cereal::JSONInputArchive &archive_in);
+
+    /// \brief Get the name.
+    /// \return The name.
+    const std::string &
+    getName() const
+    {
+        return name;
+    }
+private:
+    std::string name;
+};
+
 #endif //__TRIGGERS_CONFIG_H__

components/include/http_event_impl/filter_verdict.h

@@ -27,9 +27,18 @@ public:
         verdict(_verdict)
     {}
 
+    FilterVerdict(
+        ngx_http_cp_verdict_e _verdict,
+        const std::string &_web_reponse_id)
+            :
+        verdict(_verdict),
+        web_user_response_id(_web_reponse_id)
+    {}
+
     FilterVerdict(const EventVerdict &_verdict, ModifiedChunkIndex _event_idx = -1)
             :
-        verdict(_verdict.getVerdict())
+        verdict(_verdict.getVerdict()),
+        web_user_response_id(_verdict.getWebUserResponseByPractice())
     {
         if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) {
             addModifications(_verdict.getModifications(), _event_idx);
@@ -59,10 +68,12 @@ public:
     uint getModificationsAmount() const { return total_modifications; }
     ngx_http_cp_verdict_e getVerdict() const { return verdict; }
     const std::vector<EventModifications> & getModifications() const { return modifications; }
+    const std::string getWebUserResponseID() const { return web_user_response_id; }
 
 private:
     ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
     std::vector<EventModifications> modifications;
+    std::string web_user_response_id;
     uint total_modifications = 0;
 };
 

components/include/http_event_impl/i_http_event_impl.h

@@ -376,16 +376,31 @@ public:
         verdict(event_verdict)
     {}
 
+    EventVerdict(
+        const ModificationList &mods,
+        ngx_http_cp_verdict_e event_verdict,
+        std::string response_id) :
+        modifications(mods),
+        verdict(event_verdict),
+        webUserResponseByPractice(response_id)
+    {}
+
 // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists
     template <typename T> void serialize(T &ar, uint) { ar(verdict); }
 // LCOV_EXCL_STOP
 
     const ModificationList & getModifications() const { return modifications; }
     ngx_http_cp_verdict_e getVerdict() const { return verdict; }
+    const std::string getWebUserResponseByPractice() const { return webUserResponseByPractice; }
+    void setWebUserResponseByPractice(const std::string id) {
+        dbgTrace(D_HTTP_MANAGER) << "current verdict web user response set to: " << id;
+        webUserResponseByPractice = id;
+    }
 
 private:
     ModificationList modifications;
     ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
+    std::string webUserResponseByPractice;
 };
 
 #endif // __I_HTTP_EVENT_IMPL_H__

components/include/http_transaction_data.h

@@ -72,7 +72,8 @@ public:
             parsed_uri,
             client_ip,
             client_port,
-            response_content_encoding
+            response_content_encoding,
+            waf_tag
         );
     }
 
@@ -91,7 +92,8 @@ public:
             parsed_uri,
             client_ip,
             client_port,
-            response_content_encoding
+            response_content_encoding,
+            waf_tag
         );
     }
 // LCOV_EXCL_STOP
@@ -122,6 +124,9 @@ public:
         response_content_encoding = _response_content_encoding;
     }
 
+    const std::string & getWafTag() const { return waf_tag; }
+    void setWafTag(const std::string &_waf_tag) { waf_tag = _waf_tag; }
+
     static const std::string http_proto_ctx;
     static const std::string method_ctx;
     static const std::string host_name_ctx;
@@ -154,6 +159,7 @@ private:
     uint16_t client_port;
     bool is_request;
     CompressionType response_content_encoding;
+    std::string waf_tag;
 };
 
 #endif // __HTTP_TRANSACTION_DATA_H__

components/include/i_details_resolver.h

@@ -26,6 +26,7 @@ public:
     virtual Maybe<std::string> getArch() = 0;
     virtual std::string getAgentVersion() = 0;
     virtual bool isKernelVersion3OrHigher() = 0;
+    virtual bool isGw() = 0;
     virtual bool isGwNotVsx() = 0;
     virtual bool isVersionAboveR8110() = 0;
     virtual bool isReverseProxy() = 0;

components/include/i_waap_telemetry.h

@@ -27,6 +27,7 @@ struct DecisionTelemetryData
     int responseCode;
     uint64_t elapsedTime;
     std::set<std::string> attackTypes;
+    bool temperatureDetected;
 
     DecisionTelemetryData() :
         blockType(NOT_BLOCKING),
@@ -38,7 +39,8 @@ struct DecisionTelemetryData
         method(POST),
         responseCode(0),
         elapsedTime(0),
-        attackTypes()
+        attackTypes(),
+        temperatureDetected(false)
     {
     }
 };

components/include/ips_comp.h

@@ -4,6 +4,7 @@
 #include "singleton.h"
 #include "i_keywords_rule.h"
 #include "i_table.h"
+#include "i_mainloop.h"
 #include "i_http_manager.h"
 #include "i_environment.h"
 #include "http_inspection_events.h"
@@ -16,7 +17,8 @@ class IPSComp
     Singleton::Consume<I_KeywordsRule>,
     Singleton::Consume<I_Table>,
     Singleton::Consume<I_Environment>,
-    Singleton::Consume<I_GenericRulebase>
+    Singleton::Consume<I_GenericRulebase>,
+    Singleton::Consume<I_MainLoop>
 {
 public:
     IPSComp();

components/include/telemetry.h

@@ -76,6 +76,20 @@ private:
     std::unordered_set<std::string> sources_seen;
 };
 
+class WaapAdditionalTrafficTelemetrics : public WaapTelemetryBase
+{
+public:
+    void updateMetrics(const std::string &asset_id, const DecisionTelemetryData &data);
+    void initMetrics();
+
+private:
+    MetricCalculations::Counter requests{this, "reservedNgenA"};
+    MetricCalculations::Counter sources{this, "reservedNgenB"};
+    MetricCalculations::Counter blocked{this, "reservedNgenC"};
+    MetricCalculations::Counter temperature_count{this, "reservedNgenD"};
+    std::unordered_set<std::string> sources_seen;
+};
+
 class WaapTrafficTelemetrics : public WaapTelemetryBase
 {
 public:
@@ -124,6 +138,7 @@ private:
     std::map<std::string, std::shared_ptr<WaapTrafficTelemetrics>> traffic_telemetries;
     std::map<std::string, std::shared_ptr<WaapAttackTypesMetrics>> attack_types;
     std::map<std::string, std::shared_ptr<WaapAttackTypesMetrics>> attack_types_telemetries;
+    std::map<std::string, std::shared_ptr<WaapAdditionalTrafficTelemetrics>> additional_traffic_telemetries;
 
     template <typename T>
     void initializeTelemetryData(

components/security_apps/http_geo_filter/http_geo_filter.cc

@@ -96,6 +96,7 @@ public:
         if (ignore_source_ip){
             dbgDebug(D_GEO_FILTER) << "Geo protection ignoring source ip: " << source_ip;
         } else {
+            dbgTrace(D_GEO_FILTER) << "Geo protection source ip: " << source_ip;
             ip_set.insert(convertIpAddrToString(maybe_source_ip.unpack()));
         }
 
@@ -335,6 +336,14 @@ private:
         ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
         I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
         EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
+        auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
+        string source_id;
+        auto maybe_source_id = env->get<std::string>(HttpTransactionData::source_identifier);
+        if (!maybe_source_id.ok()) {
+            dbgTrace(D_GEO_FILTER) << "failed to get source identifier from env";
+        } else {
+            source_id = maybe_source_id.unpack();
+        }
 
         for (const std::string& source : sources) {
 
@@ -366,11 +375,15 @@ private:
             << country_code
             << ", country name: "
             << country_name
-            << ", source ip address: "
-            << source;
+            << ", ip address: "
+            << source
+            << ", source identifier: "
+            << source_id;
+
 
             unordered_map<string, set<string>> exception_value_country_code = {
-                {"countryCode", {country_code}}
+                {"countryCode", {country_code}},
+                {"sourceIdentifier", {source_id}}
             };
             auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, geo_location_data);
             if (matched_behavior_maybe.ok()) {
@@ -382,7 +395,8 @@ private:
             }
 
             unordered_map<string, set<string>> exception_value_country_name = {
-                {"countryName", {country_name}}
+                {"countryName", {country_name}},
+                {"sourceIdentifier", {source_id}}
             };
             matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, geo_location_data);
             if (matched_behavior_maybe.ok()) {

components/security_apps/ips/include/ips_signatures.h

@@ -29,6 +29,8 @@
 #include "pm_hook.h"
 #include "i_generic_rulebase.h"
 
+#define DEFAULT_IPS_YIELD_COUNT 500
+
 /// \namespace IPSSignatureSubTypes
 /// \brief Namespace containing subtypes for IPS signatures.
 namespace IPSSignatureSubTypes
@@ -342,10 +344,17 @@ public:
         return is_loaded;
     }
 
+    static void
+    setYieldCounter(int new_yield_cnt)
+    {
+        yield_on_load_cnt = new_yield_cnt;
+    }
+
 private:
     IPSSignatureMetaData metadata;
     std::shared_ptr<BaseSignature> rule;
     bool is_loaded;
+    static int yield_on_load_cnt;
 };
 
 /// \class SignatureAndAction

components/security_apps/ips/ips_comp.cc

@@ -98,6 +98,7 @@ public:
         registerListener();
         table = Singleton::Consume<I_Table>::by<IPSComp>();
         env = Singleton::Consume<I_Environment>::by<IPSComp>();
+        updateSigsYieldCount();
     }
 
     void
@@ -307,6 +308,20 @@ public:
 
     EventVerdict respond (const EndTransactionEvent &) override { return ACCEPT; }
 
+    void
+    updateSigsYieldCount()
+    {
+        const char *ips_yield_env_str = getenv("CPNANO_IPS_LOAD_YIELD_CNT");
+        int ips_yield_default = DEFAULT_IPS_YIELD_COUNT;
+        if (ips_yield_env_str != nullptr) {
+            dbgDebug(D_IPS) << "CPNANO_IPS_LOAD_YIELD_CNT env variable is set to " << ips_yield_env_str;
+            ips_yield_default = atoi(ips_yield_env_str);
+        }
+        int yield_limit = getProfileAgentSettingWithDefault<int>(ips_yield_default, "ips.sigsYieldCnt");
+        dbgDebug(D_IPS) << "Setting IPS yield count to  " << yield_limit;
+        IPSSignatureSubTypes::CompleteSignature::setYieldCounter(yield_limit);
+    }
+
 private:
     static void setDrop(IPSEntry &state) { state.setDrop(); }
     static bool isDrop(const IPSEntry &state) { return state.isDrop(); }
@@ -373,6 +388,7 @@ IPSComp::preload()
     registerExpectedConfigFile("ips", Config::ConfigFileType::Policy);
     registerExpectedConfigFile("ips", Config::ConfigFileType::Data);
     registerExpectedConfigFile("snort", Config::ConfigFileType::Policy);
+    registerConfigLoadCb([this]() { pimpl->updateSigsYieldCount(); });
 
     ParameterException::preload();
 

components/security_apps/ips/ips_signatures.cc

@@ -45,6 +45,8 @@ static const map<string, IPSLevel> levels = {
     { "Very Low",    IPSLevel::VERY_LOW }
 };
 
+int CompleteSignature::yield_on_load_cnt = DEFAULT_IPS_YIELD_COUNT;
+
 static IPSLevel
 getLevel(const string &level_string, const string &attr_name)
 {
@@ -219,6 +221,18 @@ IPSSignatureMetaData::getYear() const
 void
 CompleteSignature::load(cereal::JSONInputArchive &ar)
 {
+    static int sigs_load_counter = 0;
+    static I_Environment *env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    static bool post_init = false;
+
+    if (!post_init) {
+        auto routine_id = Singleton::Consume<I_MainLoop>::by<IPSComp>()->getCurrentRoutineId();
+        if (routine_id.ok()) {
+            post_init = true;
+            dbgInfo(D_IPS) << "Loading signatures post init, enabling yield with limit " << yield_on_load_cnt;
+        }
+    }
+
     try {
         ar(cereal::make_nvp("protectionMetadata", metadata));
         RuleDetection rule_detection(metadata.getName());
@@ -229,6 +243,15 @@ CompleteSignature::load(cereal::JSONInputArchive &ar)
         is_loaded = false;
         dbgWarning(D_IPS) << "Failed to load signature: " << e.what();
     }
+
+    if (post_init && (yield_on_load_cnt > 0) && (++sigs_load_counter == yield_on_load_cnt)) {
+        sigs_load_counter = 0;
+        auto maybe_is_async = env->get<bool>("Is Async Config Load");
+        if (maybe_is_async.ok() && *maybe_is_async == true) {
+            dbgTrace(D_IPS) << "Yielding after " << yield_on_load_cnt << " signatures";
+            Singleton::Consume<I_MainLoop>::by<IPSComp>()->yield(false);
+        }
+    }
 }
 
 MatchType

components/security_apps/ips/ips_ut/component_ut.cc

@@ -29,6 +29,8 @@ public:
     {
         comp.preload();
         comp.init();
+        auto err = genError("not coroutine");
+        EXPECT_CALL(mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe<I_MainLoop::RoutineID>(err)));
     }
 
     ~ComponentTest()

components/security_apps/ips/ips_ut/entry_ut.cc

@@ -41,6 +41,8 @@ public:
     EntryTest()
     {
         ON_CALL(table, getState(_)).WillByDefault(Return(ptr));
+        auto err = genError("not coroutine");
+        EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe<I_MainLoop::RoutineID>(err)));
     }
 
     void

components/security_apps/ips/ips_ut/resource_ut.cc

@@ -2,6 +2,7 @@
 #include "cptest.h"
 #include "environment.h"
 #include "config_component.h"
+#include "mock/mock_mainloop.h"
 
 using namespace std;
 using namespace testing;
@@ -61,6 +62,9 @@ TEST(resources, basic_resource)
 {
     ConfigComponent conf;
     ::Environment env;
+    NiceMock<MockMainLoop> mock_mainloop;
+    auto err = genError("not coroutine");
+    EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe<I_MainLoop::RoutineID>(err)));
 
     conf.preload();
 

components/security_apps/ips/ips_ut/signatures_ut.cc

@@ -60,7 +60,12 @@ public:
     {
         IPSHelper::has_deobfuscation = true;
         generic_rulebase.preload();
+        env.preload();
+        env.init();
+
         EXPECT_CALL(logs, getCurrentLogId()).Times(AnyNumber());
+        auto err = genError("not coroutine");
+        EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe<I_MainLoop::RoutineID>(err)));
         ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state));
         {
             stringstream ss;
@@ -123,9 +128,6 @@ public:
     void
     loadExceptions()
     {
-        env.preload();
-        env.init();
-
         BasicRuleConfig::preload();
         registerExpectedConfiguration<ParameterException>("rulebase", "exception");
 
@@ -195,6 +197,7 @@ public:
     void
     load(const IPSSignaturesResource &policy, const string &severity, const string &confidence)
     {
+        Singleton::Consume<I_Environment>::from(env)->registerValue<bool>("Is Async Config Load", false);
         setResource(policy, "IPS", "protections");
         stringstream ss;
         ss << "{";

components/security_apps/layer_7_access_control/layer_7_access_control.cc

@@ -131,8 +131,12 @@ public:
     EventVerdict
     respond(const WaitTransactionEvent &) override
     {
-        dbgFlow(D_L7_ACCESS_CONTROL) << "Handling wait verdict";
+        if (!isAppEnabled()) {
+            dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled";
+            return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+        }
 
+        dbgTrace(D_L7_ACCESS_CONTROL) << "Handling wait verdict";
         return handleEvent();
     }
 

components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h

@@ -170,6 +170,7 @@ public:
         ss.str(modified_json);
         try {
             cereal::JSONInputArchive in_ar(ss);
+            in_ar(cereal::make_nvp("apiVersion", api_version));
             in_ar(cereal::make_nvp("spec", spec));
             in_ar(cereal::make_nvp("metadata", meta_data));
         } catch (cereal::Exception &e) {
@@ -191,11 +192,18 @@ public:
         return meta_data;
     }
 
+    const std::string &
+    getApiVersion() const
+    {
+        return api_version;
+    }
+
     const T & getSpec() const { return spec; }
 
 private:
     T spec;
     AppsecSpecParserMetaData meta_data;
+    std::string api_version;
 };
 
 #endif // __LOCAL_POLICY_COMMON_H__

components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc

@@ -515,17 +515,6 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
 }
 // LCOV_EXCL_STOP
 
-bool
-doesVersionExist(const map<string, string> &annotations, const string &version)
-{
-    for (auto annotation : annotations) {
-        if(annotation.second.find(version) != std::string::npos) {
-            return true;
-        }
-    }
-    return false;
-}
-
 std::tuple<Maybe<AppsecLinuxPolicy>, Maybe<V1beta2AppsecLinuxPolicy>>
 K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &ingress_mode) const
 {
@@ -534,7 +523,7 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
     );
 
     if (!maybe_appsec_policy_spec.ok() ||
-        !doesVersionExist(maybe_appsec_policy_spec.unpack().getMetaData().getAnnotations(), "v1beta1")
+        maybe_appsec_policy_spec.unpack().getApiVersion().find("v1beta1") == std::string::npos
     ) {
         try {
             std::string v1beta1_error =

components/security_apps/orchestration/details_resolver/details_resolver.cc

@@ -41,6 +41,7 @@ public:
 
     string getAgentVersion() override;
     bool isKernelVersion3OrHigher() override;
+    bool isGw() override;
     bool isGwNotVsx() override;
     bool isVersionAboveR8110() override;
     bool isReverseProxy() override;
@@ -167,6 +168,19 @@ DetailsResolver::Impl::isKernelVersion3OrHigher()
     return false;
 }
 
+bool
+DetailsResolver::Impl::isGw()
+{
+#if defined(gaia) || defined(smb)
+    static const string is_gw_cmd = "cpprod_util FwIsFirewallModule";
+    auto is_gw = DetailsResolvingHanlder::getCommandOutput(is_gw_cmd);
+    if (is_gw.ok() && !is_gw.unpack().empty()) {
+        return is_gw.unpack().front() == '1';
+    }
+#endif
+    return false;
+}
+
 bool
 DetailsResolver::Impl::isGwNotVsx()
 {
@@ -238,15 +252,21 @@ DetailsResolver::Impl::parseNginxMetadata()
         "orchestration",
         "Nginx metadata temp file"
     );
+
+    const string &filesystem_path_config = getFilesystemPathConfig();
+
     const string srcipt_exe_cmd =
-        getFilesystemPathConfig() +
+        filesystem_path_config +
         "/scripts/cp-nano-makefile-generator.sh -f -o " +
         output_path;
 
     const string script_fresh_exe_cmd =
-        getFilesystemPathConfig() +
+        filesystem_path_config +
         "/scripts/cp-nano-makefile-generator-fresh.sh save --save-location " +
-        output_path;
+        output_path +
+        " --strings_bin_path " +
+        filesystem_path_config +
+        "/bin/strings";
 
     dbgTrace(D_ORCHESTRATOR) << "Details resolver, srcipt exe cmd: " << srcipt_exe_cmd;
     if (isNoResponse("which nginx") && isNoResponse("which kong")) {

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -26,9 +26,7 @@
 Maybe<string>
 checkSAMLSupportedBlade(const string &command_output)
 {
-    // uncomment when vpn will support SAML authentication
-    // string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
-    string supportedBlades[1] = {"identityServer"};
+    string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
     for(const string &blade : supportedBlades) {
         if (command_output.find(blade) != string::npos) {
             return string("true");
@@ -49,6 +47,17 @@ checkIDABlade(const string &command_output)
     return string("false");
 }
 
+Maybe<string>
+checkVPNBlade(const string &command_output)
+{
+    string vpnBlade = "vpn";
+    if (command_output.find(vpnBlade) != string::npos) {
+        return string("true");
+    }
+
+    return string("false");
+}
+
 Maybe<string>
 checkSAMLPortal(const string &command_output)
 {
@@ -60,9 +69,9 @@ checkSAMLPortal(const string &command_output)
 }
 
 Maybe<string>
-checkPepIdaIdnStatus(const string &command_output)
+checkInfinityIdentityEnabled(const string &command_output)
 {
-    if (command_output.find("nac_pep_identity_next_enabled = 1") != string::npos) {
+    if (command_output.find("get_identities_from_infinity_identity (true)") != string::npos) {
         return string("true");
     }
     return string("false");
@@ -90,9 +99,6 @@ checkIDP(shared_ptr<istream> file_stream)
 {
     string line;
     while (getline(*file_stream, line)) {
-        if (line.find("<identity_portal/>") != string::npos) {
-            return string("false");
-        }
         if (line.find("<central_idp ") != string::npos) {
             return string("true");
         }
@@ -101,6 +107,26 @@ checkIDP(shared_ptr<istream> file_stream)
     return string("false");
 }
 
+Maybe<string>
+checkVPNCIDP(shared_ptr<istream> file_stream)
+{
+    string line;
+    while (getline(*file_stream, line)) {
+        if (line.find("<vpn") != string::npos) {
+            while (getline(*file_stream, line)) {
+                if (line.find("<central_idp ") != string::npos) {
+                    return string("true");
+                }
+                if (line.find("</vpn>") != string::npos) {
+                    break;
+                }
+            }
+        }
+    }
+
+    return string("false");
+}
+
 #endif // gaia
 
 #if defined(gaia) || defined(smb)
@@ -140,6 +166,17 @@ getIsAiopsRunning(const string &command_output)
     return command_output;
 }
 
+Maybe<string>
+getInterfaceMgmtIp(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    return genError("Eth Management IP was not found");
+}
+
+
 Maybe<string>
 checkHasSDWan(const string &command_output)
 {
@@ -451,6 +488,14 @@ extractManagements(const string &command_output)
     json_output += "]";
     return json_output;
 }
+
+Maybe<string>
+checkQosLegacyBlade(const string &command_output)
+{
+    if (command_output == "true" || command_output == "false") return command_output;
+
+    return string("false");
+}
 #endif // gaia || smb
 
 #if defined(gaia)

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -79,6 +79,14 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
 SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
     "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
     getOtlpAgentGaiaOsRole)
+SHELL_CMD_HANDLER("ETH_MGMT_IP",
+    "FS_PATH=<FILESYSTEM-PREFIX>;"
+    "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
+    "[ -z \"${VS_ID}\" ] && "
+    "(eth=\"$(grep 'management:interface' /config/active | awk '{print $2}')\" &&"
+    " ip addr show \"${eth}\" | grep inet | awk '{print $2}' | cut -d '/' -f1) || "
+    "(ip a | grep UP | grep -v lo | head -n 1 | cut -d ':' -f2 | tr -d ' ')",
+    getInterfaceMgmtIp)
 #endif
 #if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
 SHELL_CMD_HANDLER("GLOBAL_QUID",
@@ -89,6 +97,8 @@ SHELL_CMD_HANDLER("QUID",
     "cat $FWDIR/database/myown.C "
     "| awk -F'[()]' '/:name/ { found=1; next } found && /:uuid/ { uid=tolower($2); print uid; exit }'",
     getQUID)
+
+
 SHELL_CMD_HANDLER("SMO_QUID", "echo ''", getQUID)
 SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID)
 SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole)
@@ -114,12 +124,6 @@ SHELL_CMD_HANDLER(
     "jq -r .lsm_profile_uuid /tmp/cpsdwan_getdata_orch.json",
     checkLsmProfileUuid
 )
-SHELL_CMD_HANDLER(
-    "IP Address",
-    "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" "
-    "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)",
-    getGWIPAddress
-)
 SHELL_CMD_HANDLER(
     "Version",
     "cat /etc/cp-release | grep -oE 'R[0-9]+(\\.[0-9]+)?'",
@@ -138,13 +142,22 @@ SHELL_CMD_HANDLER(
     "fw ctl get int support_fec |& grep -sq \"support_fec =\";echo $?",
     getFecApplicable
 )
+SHELL_CMD_HANDLER("is_legacy_qos_blade_enabled",
+    "cpprod_util CPPROD_GetValue FG1 ProdActive 1 | grep -q '^1$' "
+    "&& (cpprod_util CPPROD_GetValue FG1 FgSDWAN 1 | grep -q '^1$' && echo false || echo true) || "
+    "echo false",
+    checkQosLegacyBlade)
 #endif //gaia || smb
 
 #if defined(gaia)
 SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade)
 SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade)
+SHELL_CMD_HANDLER("hasVPNBlade", "enabled_blades", checkVPNBlade)
 SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal)
-SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_identity_next_enabled", checkPepIdaIdnStatus)
+SHELL_CMD_HANDLER("hasInfinityIdentityEnabled",
+    "cat $FWDIR/database/myself_objects.C | grep get_identities_from_infinity_identity",
+    checkInfinityIdentityEnabled
+)
 SHELL_CMD_HANDLER("requiredNanoServices", "echo ida", getRequiredNanoServices)
 SHELL_CMD_HANDLER(
     "cpProductIntegrationMgmtObjectName",
@@ -209,6 +222,14 @@ SHELL_CMD_HANDLER(
     "echo 1",
     extractManagements
 )
+SHELL_CMD_HANDLER(
+    "IP Address",
+    "( [ $(cpprod_util FwIsHighAvail) -eq 1 ] && [ $(cpprod_util FwIsVSX) -eq 1 ]"
+    "&& (jq -r .cluster_main_ip /tmp/cpsdwan_getdata_orch.json) )"
+    "|| ( [ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" )"
+    "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)",
+    getGWIPAddress
+)
 #endif //gaia
 
 #if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
@@ -270,6 +291,17 @@ SHELL_CMD_HANDLER(
     "echo 1",
     extractManagements
 )
+SHELL_CMD_HANDLER(
+    "IP Address",
+    "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" "
+    "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)",
+    getGWIPAddress
+)
+SHELL_CMD_HANDLER(
+    "Hardware",
+    R"(ver | sed -E 's/^This is Check Point'\''s +([^ ]+).*$/\1/')",
+    getHardware
+)
 #endif//smb
 
 SHELL_CMD_OUTPUT("kernel_version", "uname -r")
@@ -287,6 +319,11 @@ FILE_CONTENT_HANDLER(
     (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml",
     checkIDP
 )
+FILE_CONTENT_HANDLER(
+    "hasVPNCidpConfigured",
+    (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml",
+    checkVPNCIDP
+)
 #endif //gaia
 
 #if defined(alpine)

components/security_apps/orchestration/downloader/https_client.cc

@@ -41,8 +41,13 @@ HTTPSClient::getFile(const URLParser &url, const string &out_file, bool auth_req
 
     if (!url.isOverSSL()) return genError("URL is not over SSL.");
 
-    if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
-    dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
+    bool skip_direct_download = (url.getQuery().find("/resources/") != string::npos);
+    if (skip_direct_download) {
+        dbgWarning(D_ORCHESTRATOR) << "Resources path: " << url.getQuery() << ". Skipping direct download.";
+    } else {
+        if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
+        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
+    }
 
     if (getFileSSL(url, out_file, token).ok()) return Maybe<void>();
     dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL).";

components/security_apps/orchestration/include/mock/mock_details_resolver.h

@@ -42,13 +42,14 @@ public:
     MOCK_METHOD0(getPlatform,                Maybe<std::string>());
     MOCK_METHOD0(getArch,                    Maybe<std::string>());
     MOCK_METHOD0(getAgentVersion,            std::string());
-    MOCK_METHOD0(isCloudStorageEnabled,        bool());
+    MOCK_METHOD0(isCloudStorageEnabled,      bool());
     MOCK_METHOD0(isReverseProxy,             bool());
     MOCK_METHOD0(isKernelVersion3OrHigher,   bool());
+    MOCK_METHOD0(isGw,                       bool());
     MOCK_METHOD0(isGwNotVsx,                 bool());
     MOCK_METHOD0(getResolvedDetails,         std::map<std::string, std::string>());
-    MOCK_METHOD0(isVersionAboveR8110, bool());
-    MOCK_METHOD0(parseNginxMetadata, Maybe<std::tuple<std::string, std::string, std::string, std::string>>());
+    MOCK_METHOD0(isVersionAboveR8110,        bool());
+    MOCK_METHOD0(parseNginxMetadata,         Maybe<std::tuple<std::string, std::string, std::string, std::string>>());
     MOCK_METHOD0(
         readCloudMetadata, Maybe<std::tuple<std::string, std::string, std::string, std::string, std::string>>());
 };

components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc

@@ -115,9 +115,9 @@ ManifestDiffCalculator::buildRecInstallationQueue(
     const map<string, Package> &current_packages,
     const map<string, Package> &new_packages)
 {
-    const vector<string> &requires = package.getRequire();
+    const vector<string> &requires_packages = package.getRequire();
 
-    for (const auto &require : requires) {
+    for (const auto &require : requires_packages) {
         auto installed_package = current_packages.find(require);
         auto new_package = new_packages.find(require);
 

components/security_apps/orchestration/orchestration_comp.cc

@@ -1471,7 +1471,8 @@ private:
             string cc_opt;
             tie(config_opt, cc_opt, nginx_version, nginx_signature) = nginx_data.unpack();
             agent_data_report
-                << make_pair("attachmentVersion", "Legacy")
+                << make_pair("configureOptStatus", "Enabled")
+                << make_pair("moduleSignatureStatus", "Enabled")
                 << make_pair("nginxSignature",    nginx_signature)
                 << make_pair("nginxVersion",      nginx_version)
                 << make_pair("configureOpt",      config_opt)
@@ -1496,6 +1497,10 @@ private:
             agent_data_report << AgentReportFieldWithLabel("isKernelVersion3OrHigher", "true");
         }
 
+        if (i_details_resolver->isGw()) {
+            agent_data_report << AgentReportFieldWithLabel("isGw", "true");
+        }
+
         if (i_details_resolver->isGwNotVsx()) {
             agent_data_report << AgentReportFieldWithLabel("isGwNotVsx", "true");
         }

components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc

@@ -150,7 +150,8 @@ getNamespaceDataFromCluster()
     string auth_header = "Authorization: Bearer " + token;
     string connection_header = "Connection: close";
     string host = "https://kubernetes.default.svc:443/api/v1/namespaces/";
-    string culr_cmd = "curl -s -k -H \"" + auth_header + "\" -H \"" + connection_header + "\" " + host +
+    string culr_cmd =
+        "LD_LIBRARY_PATH=\"\" curl -s -k -H \"" + auth_header + "\" -H \"" + connection_header + "\" " + host +
         " | /etc/cp/bin/cpnano_json";
 
     auto output_res = Singleton::Consume<I_ShellCmd>::by<OrchestrationTools>()->getExecOutput(culr_cmd);

components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc

@@ -86,7 +86,7 @@ TEST_F(OrchestrationToolsTest, setClusterId)
     EXPECT_CALL(
         mock_shell_cmd,
         getExecOutput(
-            "curl -s -k -H \"Authorization: Bearer 123\" -H \"Connection: close\" "
+            "LD_LIBRARY_PATH=\"\" curl -s -k -H \"Authorization: Bearer 123\" -H \"Connection: close\" "
             "https://kubernetes.default.svc:443/api/v1/namespaces/ | /etc/cp/bin/cpnano_json",
             200,
             false

components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc

@@ -145,6 +145,7 @@ public:
         EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64")));
         EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false));
+        EXPECT_CALL(mock_details_resolver, isGw()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx));

components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc

@@ -174,6 +174,7 @@ public:
         EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isCloudStorageEnabled()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false));
+        EXPECT_CALL(mock_details_resolver, isGw()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx));

components/security_apps/orchestration/service_controller/service_controller.cc

@@ -209,6 +209,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy
     new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
     new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
     new_config_req_md.setSuspension(false);
+    new_config_req_md.setShouldSendAccessToken(false);
     auto res = messaging->sendSyncMessage(
         HTTPMethod::POST,
         "/set-new-configuration",

components/security_apps/orchestration/update_communication/fog_authenticator.cc

@@ -139,6 +139,25 @@ FogAuthenticator::RegistrationData::serialize(JSONOutputArchive &out_ar) const
     );
 }
 
+static string
+getDeplymentType()
+{
+    auto deplyment_type = Singleton::Consume<I_EnvDetails>::by<FogAuthenticator>()->getEnvType();
+    switch (deplyment_type) {
+        case EnvType::LINUX: return "Embedded";
+        case EnvType::DOCKER: return "Docker";
+        case EnvType::NON_CRD_K8S:
+        case EnvType::K8S: return "K8S";
+        case EnvType::COUNT: break;
+    }
+
+    dbgAssertOpt(false)
+        << AlertInfo(AlertTeam::CORE, "fog communication")
+        << "Failed to get a legitimate deployment type: "
+        << static_cast<uint>(deplyment_type);
+    return "Embedded";
+}
+
 Maybe<FogAuthenticator::UserCredentials>
 FogAuthenticator::registerAgent(
     const FogAuthenticator::RegistrationData &reg_data,
@@ -208,6 +227,13 @@ FogAuthenticator::registerAgent(
 
     request << make_pair("userEdition", getUserEdition());
 
+    if (getDeplymentType() == "Docker" || getDeplymentType() == "K8S") {
+        const char *image_version_otp = getenv("IMAGE_VERSION");
+        if (image_version_otp) {
+            request << make_pair("imageVersion", image_version_otp);
+        }
+    }
+
     if (details_resolver->isReverseProxy()) {
         request << make_pair("reverse_proxy", "true");
     }
@@ -220,6 +246,10 @@ FogAuthenticator::registerAgent(
         request << make_pair("isKernelVersion3OrHigher", "true");
     }
 
+    if (details_resolver->isGw()) {
+        request << make_pair("isGw", "true");
+    }
+
     if (details_resolver->isGwNotVsx()) {
         request << make_pair("isGwNotVsx", "true");
     }
@@ -283,11 +313,14 @@ FogAuthenticator::getAccessToken(const UserCredentials &user_credentials) const
     static const string grant_type_string = "/oauth/token?grant_type=client_credentials";
     TokenRequest request = TokenRequest();
 
-    MessageMetadata request_token_md;
+    MessageMetadata request_token_md(true);
     request_token_md.insertHeader(
         "Authorization",
         buildBasicAuthHeader(user_credentials.getClientId(), user_credentials.getSharedSecret())
     );
+    dbgInfo(D_ORCHESTRATOR)
+        << "Sending request for access token. Trace: "
+        << (request_token_md.getTraceId().ok() ? request_token_md.getTraceId().unpack() : "No trace id");
     auto request_token_status = Singleton::Consume<I_Messaging>::by<FogAuthenticator>()->sendSyncMessage(
         HTTPMethod::POST,
         grant_type_string,
@@ -461,25 +494,6 @@ FogAuthenticator::getCredentialsFromFile() const
     return orchestration_tools->jsonStringToObject<UserCredentials>(encrypted_cred.unpack());
 }
 
-static string
-getDeplymentType()
-{
-    auto deplyment_type = Singleton::Consume<I_EnvDetails>::by<FogAuthenticator>()->getEnvType();
-    switch (deplyment_type) {
-        case EnvType::LINUX: return "Embedded";
-        case EnvType::DOCKER: return "Docker";
-        case EnvType::NON_CRD_K8S:
-        case EnvType::K8S: return "K8S";
-        case EnvType::COUNT: break;
-    }
-
-    dbgAssertOpt(false)
-        << AlertInfo(AlertTeam::CORE, "fog communication")
-        << "Failed to get a legitimate deployment type: "
-        << static_cast<uint>(deplyment_type);
-    return "Embedded";
-}
-
 Maybe<FogAuthenticator::UserCredentials>
 FogAuthenticator::getCredentials()
 {

components/security_apps/rate_limit/rate_limit.cc

@@ -250,13 +250,14 @@ public:
     fetchReplicaCount()
     {
         string curl_cmd =
-            "curl -H \"Authorization: Bearer " + kubernetes_token + "\" "
+            base_curl_cmd + " -H \"Authorization: Bearer " + kubernetes_token + "\" "
             "https://kubernetes.default.svc.cluster.local/apis/apps/v1/namespaces/" + kubernetes_namespace +
-            "/deployments/${AGENT_DEPLOYMENT_NAME} -k  -s | jq .status.replicas";
+            "/deployments/${AGENT_DEPLOYMENT_NAME} -k -s | jq .status.replicas";
         auto maybe_replicas = i_shell_cmd->getExecOutput(curl_cmd);
         if (maybe_replicas.ok()) {
             try {
                 replicas = std::stoi(maybe_replicas.unpack());
+                dbgTrace(D_RATE_LIMIT) << "replicas is set to " << replicas;
             } catch (const std::exception &e) {
                 dbgWarning(D_RATE_LIMIT) << "error while converting replicas: " << e.what();
             }
@@ -706,7 +707,9 @@ public:
         i_shell_cmd = Singleton::Consume<I_ShellCmd>::by<RateLimit>();
         i_env_details = Singleton::Consume<I_EnvDetails>::by<RateLimit>();
         env_type = i_env_details->getEnvType();
-        if (env_type == EnvType::K8S) {
+        const char *nexus_env = getenv("KUBERNETES_METADATA");
+        if (nexus_env == nullptr) return;
+        if (env_type == EnvType::K8S && string(nexus_env) == "true") {
             kubernetes_token = i_env_details->getToken();
             kubernetes_namespace = i_env_details->getNameSpace();
             fetchReplicaCount();
@@ -742,6 +745,13 @@ private:
     EnvType env_type;
     string kubernetes_namespace = "";
     string kubernetes_token = "";
+#if defined(gaia)
+    const string base_curl_cmd = "curl_cli";
+#elif defined(alpine)
+    const string base_curl_cmd = "LD_LIBRARY_PATH=/usr/lib/:/usr/lib/cpnano curl";
+#else
+    const string base_curl_cmd = "curl";
+#endif
 };
 
 RateLimit::RateLimit() : Component("RateLimit"), pimpl(make_unique<Impl>()) {}

components/security_apps/waap/include/i_serialize.h

@@ -23,7 +23,7 @@
 static const uint max_send_obj_retries = 3;
 static const std::chrono::microseconds wait_next_attempt(5000000);
 
-USE_DEBUG_FLAG(D_WAAP);
+USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
 
 class RestGetFile : public ClientRest
 {
@@ -151,13 +151,14 @@ protected:
         I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<WaapComponent>();
         I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
         if (agentDetails->getOrchestrationMode() == OrchestrationMode::OFFLINE) {
-            dbgDebug(D_WAAP) << "offline mode not sending object";
+            dbgDebug(D_WAAP_SERIALIZE) << "offline mode not sending object";
             return false;
         }
         if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) {
             MessageMetadata req_md(getSharedStorageHost(), 80);
             req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
             req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+            req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
             auto req_status = messaging->sendSyncMessage(
                 method,
                 uri,
@@ -166,19 +167,22 @@ protected:
                 req_md
             );
             if (!req_status.ok()) {
-                dbgWarning(D_WAAP) << "failed to send request to uri: " << uri
+                dbgWarning(D_WAAP_SERIALIZE) << "failed to send request to uri: " << uri
                     << ", error: " << req_status.getErr().toString();
             }
             return req_status.ok();
         }
+        MessageMetadata req_md;
+        req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN);
         auto req_status = messaging->sendSyncMessage(
             method,
             uri,
             obj,
-            MessageCategory::GENERIC
+            MessageCategory::GENERIC,
+            req_md
         );
         if (!req_status.ok()) {
-            dbgWarning(D_WAAP) << "failed to send request to uri: " << uri
+            dbgWarning(D_WAAP_SERIALIZE) << "failed to send request to uri: " << uri
                 << ", error: " << req_status.getErr().toString();
         }
         return req_status.ok();
@@ -192,14 +196,14 @@ protected:
         {
             if (sendObject(obj, method, uri))
             {
-                dbgTrace(D_WAAP) <<
+                dbgTrace(D_WAAP_SERIALIZE) <<
                     "object sent successfully after " << i << " retry attempts";
                 return true;
             }
-            dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i;
+            dbgInfo(D_WAAP_SERIALIZE) << "Failed to send object. Attempt: " << i;
             mainloop->yield(wait_next_attempt);
         }
-        dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
             max_send_obj_retries;
         return false;
     }
@@ -210,13 +214,14 @@ protected:
         I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<WaapComponent>();
         I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
         if (agentDetails->getOrchestrationMode() == OrchestrationMode::OFFLINE) {
-            dbgDebug(D_WAAP) << "offline mode not sending object";
+            dbgDebug(D_WAAP_SERIALIZE) << "offline mode not sending object";
             return false;
         }
         if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) {
             MessageMetadata req_md(getSharedStorageHost(), 80);
             req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
             req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+            req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
             return messaging->sendSyncMessageWithoutResponse(
                 method,
                 uri,
@@ -225,11 +230,14 @@ protected:
                 req_md
             );
         }
+        MessageMetadata req_md;
+        req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN);
         return messaging->sendSyncMessageWithoutResponse(
             method,
             uri,
             obj,
-            MessageCategory::GENERIC
+            MessageCategory::GENERIC,
+            req_md
         );
     }
 
@@ -241,14 +249,14 @@ protected:
         {
             if (sendNoReplyObject(obj, method, uri))
             {
-                dbgTrace(D_WAAP) <<
+                dbgTrace(D_WAAP_SERIALIZE) <<
                     "object sent successfully after " << i << " retry attempts";
                 return true;
             }
-            dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i;
+            dbgInfo(D_WAAP_SERIALIZE) << "Failed to send object. Attempt: " << i;
             mainloop->yield(wait_next_attempt);
         }
-        dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
             max_send_obj_retries;
         return false;
     }
@@ -257,6 +265,7 @@ protected:
     std::chrono::seconds m_interval;
     std::string m_owner;
     const std::string m_assetId;
+    bool m_remoteSyncEnabled;
 
 private:
     bool localSyncAndProcess();
@@ -272,7 +281,6 @@ private:
     size_t m_daysCount;
     size_t m_windowsCount;
     size_t m_intervalsCounter;
-    bool m_remoteSyncEnabled;
     const bool m_isAssetIdUuid;
     std::string m_type;
     std::string m_lastProcessedModified;

components/security_apps/waap/include/i_transaction.h

@@ -84,6 +84,7 @@ public:
     virtual const std::string getUri() const = 0;
     virtual const std::string getUriStr() const = 0;
     virtual const std::string& getSourceIdentifier() const = 0;
+    virtual const std::string getCurrentWebUserResponse() = 0;
     virtual double getScore() const = 0;
     virtual double getOtherModelScore() const = 0;
     virtual const std::vector<double> getScoreArray() const = 0;
@@ -130,6 +131,7 @@ public:
     virtual void add_request_body_chunk(const char* data, int data_len) = 0;
     virtual void end_request_body() = 0;
     virtual void end_request() = 0;
+    virtual bool shouldLimitResponseHeadersInspection() = 0;
     // Response
     virtual void start_response(int response_status, int http_version) = 0;
     virtual void start_response_hdrs() = 0;
@@ -145,4 +147,7 @@ public:
     virtual ReportIS::Severity computeEventSeverityFromDecision() const = 0;
     virtual void finish() = 0;
     virtual Waf2TransactionFlags &getTransactionFlags() = 0;
+
+    virtual void setTemperatureDetected(bool detected) = 0;
+    virtual bool wasTemperatureDetected() const = 0;
 };

components/security_apps/waap/include/i_waapConfig.h

@@ -26,7 +26,6 @@
 #include "../waap_clib/SecurityHeadersPolicy.h"
 #include <memory>
 
-
 enum class BlockingLevel {
     NO_BLOCKING = 0,
     LOW_BLOCKING_LEVEL,

components/security_apps/waap/waap_clib/AutonomousSecurityDecision.cc

@@ -19,7 +19,6 @@ AutonomousSecurityDecision::AutonomousSecurityDecision(DecisionType type) :
     m_fpMitigationScore(0.0f),
     m_finalScore(0.0f),
     m_threatLevel(NO_THREAT),
-    m_overridesLog(false),
     m_relativeReputationMean(0.0),
     m_variance(0.0)
 {}
@@ -52,10 +51,6 @@ void AutonomousSecurityDecision::setThreatLevel(ThreatLevel threatLevel)
     m_threatLevel = threatLevel;
 }
 
-void AutonomousSecurityDecision::setOverridesLog(bool overridesLog)
-{
-    m_overridesLog = overridesLog;
-}
 void AutonomousSecurityDecision::setRelativeReputationMean(double relativeReputationMean)
 {
     m_relativeReputationMean = relativeReputationMean;
@@ -80,10 +75,6 @@ ThreatLevel AutonomousSecurityDecision::getThreatLevel() const
 {
     return m_threatLevel;
 }
-bool AutonomousSecurityDecision::getOverridesLog() const
-{
-    return m_overridesLog;
-}
 double AutonomousSecurityDecision::getRelativeReputationMean() const
 {
     return m_relativeReputationMean;

components/security_apps/waap/waap_clib/AutonomousSecurityDecision.h

@@ -30,14 +30,12 @@ public:
     void setFpMitigationScore(double fpMitigationScore);
     void setFinalScore(double finalScore);
     void setThreatLevel(ThreatLevel threatLevel);
-    void setOverridesLog(bool overridesLog);
     void setRelativeReputationMean(double relativeReputationMean);
     void setVariance(double variance);
     double getRelativeReputation() const;
     double getFpMitigationScore() const;
     double getFinalScore() const;
     ThreatLevel getThreatLevel() const;
-    bool getOverridesLog() const;
     double getRelativeReputationMean() const;
     double getVariance() const;
 
@@ -46,7 +44,6 @@ private:
     double m_fpMitigationScore;
     double m_finalScore;
     ThreatLevel m_threatLevel;
-    bool m_overridesLog;
     double m_relativeReputationMean;
     double m_variance;
 };

components/security_apps/waap/waap_clib/ConfidenceCalculator.h

@@ -28,6 +28,8 @@
 #include "i_ignoreSources.h"
 #include "TuningDecisions.h"
 
+static constexpr size_t defaultConfidenceMemUsage = 40 * 1024 * 1024; // 40MB
+
 USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
 
 class WaapComponent;
@@ -39,9 +41,10 @@ struct ConfidenceCalculatorParams
     std::chrono::minutes intervalDuration;
     double ratioThreshold;
     bool learnPermanently;
+    size_t maxMemoryUsage;
 
     template <class Archive>
-    void serialize(Archive& ar)
+    void serialize(Archive &ar)
     {
         size_t duration = intervalDuration.count();
         ar(cereal::make_nvp("minSources", minSources),
@@ -50,10 +53,17 @@ struct ConfidenceCalculatorParams
             cereal::make_nvp("ratioThreshold", ratioThreshold),
             cereal::make_nvp("learnPermanently", learnPermanently));
         intervalDuration = std::chrono::minutes(duration);
+        try {
+            ar(cereal::make_nvp("maxMemoryUsage", maxMemoryUsage));
+        } catch (cereal::Exception &e) {
+            maxMemoryUsage = defaultConfidenceMemUsage;
+            dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "maxMemoryUsage not found in serialized data";
+            ar.setNextName(nullptr);
+        }
     }
 
-    bool operator==(const ConfidenceCalculatorParams& other);
-    friend std::ostream& operator<<(std::ostream& os, const ConfidenceCalculatorParams& ccp);
+    bool operator==(const ConfidenceCalculatorParams &other);
+    friend std::ostream & operator<<(std::ostream &os, const ConfidenceCalculatorParams &ccp);
 };
 
 class ConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase
@@ -74,7 +84,6 @@ public:
     typedef std::list<ValuesSet> ValuesList;
     typedef UMap<Key, ValuesList> WindowsConfidentValuesList;
     typedef UMap<Key, UMap<Val, double>> ConfidenceLevels;
-    typedef UMap<Key, int> WindowsCounter;
     typedef UMap<Key, ValueSetWithTime> ConfidenceSet;
 
     ConfidenceCalculator(size_t minSources,
@@ -82,19 +91,19 @@ public:
         std::chrono::minutes intervalDuration,
         double ratioThreshold,
         const Val &nullObj,
-        const std::string& backupPath,
-        const std::string& remotePath,
-        const std::string& assetId,
+        const std::string &backupPath,
+        const std::string &remotePath,
+        const std::string &assetId,
         TuningDecision* tuning = nullptr,
         I_IgnoreSources* ignoreSrc = nullptr);
 
     ~ConfidenceCalculator();
 
-    void setOwner(const std::string& owner);
+    void setOwner(const std::string &owner);
 
     void hardReset();
     void reset();
-    bool reset(ConfidenceCalculatorParams& params);
+    bool reset(ConfidenceCalculatorParams &params);
 
     virtual bool postData();
     virtual void pullData(const std::vector<std::string>& files);
@@ -103,10 +112,12 @@ public:
     virtual void pullProcessedData(const std::vector<std::string>& files);
     virtual void updateState(const std::vector<std::string>& files);
 
-    virtual void serialize(std::ostream& stream);
-    virtual void deserialize(std::istream& stream);
+    virtual void serialize(std::ostream &stream);
+    virtual void deserialize(std::istream &stream);
 
-    void mergeFromRemote(const ConfidenceSet& remote_confidence_set, bool is_first_pull);
+    Maybe<void> writeToFile(const std::string& path, const std::vector<unsigned char>& data);
+
+    void mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull);
 
     bool is_confident(const Key &key, const Val &value) const;
 
@@ -121,35 +132,50 @@ public:
 
     void calculateInterval();
 
-    static void mergeConfidenceSets(ConfidenceSet& confidence_set,
-                                    const ConfidenceSet& confidence_set_to_merge,
-                                    size_t& last_indicators_update);
+    static void mergeConfidenceSets(ConfidenceSet &confidence_set,
+                                    const ConfidenceSet &confidence_set_to_merge,
+                                    size_t &last_indicators_update);
 private:
-    void loadVer0(cereal::JSONInputArchive& archive);
-    void loadVer1(cereal::JSONInputArchive& archive);
-    void loadVer2(cereal::JSONInputArchive& archive);
-    void loadVer3(cereal::JSONInputArchive& archive);
+    void loadVer0(cereal::JSONInputArchive &archive);
+    void loadVer1(cereal::JSONInputArchive &archive);
+    void loadVer2(cereal::JSONInputArchive &archive);
+    void loadVer3(cereal::JSONInputArchive &archive);
     bool tryParseVersionBasedOnNames(
-        cereal::JSONInputArchive& archive,
+        cereal::JSONInputArchive &archive,
         const std::string &params_field_name,
         const std::string &indicators_update_field_name,
         const std::string &windows_summary_field_name,
         const std::string &confident_sets_field_name);
-    void convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList& windows);
+    void convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList &windows);
 
-    std::string getParamName(const Key& key);
-    size_t sumSourcesWeight(const SourcesSet& sources);
-    void mergeSourcesCounter(const Key& key, const SourcesCounters& counters);
-    void removeBadSources(SourcesSet& sources, const std::vector<std::string>* badSources);
+    void loadConfidenceLevels();
+    void saveConfidenceLevels(Maybe<ConfidenceCalculator::ConfidenceLevels> confidenceLevels);
+    void saveConfidenceLevels();
+
+    void saveTimeWindowLogger();
+    std::shared_ptr<KeyValSourcesLogger> loadTimeWindowLogger();
+
+    std::string getParamName(const Key &key);
+    size_t sumSourcesWeight(const SourcesSet &sources);
+    void removeBadSources(SourcesSet &sources, const std::vector<std::string>* badSources);
+
+    // Delete existing carry-on data files asynchronously with yields
+    void garbageCollector();
 
     ConfidenceCalculatorParams m_params;
     Val m_null_obj;
-    KeyValSourcesLogger m_time_window_logger;
-    KeyValSourcesLogger m_time_window_logger_backup;
+    std::shared_ptr<KeyValSourcesLogger> m_time_window_logger;
+    std::shared_ptr<KeyValSourcesLogger> m_time_window_logger_backup;
+    std::string m_path_to_backup;
     ConfidenceSet m_confident_sets;
     ConfidenceLevels m_confidence_level;
-    WindowsCounter m_windows_counter;
     size_t m_last_indicators_update;
+    size_t m_latest_index;
     I_IgnoreSources* m_ignoreSources;
     TuningDecision* m_tuning;
+    size_t m_estimated_memory_usage; // Variable to track estimated memory usage
+    size_t m_post_index;
+    I_MainLoop *m_mainLoop;
+    I_MainLoop::RoutineID m_routineId;
+    std::vector<std::string> m_filesToRemove;
 };

components/security_apps/waap/waap_clib/Csrf.cc

@@ -45,6 +45,21 @@ State::decide
     }
 
     auto csrfDecision = decision.getDecision(CSRF_DECISION);
+    auto autonomousDecision = decision.getDecision(AUTONOMOUS_SECURITY_DECISION);
+    if (autonomousDecision->shouldForceBlock())
+    {
+        dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): Autonomous decision force should block.";
+        csrfDecision->setBlock(true);
+        csrfDecision->setForceBlock(true);
+        return true;
+    }
+    if (autonomousDecision->shouldForceAllow())
+    {
+        dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): Autonomous decision force should allow.";
+        csrfDecision->setBlock(false);
+        csrfDecision->setForceAllow(true);
+        return false;
+    }
     if (csrf_token.empty())
     {
         dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): missing token.";

components/security_apps/waap/waap_clib/DecisionType.h

@@ -14,6 +14,8 @@
 #ifndef __DECISION_TYPE_H__
 #define __DECISION_TYPE_H__
 
+#include <ostream>
+
 enum DecisionType
 {
     // This order determines the priority of the decisions sent to management
@@ -28,4 +30,35 @@ enum DecisionType
     // Must be kept last
     NO_WAAP_DECISION
 };
+
+inline const char *
+decisionTypeToString(DecisionType type)
+{
+    switch (type) {
+        case DecisionType::AUTONOMOUS_SECURITY_DECISION:
+            return "AUTONOMOUS_SECURITY_DECISION";
+        case DecisionType::CSRF_DECISION:
+            return "CSRF_DECISION";
+        case DecisionType::OPEN_REDIRECT_DECISION:
+            return "OPEN_REDIRECT_DECISION";
+        case DecisionType::ERROR_DISCLOSURE_DECISION:
+            return "ERROR_DISCLOSURE_DECISION";
+        case DecisionType::ERROR_LIMITING_DECISION:
+            return "ERROR_LIMITING_DECISION";
+        case DecisionType::USER_LIMITS_DECISION:
+            return "USER_LIMITS_DECISION";
+        case DecisionType::RATE_LIMITING_DECISION:
+            return "RATE_LIMITING_DECISION";
+        case DecisionType::NO_WAAP_DECISION:
+            return "NO_WAAP_DECISION";
+        default:
+            return "INVALID_DECISION_TYPE";
+    }
+}
+
+inline std::ostream & operator<<(std::ostream& os, const DecisionType& type)
+{
+    return os << decisionTypeToString(type);
+}
+
 #endif

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -36,6 +36,7 @@
 #include "debug.h"
 #include "i_transaction.h"
 #include "agent_core_utilities.h"
+#include <boost/algorithm/string.hpp>
 
 USE_DEBUG_FLAG(D_WAAP_DEEP_PARSER);
 USE_DEBUG_FLAG(D_WAAP_ULIMITS);
@@ -93,6 +94,12 @@ DeepParser::depth() const
     return m_depth;
 }
 
+static bool err = false;
+static const SingleRegex temperature_value_re(
+    "^\\s*([0-9](?:\\.\\d+)?)\\s*$",
+    err,
+    "temperature_value");
+
 // Called when another key/value pair is ready
 int
 DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth)
@@ -195,6 +202,14 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
     bool isBodyPayload = (m_key.first().size() == 4 && m_key.first() == "body");
 
 
+    if (isBodyPayload && v_len < 32 && k_len == 11 &&
+        boost::to_lower_copy(std::string(k, k_len)) == "temperature" &&
+        temperature_value_re.hasMatch(std::string(v, v_len))) {
+        m_pTransaction->setTemperatureDetected(true);
+        dbgTrace(D_WAAP_DEEP_PARSER) << "temperature detected, value: " << std::string(v, v_len);
+    }
+
+
     // If csrf/antibot cookie - send to Waf2Transaction for collection of cookie value.
     if (m_depth == 1 && isCookiePayload && (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) {
         std::string cur_val = std::string(v, v_len);
@@ -288,6 +303,11 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
             dbgTrace(D_WAAP_DEEP_PARSER) << "removing leading '/' from URL param value";
             base64_offset = 1;
         }
+        if (m_depth == 1 && (isUrlParamPayload || isRefererParamPayload) &&
+            k_len != 0  && (v_len == 0 || (v[0] == '=' && v_len == 1))) {
+            // if the value is empty or starts with '=' - replace it with key
+            cur_val = std::string(k, k_len);
+        }
         std::string decoded_val, decoded_key;
         base64_variants base64_status = Waap::Util::b64Test(
             cur_val,
@@ -477,6 +497,19 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
         }
     }
 
+    // If this is url_paran and key is match to nosql_key_evasion_detector_re and this is 1st and last buffer
+    // than add to beginning of cur_val "<key>=" where key is the key
+    if (flags == BUFFERED_RECEIVER_F_BOTH) {
+        std::string key = std::string(k, k_len);
+        if (Waap::Util::testNoSQLKeySuspect(key)) {
+            cur_val = key + "=" + cur_val;
+            dbgTrace(D_WAAP_DEEP_PARSER)
+                << "DeepParser::onKv(): found: key = "
+                << key
+                << " is a candidate for NoSQL key evasion - sending to updated string for scanning.";
+        }
+    }
+
     // If there's a parser in parsers stack, push the value to the top parser
     if (!m_parsersDeque.empty()
         && offset >= 0
@@ -1326,7 +1359,7 @@ DeepParser::createInternalParser(
         } else if (b64FileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE) {
             dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a known binary file, base64 encoded";
             m_parsersDeque.push_back(
-                std::make_shared<BufferedParser<ParserBinaryFile>>(*this, parser_depth + 1, true, b64FileType)
+                std::make_shared<BufferedParser<ParserBinaryFile>>(*this, parser_depth + 1, false, b64FileType)
             );
             offset = 0;
         }

components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc

@@ -89,6 +89,7 @@ bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const
             shouldFilter |= m_keywordsFreqFilter->shouldFilterKeyword(type, keyword);
         }
     }
+
     if (m_matchedOverrideKeywords.size() > 0 &&
             m_matchedOverrideKeywords.find(keyword) != m_matchedOverrideKeywords.end())
     {

components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc

@@ -86,10 +86,11 @@ bool KeywordIndicatorFilter::loadParams(std::shared_ptr<Waap::Parameters::WaapPa
         std::to_string(CONFIDENCE_THRESHOLD)));
     std::string learnPermanentlyStr = pParams->getParamVal("learnIndicators.learnPermanently", "true");
     params.learnPermanently = !boost::iequals(learnPermanentlyStr.c_str(), "false");
+    params.maxMemoryUsage = std::stoul(pParams->getParamVal("learnIndicators.maxMemoryUsage",
+        std::to_string(CONFIDENCE_MAX_MEMORY_USAGE)));
 
     std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true");
     bool syncEnabled = !boost::iequals(remoteSyncStr, "false");
-
     dbgTrace(D_WAAP) << params << " remote sync: " << remoteSyncStr;
 
     m_confidence_calc.setRemoteSyncEnabled(syncEnabled);

components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h

@@ -21,6 +21,7 @@
 #define CONFIDENCE_MIN_INTERVALS 5
 #define CONFIDENCE_THRESHOLD 0.8
 #define CONFIDENCE_WINDOW_INTERVAL std::chrono::minutes(120)
+#define CONFIDENCE_MAX_MEMORY_USAGE (40 * 1024 * 1024) // 40MB
 
 
 class KeywordIndicatorFilter : public IndicatorFilterBase

components/security_apps/waap/waap_clib/LogGenWrapper.cc

@@ -41,15 +41,16 @@ LogGenWrapper::LogGenWrapper(
     }
     else {
         m_log_gen = std::make_unique<LogGen>(
-            maybe_trigger.unpack(),
             title,
-            security_type,
+            ReportIS::Level::LOG,
+            ReportIS::Audience::SECURITY,
             severity,
             priority,
-            is_action_drop_or_prevent,
             ReportIS::Tags::WAF,
-            ReportIS::Tags::THREAT_PREVENTION
-            );
+            ReportIS::Tags::THREAT_PREVENTION,
+            maybe_trigger.unpack().getStreams(security_type, is_action_drop_or_prevent),
+            maybe_trigger.unpack().getEnrechments(security_type)
+        );
     }
 }
 

components/security_apps/waap/waap_clib/ParserBinaryFile.cc

@@ -78,7 +78,6 @@ ParserBinaryFile::detectBinaryFileHeader(const string &buf)
     return BinaryFileType::FILE_TYPE_NONE;
 }
 
-
 size_t
 ParserBinaryFile::push(const char *buf, size_t len)
 {
@@ -151,7 +150,10 @@ ParserBinaryFile::push(const char *buf, size_t len)
                 } else {
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
                     size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
-                    c = strstr(buf + tail_lookup_offset, tail.c_str());
+                    c = static_cast<const char *>(memmem(buf + tail_lookup_offset,
+                        len - tail_lookup_offset,
+                        tail.c_str(),
+                        tail.size()));
                     dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
                     if (c) {
                         m_state = s_end;

components/security_apps/waap/waap_clib/ParserUrlEncode.cc

@@ -21,7 +21,11 @@ USE_DEBUG_FLAG(D_WAAP);
 const std::string ParserUrlEncode::m_parserName = "ParserUrlEncode";
 
 ParserUrlEncode::ParserUrlEncode(
-    IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar, bool should_decode_per
+    IParserStreamReceiver &receiver,
+    size_t parser_depth,
+    char separatorChar,
+    bool should_decode_per,
+    bool should_decode_plus
 ) :
     m_receiver(receiver),
     m_state(s_start),
@@ -29,13 +33,16 @@ ParserUrlEncode::ParserUrlEncode(
     m_separatorChar(separatorChar),
     m_escapedCharCandidate(0),
     should_decode_percent(should_decode_per),
+    m_should_decode_plus(should_decode_plus),
     m_parser_depth(parser_depth)
 {
     dbgTrace(D_WAAP)
         << "should_decode_percent="
         << should_decode_per
         << "parser_depth="
-        << parser_depth;
+        << parser_depth
+        << "m_should_decode_plus="
+        << m_should_decode_plus;
 
     // TODO:: is there a need for this?
     memset(m_escaped, 0, sizeof(m_escaped));
@@ -124,7 +131,7 @@ ParserUrlEncode::push(const char *buf, size_t len)
                 }
                 m_state = s_key_escaped1;
                 break;
-                } else if (c == '+') {
+                } else if (c == '+' && m_should_decode_plus) {
                 // convert plus character to space
                 if (i - mark > 0) {
                     if (m_receiver.onKey(buf + mark, i - mark) != 0) {
@@ -281,7 +288,7 @@ ParserUrlEncode::push(const char *buf, size_t len)
                 }
                 m_state = s_value_escaped1;
                 break;
-                } else if (c == '+') {
+                } else if (c == '+' && m_should_decode_plus) {
                 // convert plus character to space
                 if (i - mark > 0) {
                     if (m_receiver.onValue(buf + mark, i - mark) != 0) {

components/security_apps/waap/waap_clib/ParserUrlEncode.h

@@ -25,7 +25,8 @@ public:
         IParserStreamReceiver &receiver,
         size_t parser_depth,
         char separatorChar = '&',
-        bool should_decode_per = true);
+        bool should_decode_per = true,
+        bool should_decode_plus = true);
     virtual ~ParserUrlEncode();
     size_t push(const char *data, size_t data_len);
     void finish();
@@ -55,6 +56,7 @@ private:
     char m_separatorChar;
     char m_escapedCharCandidate;
     bool should_decode_percent;
+    bool m_should_decode_plus;
     static const std::string m_parserName;
     size_t m_parser_depth;
 };

components/security_apps/waap/waap_clib/ParserXML.cc

@@ -170,19 +170,22 @@ ParserXML::onEntityDeclaration(
 {
     dbgTrace(D_WAAP_PARSER_XML) << "ENTITY FOUND WITH VALUE: '" << (content ? (const char*)content : "null") << "'";
 
-    ParserXML* p = (ParserXML*)ctx;
-    std::string kw = "08a80340-06d3-11ea-9f87-0242ac11000f";
+    if (systmeid != nullptr) {
+        dbgTrace(D_WAAP_PARSER_XML) << "ENTITY FOUND WITH SYSTEM ID: '" << (const char*)systmeid << "'";
+        ParserXML* p = (ParserXML*)ctx;
+        std::string kw = "08a80340-06d3-11ea-9f87-0242ac11000f";
 
-    if (p->m_receiver.onKey(p->m_key.c_str(), p->m_key.size()) != 0) {
-        p->m_state = s_error;
-    }
+        if (p->m_receiver.onKey(p->m_key.c_str(), p->m_key.size()) != 0) {
+            p->m_state = s_error;
+        }
 
-    if (p->m_receiver.onValue(kw.data(), kw.size()) != 0) {
-        p->m_state = s_error;
-    }
+        if (p->m_receiver.onValue(kw.data(), kw.size()) != 0) {
+            p->m_state = s_error;
+        }
 
-    if (p->m_receiver.onKvDone() != 0) {
-        p->m_state = s_error; // error
+        if (p->m_receiver.onKvDone() != 0) {
+            p->m_state = s_error; // error
+        }
     }
 }
 

components/security_apps/waap/waap_clib/RequestsMonitor.cc

@@ -18,9 +18,11 @@ SourcesRequestMonitor::SourcesRequestMonitor(
         filePath,
         remotePath != "" ? remotePath + "/Monitor" : remotePath,
         assetId,
-        owner
-    ), m_sourcesRequests()
+        owner),
+    m_sourcesRequests(),
+    m_enabled(false)
 {
+    m_enabled = getProfileAgentSettingWithDefault<bool>(false, "appsec.sourceRequestsMonitor.enabled");
 }
 
 SourcesRequestMonitor::~SourcesRequestMonitor()
@@ -35,17 +37,18 @@ void SourcesRequestMonitor::syncWorker()
     OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
         Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
 
-    bool enabled = getProfileAgentSettingWithDefault<bool>(false, "appsec.sourceRequestsMonitor.enabled");
+    m_enabled = getProfileAgentSettingWithDefault<bool>(false, "appsec.sourceRequestsMonitor.enabled");
 
-    if (mode == OrchestrationMode::OFFLINE || !enabled || isBase() || !postData()) {
+    if (mode == OrchestrationMode::OFFLINE || !m_enabled || isBase() || !postData()) {
         dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR)
             << "Did not report data. for asset: "
             << m_assetId
             << " Remote URL: "
             << m_remotePath
             << " is enabled: "
-            << to_string(enabled)
+            << to_string(m_enabled)
             << ", mode: " << int(mode);
+        m_sourcesRequests.clear();
         return;
     }
 
@@ -72,6 +75,9 @@ void SourcesRequestMonitor::syncWorker()
 
 void SourcesRequestMonitor::logSourceHit(const string& source)
 {
+    if (!m_enabled) {
+        return;
+    }
     m_sourcesRequests[chrono::duration_cast<chrono::minutes>(
         Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime()
     ).count()][source]++;

components/security_apps/waap/waap_clib/RequestsMonitor.h

@@ -28,6 +28,7 @@ protected:
 private:
     // map of sources and their requests per minute (UNIX)
     MonitorData m_sourcesRequests;
+    bool m_enabled;
 };
 
 #endif // __REQUESTS_MONITOR_H__

components/security_apps/waap/waap_clib/Serializator.cc

@@ -29,7 +29,7 @@
 #include "compression_utils.h"
 #include "config.h"
 
-USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
+USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
 
 namespace ch = std::chrono;
 using namespace std;
@@ -52,6 +52,22 @@ isGZipped(const string &stream)
     return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b;
 }
 
+void yieldIfPossible(const string& func, int line)
+{
+    // Check if we are in the main loop
+    if (Singleton::exists<I_MainLoop>() &&
+        Singleton::Consume<I_MainLoop>::by<WaapComponent>()->getCurrentRoutineId().ok())
+    {
+        // If we are not in the main loop, yield to allow other routines to run
+        // This is important for the main loop to be able to process other events
+        // and avoid blocking the entire system.
+        dbgTrace(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line;
+        Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
+    }
+}
+
+#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__)
+
 bool RestGetFile::loadJson(const string& json)
 {
     string json_str;
@@ -61,6 +77,9 @@ bool RestGetFile::loadJson(const string& json)
     {
         return ClientRest::loadJson(json_str);
     }
+    YIELD_IF_POSSIBLE();
+    dbgTrace(D_WAAP_SERIALIZE) << "before decompression in loadJson, data size: "
+        << json_str.size() << " bytes";
     auto compression_stream = initCompressionStream();
     DecompressionResult res = decompressData(
         compression_stream,
@@ -75,33 +94,81 @@ bool RestGetFile::loadJson(const string& json)
     }
 
     finiCompressionStream(compression_stream);
+    YIELD_IF_POSSIBLE();
+    dbgTrace(D_WAAP_SERIALIZE) << "Yielded after decompression in loadJson, decompressed size: "
+        << json_str.size() << " bytes";
+
     return ClientRest::loadJson(json_str);
 }
 
 Maybe<string> RestGetFile::genJson() const
 {
     Maybe<string> json = ClientRest::genJson();
+    YIELD_IF_POSSIBLE();
+
     if (json.ok())
     {
         string data = json.unpack();
+
+        // Get chunk size from profile settings for compression chunks
+        const size_t COMPRESSED_CHUNK_SIZE = static_cast<size_t>(
+            getProfileAgentSettingWithDefault<uint>(64 * 1024, "appsecLearningSettings.compressionChunkSize"));
+
         auto compression_stream = initCompressionStream();
-        CompressionResult res = compressData(
-            compression_stream,
-            CompressionType::GZIP,
-            data.size(),
-            reinterpret_cast<const unsigned char *>(data.c_str()),
-            true);
+        size_t offset = 0;
+        std::vector<unsigned char> compressed_data;
+        bool ok = true;
+        size_t chunk_count = 0;
+
+        // Process data in chunks for compression
+        while (offset < data.size()) {
+            size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
+            bool is_last = (offset + chunk_size >= data.size());
+            CompressionResult chunk_res = compressData(
+                compression_stream,
+                CompressionType::GZIP,
+                static_cast<uint32_t>(chunk_size),
+                reinterpret_cast<const unsigned char *>(data.c_str() + offset),
+                is_last ? 1 : 0
+            );
+
+            if (!chunk_res.ok) {
+                ok = false;
+                break;
+            }
+
+            if (chunk_res.output && chunk_res.num_output_bytes > 0) {
+                compressed_data.insert(
+                    compressed_data.end(),
+                    chunk_res.output,
+                    chunk_res.output + chunk_res.num_output_bytes
+                );
+                free(chunk_res.output);
+                chunk_res.output = nullptr;
+            }
+
+            offset += chunk_size;
+            chunk_count++;
+            YIELD_IF_POSSIBLE();
+            dbgTrace(D_WAAP_SERIALIZE) << "Processed compression chunk " << chunk_count
+                << ", progress: " << offset << "/" << data.size() << " bytes ("
+                << (offset * 100 / data.size()) << "%) - yielded";
+        }
+
         finiCompressionStream(compression_stream);
-        if (!res.ok) {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data";
+        dbgDebug(D_WAAP_SERIALIZE) << "Yielded after finalizing compression stream. "
+            << "Total chunks: " << chunk_count << ", Compression ratio: "
+            << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x";
+
+        if (!ok) {
+            dbgWarning(D_WAAP_SERIALIZE) << "Failed to gzip data";
             return genError("Failed to compress data");
         }
-        data = string((const char *)res.output, res.num_output_bytes);
-        json = data;
 
-        if (res.output) free(res.output);
-        res.output = nullptr;
-        res.num_output_bytes = 0;
+        // Create string from compressed data
+        string compressed_str(reinterpret_cast<const char*>(compressed_data.data()), compressed_data.size());
+
+        json = compressed_str;
     }
     return json;
 }
@@ -128,16 +195,16 @@ void SerializeToFilePeriodically::backupWorker()
     I_TimeGet* timer = Singleton::Consume<I_TimeGet>::by<WaapComponent>();
     auto currentTime = timer->getMonotonicTime();
 
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: current time: " << currentTime.count();
+    dbgTrace(D_WAAP_SERIALIZE) << "backup worker: current time: " << currentTime.count();
 
     if (currentTime - m_lastSerialization >= m_interval)
     {
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: backing up data";
+        dbgTrace(D_WAAP_SERIALIZE) << "backup worker: backing up data";
         m_lastSerialization = currentTime;
         // save data
         saveData();
 
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: data is backed up";
+        dbgTrace(D_WAAP_SERIALIZE) << "backup worker: data is backed up";
     }
 }
 
@@ -153,7 +220,7 @@ void SerializeToFilePeriodically::setInterval(ch::seconds newInterval)
 
 SerializeToFileBase::SerializeToFileBase(string fileName) : m_filePath(fileName)
 {
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath
+    dbgTrace(D_WAAP_SERIALIZE) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath
         << "'";
 }
 
@@ -165,52 +232,119 @@ SerializeToFileBase::~SerializeToFileBase()
 void SerializeToFileBase::saveData()
 {
     fstream filestream;
-
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "saving to file: " << m_filePath;
+    auto maybe_routine = Singleton::Consume<I_MainLoop>::by<WaapComponent>()->getCurrentRoutineId();
+    dbgTrace(D_WAAP_SERIALIZE) << "saving to file: " << m_filePath;
     filestream.open(m_filePath, fstream::out);
 
     stringstream ss;
 
     if (filestream.is_open() == false) {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to open file: " << m_filePath << " Error: "
+        dbgWarning(D_WAAP_SERIALIZE) << "failed to open file: " << m_filePath << " Error: "
             << strerror(errno);
         return;
     }
-
+    if (maybe_routine.ok()) {
+        Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
+    }
     serialize(ss);
 
+    if (maybe_routine.ok()) {
+        Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
+    }
     string data = ss.str();
+    dbgDebug(D_WAAP_SERIALIZE) << "Serialized data size: " << data.size() << " bytes";
+
+    // Get chunk size from profile settings, with default of 16 MiB for compression chunks
+    const size_t CHUNK_SIZE = static_cast<size_t>(
+        getProfileAgentSettingWithDefault<uint>(16 * 1024 * 1024, "appsecLearningSettings.writeChunkSize"));
+    // Get chunk size for writing compressed data, with default of 16 MiB
+    const size_t COMPRESSED_CHUNK_SIZE = static_cast<size_t>(
+        getProfileAgentSettingWithDefault<uint>(16 * 1024 * 1024, "appsecLearningSettings.compressionChunkSize"));
 
     auto compression_stream = initCompressionStream();
-    CompressionResult res = compressData(
-        compression_stream,
-        CompressionType::GZIP,
-        data.size(),
-        reinterpret_cast<const unsigned char *>(data.c_str()),
-        true
-    );
-    finiCompressionStream(compression_stream);
-    if (!res.ok) {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data";
-    } else {
-        ss.str(string((const char *)res.output, res.num_output_bytes));
-        // free the memory allocated by compressData
-        if (res.output) free(res.output);
-        res.output = nullptr;
-        res.num_output_bytes = 0;
+    size_t offset = 0;
+    std::vector<unsigned char> compressed_data;
+    bool ok = true;
+    size_t chunk_count = 0;
+
+    // Process data in chunks for compression
+    while (offset < data.size()) {
+        size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
+        bool is_last = (offset + chunk_size >= data.size());
+        CompressionResult chunk_res = compressData(
+            compression_stream,
+            CompressionType::GZIP,
+            static_cast<uint32_t>(chunk_size),
+            reinterpret_cast<const unsigned char *>(data.c_str() + offset),
+            is_last ? 1 : 0
+        );
+
+        if (!chunk_res.ok) {
+            ok = false;
+            break;
+        }
+
+        if (chunk_res.output && chunk_res.num_output_bytes > 0) {
+            compressed_data.insert(
+                compressed_data.end(),
+                chunk_res.output,
+                chunk_res.output + chunk_res.num_output_bytes
+            );
+            free(chunk_res.output);
+            chunk_res.output = nullptr;
+        }
+
+        offset += chunk_size;
+        chunk_count++;
+        if (maybe_routine.ok()) {
+            Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
+            dbgTrace(D_WAAP_SERIALIZE) << "Compression chunk " << chunk_count
+                << " processed (" << offset << "/" << data.size() << " bytes, "
+                << (offset * 100 / data.size()) << "%) - yielded";
+        }
     }
-    if (res.output) free(res.output);
-    res.output = nullptr;
-    res.num_output_bytes = 0;
+    finiCompressionStream(compression_stream);
+    dbgDebug(D_WAAP_SERIALIZE) << "Finished compression stream. "
+        << "Total chunks: " << chunk_count << ", Compression ratio: "
+        << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x";
+
+    if (!ok) {
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to compress data";
+        filestream.close();
+        return;
+    }
+
+    dbgDebug(D_WAAP_SERIALIZE) << "Compression complete: " << data.size() << " bytes -> "
+        << compressed_data.size() << " bytes (ratio: "
+        << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x)";
 
 
-    filestream << ss.str();
+    // Use compressed data directly
+    string data_to_write(reinterpret_cast<const char*>(compressed_data.data()), compressed_data.size());
+
+    // Write data to file in chunks with yield points
+    offset = 0;
+    size_t write_chunks = 0;
+
+    while (offset < data_to_write.size()) {
+        size_t current_chunk_size = std::min(CHUNK_SIZE, data_to_write.size() - offset);
+        filestream.write(data_to_write.c_str() + offset, current_chunk_size);
+        offset += current_chunk_size;
+        write_chunks++;
+        Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
+        dbgTrace(D_WAAP_SERIALIZE) << "Write chunk " << write_chunks
+            << " complete: " << offset << "/" << data_to_write.size() << " bytes ("
+            << (offset * 100 / data_to_write.size()) << "%) - yielded";
+    }
+
     filestream.close();
+    dbgDebug(D_WAAP_SERIALIZE) << "Finished writing backup file: " << m_filePath
+        << " (" << data_to_write.size() << " bytes in " << write_chunks << " chunks)";
 }
 
 string decompress(string fileContent) {
     if (!isGZipped(fileContent)) {
-        dbgTrace(D_WAAP) << "file note zipped";
+        dbgTrace(D_WAAP_SERIALIZE) << "file note zipped";
         return fileContent;
     }
     auto compression_stream = initCompressionStream();
@@ -236,13 +370,13 @@ string decompress(string fileContent) {
 
 void SerializeToFileBase::loadFromFile(string filePath)
 {
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loadFromFile() file: " << filePath;
+    dbgTrace(D_WAAP_SERIALIZE) << "loadFromFile() file: " << filePath;
     fstream filestream;
 
     filestream.open(filePath, fstream::in);
 
     if (filestream.is_open() == false) {
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to open file: " << filePath << " Error: " <<
+        dbgTrace(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " <<
             strerror(errno);
         if (!Singleton::exists<I_InstanceAwareness>() || errno != ENOENT)
         {
@@ -262,18 +396,18 @@ void SerializeToFileBase::loadFromFile(string filePath)
         if (idPosition != string::npos)
         {
             filePath.erase(idPosition, idStr.length() - 1);
-            dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "retry to load file from : " << filePath;
+            dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << filePath;
             loadFromFile(filePath);
         }
         return;
     }
 
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loading from file: " << filePath;
+    dbgTrace(D_WAAP_SERIALIZE) << "loading from file: " << filePath;
 
     int length;
     filestream.seekg(0, ios::end);    // go to the end
     length = filestream.tellg();           // report location (this is the length)
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "file length: " << length;
+    dbgTrace(D_WAAP_SERIALIZE) << "file length: " << length;
     assert(length >= 0); // length -1 really happens if filePath is a directory (!)
     char* buffer = new char[length];       // allocate memory for a buffer of appropriate dimension
     filestream.seekg(0, ios::beg);    // go back to the beginning
@@ -281,7 +415,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
     {
         filestream.close();
         delete[] buffer;
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to read file, file: " << filePath;
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to read file, file: " << filePath;
         return;
     }
     filestream.close();
@@ -298,7 +432,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
         deserialize(ss);
     }
     catch (runtime_error & e) {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to deserialize file: " << m_filePath << ", error: " <<
+        dbgWarning(D_WAAP_SERIALIZE) << "failed to deserialize file: " << m_filePath << ", error: " <<
             e.what();
     }
 }
@@ -318,11 +452,11 @@ RemoteFilesList::RemoteFilesList() : files(), filesPathsList()
 bool RemoteFilesList::loadJson(const string& xml)
 {
     xmlDocPtr doc; // the resulting document tree
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "XML input: " << xml;
+    dbgTrace(D_WAAP_SERIALIZE) << "XML input: " << xml;
     doc = xmlParseMemory(xml.c_str(), xml.length());
 
     if (doc == NULL) {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to parse " << xml;
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to parse " << xml;
         return false;
     }
 
@@ -343,7 +477,7 @@ bool RemoteFilesList::loadJson(const string& xml)
     {
         if (xmlStrEqual(contents_name, node->name) == 1)
         {
-            dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the Contents element";
+            dbgTrace(D_WAAP_SERIALIZE) << "Found the Contents element";
             xmlNodePtr contents_node = node->children;
             string file;
             string lastModified;
@@ -351,21 +485,21 @@ bool RemoteFilesList::loadJson(const string& xml)
             {
                 if (xmlStrEqual(key_name, contents_node->name) == 1)
                 {
-                    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the Key element";
+                    dbgTrace(D_WAAP_SERIALIZE) << "Found the Key element";
                     xmlChar* xml_file = xmlNodeGetContent(contents_node);
                     file = string(reinterpret_cast<const char*>(xml_file));
                     xmlFree(xml_file);
                 }
                 if (xmlStrEqual(last_modified_name, contents_node->name) == 1)
                 {
-                    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the LastModified element";
+                    dbgTrace(D_WAAP_SERIALIZE) << "Found the LastModified element";
                     xmlChar* xml_file = xmlNodeGetContent(contents_node);
                     lastModified = string(reinterpret_cast<const char*>(xml_file));
                     xmlFree(xml_file);
                 }
                 if (!file.empty() && !lastModified.empty())
                 {
-                    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Adding the file: " << file <<
+                    dbgTrace(D_WAAP_SERIALIZE) << "Adding the file: " << file <<
                         " last modified: " << lastModified;
                     break;
                 }
@@ -408,18 +542,18 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
     m_interval(0),
     m_owner(owner),
     m_assetId(replaceAllCopy(assetId, "/", "")),
+    m_remoteSyncEnabled(true),
     m_pMainLoop(nullptr),
     m_waitForSync(waitForSync),
     m_workerRoutineId(0),
     m_daysCount(0),
     m_windowsCount(0),
     m_intervalsCounter(0),
-    m_remoteSyncEnabled(true),
     m_isAssetIdUuid(Waap::Util::isUuid(assetId)),
     m_shared_storage_host(genError("not set")),
     m_learning_host(genError("not set"))
 {
-    dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId <<
+    dbgInfo(D_WAAP_SERIALIZE) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId <<
         "', owner='" << m_owner << "'";
 
     if (Singleton::exists<I_AgentDetails>() &&
@@ -429,7 +563,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
         if (sharedStorageHost != NULL) {
             m_shared_storage_host = string(sharedStorageHost);
         } else {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) <<
+            dbgWarning(D_WAAP_SERIALIZE) <<
                 "shared storage host name(" <<
                 SHARED_STORAGE_HOST_ENV_NAME <<
                 ") is not set";
@@ -438,7 +572,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
         if (learningHost != NULL) {
             m_learning_host = string(learningHost);
         } else {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) <<
+            dbgWarning(D_WAAP_SERIALIZE) <<
                 "learning host name(" <<
                 SHARED_STORAGE_HOST_ENV_NAME <<
                 ") is not set";
@@ -515,7 +649,7 @@ string SerializeToLocalAndRemoteSyncBase::getPostDataUrl()
     {
         I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
         Maybe<string> uniqueId = instance->getUniqueID();
-        if (uniqueId.ok())
+        if (uniqueId.ok() && !uniqueId.unpack().empty())
         {
             agentId += "/" + uniqueId.unpack();
         }
@@ -530,7 +664,7 @@ void SerializeToLocalAndRemoteSyncBase::setRemoteSyncEnabled(bool enabled)
 
 void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
 {
-    dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "setInterval: from " << m_interval.count() << " to " <<
+    dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " <<
         newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'";
 
     if (newInterval == m_interval)
@@ -571,7 +705,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
             if (remainingTime > m_interval) {
                 // on load between trigger and offset remaining time is larger than the interval itself
                 remainingTime -= m_interval;
-                dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "adjusting remaining time: " << remainingTime.count();
+                dbgDebug(D_WAAP_SERIALIZE) << "adjusting remaining time: " << remainingTime.count();
                 if (timeBeforeSyncWorker.count() != 0)
                 {
                     auto updateTime = timeBeforeSyncWorker - m_interval;
@@ -585,13 +719,13 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
             if (remainingTime < ch::seconds(0)) {
                 // syncWorker execution time was so large the remaining time became negative
                 remainingTime = ch::seconds(0);
-                dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "syncWorker execution time (owner='" << m_owner <<
+                dbgError(D_WAAP_SERIALIZE) << "syncWorker execution time (owner='" << m_owner <<
                     "', assetId='" << m_assetId << "') is " <<
                     ch::duration_cast<ch::seconds>(timeAfterSyncWorker - timeBeforeSyncWorker).count() <<
                     " seconds, too long to cause negative remainingTime. Waiting 0 seconds...";
             }
 
-            dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "current time: " << timeBeforeSyncWorker.count() << " \u00b5s" <<
+            dbgDebug(D_WAAP_SERIALIZE) << "current time: " << timeBeforeSyncWorker.count() << " \u00b5s" <<
                 ": assetId='" << m_assetId << "'" <<
                 ", owner='" << m_owner << "'" <<
                 ", daysCount=" << m_daysCount <<
@@ -604,7 +738,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
             m_pMainLoop->yield(remainingTime);
 
             timeBeforeSyncWorker = timer->getWalltime();
-            syncWorker();
+            m_pMainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::System, [this]() {syncWorker();}, "Sync worker");
             timeAfterSyncWorker = timer->getWalltime();
         }
     };
@@ -618,11 +752,11 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
 bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess()
 {
     bool isBackupSyncEnabled = getProfileAgentSettingWithDefault<bool>(
-        true,
+        false,
         "appsecLearningSettings.backupLocalSync");
 
     if (!isBackupSyncEnabled) {
-        dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Local sync is disabled";
+        dbgInfo(D_WAAP_SERIALIZE) << "Local sync is disabled";
         processData();
         saveData();
         return true;
@@ -630,7 +764,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess()
 
     RemoteFilesList rawDataFiles;
 
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Getting files of all agents";
+    dbgTrace(D_WAAP_SERIALIZE) << "Getting files of all agents";
 
     bool isSuccessful = sendObjectWithRetry(rawDataFiles,
         HTTPMethod::GET,
@@ -638,7 +772,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess()
 
     if (!isSuccessful)
     {
-        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files";
+        dbgError(D_WAAP_SERIALIZE) << "Failed to get the list of files";
         return false;
     }
 
@@ -662,7 +796,7 @@ void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService()
         RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
         if (remoteFiles.getFilesMetadataList().empty())
         {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "no files generated by the remote service were found";
+            dbgWarning(D_WAAP_SERIALIZE) << "no files generated by the remote service were found";
             continue;
         }
         string lastModified = remoteFiles.getFilesMetadataList().begin()->modified;
@@ -670,26 +804,26 @@ void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService()
         {
             m_lastProcessedModified = lastModified;
             updateState(remoteFiles.getFilesList());
-            dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
+            dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner <<
                 ". updated state generated by remote at " << m_lastProcessedModified;
             return;
         }
     }
-    dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "polling for update state timeout. for assetId='"
+    dbgWarning(D_WAAP_SERIALIZE) << "polling for update state timeout. for assetId='"
         << m_assetId << "', owner='" << m_owner;
     localSyncAndProcess();
 }
 
 void SerializeToLocalAndRemoteSyncBase::syncWorker()
 {
-    dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
+    dbgInfo(D_WAAP_SERIALIZE) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
         m_owner << "'" << " last modified state: " << m_lastProcessedModified;
     incrementIntervalsCount();
     OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
         Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
 
     if (mode == OrchestrationMode::OFFLINE  || !m_remoteSyncEnabled || isBase() || !postData()) {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR)
+        dbgDebug(D_WAAP_SERIALIZE)
             << "Did not synchronize the data. for asset: "
             << m_assetId
             << " Remote URL: "
@@ -702,17 +836,17 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
         return;
     }
 
-    dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data";
+    dbgTrace(D_WAAP_SERIALIZE) << "Waiting for all agents to post their data";
     waitSync();
     // check if learning service is operational
     if (m_lastProcessedModified == "")
     {
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "check if remote service is operational";
+        dbgTrace(D_WAAP_SERIALIZE) << "check if remote service is operational";
         RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
         if (!remoteFiles.getFilesMetadataList().empty())
         {
             m_lastProcessedModified = remoteFiles.getFilesMetadataList()[0].modified;
-            dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "First sync by remote service: " << m_lastProcessedModified;
+            dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service: " << m_lastProcessedModified;
         }
     }
 
@@ -721,15 +855,15 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
         true,
         "appsecLearningSettings.remoteServiceEnabled");
 
-    dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "using remote service: " << isRemoteServiceEnabled;
+    dbgDebug(D_WAAP_SERIALIZE) << "using remote service: " << isRemoteServiceEnabled;
     if ((m_lastProcessedModified == "" || !isRemoteServiceEnabled) && !localSyncAndProcess())
     {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "local sync and process failed";
+        dbgWarning(D_WAAP_SERIALIZE) << "local sync and process failed";
         return;
     }
 
     if (mode == OrchestrationMode::HYBRID) {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "detected running in standalone mode";
+        dbgDebug(D_WAAP_SERIALIZE) << "detected running in standalone mode";
         I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
         I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<WaapComponent>();
 
@@ -738,6 +872,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
         MessageMetadata req_md(getLearningHost(), 80);
         req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
         req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+        req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
         bool ok = messaging->sendSyncMessageWithoutResponse(
             HTTPMethod::POST,
             "/api/sync",
@@ -745,14 +880,14 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
             MessageCategory::GENERIC,
             req_md
         );
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sent learning sync notification ok: " << ok;
+        dbgDebug(D_WAAP_SERIALIZE) << "sent learning sync notification ok: " << ok;
         if (!ok) {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to send learning notification";
+            dbgWarning(D_WAAP_SERIALIZE) << "failed to send learning notification";
         }
     } else {
         SyncLearningNotificationObject syncNotification(m_assetId, m_type, getWindowId());
 
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sending sync notification: " << syncNotification;
+        dbgDebug(D_WAAP_SERIALIZE) << "sending sync notification: " << syncNotification;
 
         ReportMessaging(
             "sync notification for '" + m_assetId + "'",
@@ -766,6 +901,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
 
     if (m_lastProcessedModified != "" && isRemoteServiceEnabled)
     {
+        // wait for remote service to process the data
+        waitSync();
         updateStateFromRemoteService();
     }
 }
@@ -775,7 +912,7 @@ void SerializeToLocalAndRemoteSyncBase::restore()
     SerializeToFileBase::restore();
     if (!isBase())
     {
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "merge state from remote service";
+        dbgTrace(D_WAAP_SERIALIZE) << "merge state from remote service";
         mergeProcessedFromRemote();
     }
 }
@@ -789,7 +926,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList()
 
     if (!isRemoteServiceEnabled)
     {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "remote service is disabled";
+        dbgDebug(D_WAAP_SERIALIZE) << "remote service is disabled";
         return remoteFiles;
     }
 
@@ -800,7 +937,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList()
 
     if (!isSuccessful)
     {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files";
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to get the list of files";
     }
     return remoteFiles;
 }
@@ -814,12 +951,12 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
     {
         const vector<FileMetaData>& filesMD = processedFilesList.getFilesMetadataList();
         if (filesMD.size() > 1) {
-            dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "got more than 1 expected processed file";
+            dbgWarning(D_WAAP_SERIALIZE) << "got more than 1 expected processed file";
         }
         if (!filesMD.empty()) {
             m_lastProcessedModified = filesMD[0].modified;
         }
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "found " << filesMD.size() << " remote service state files. "
+        dbgTrace(D_WAAP_SERIALIZE) << "found " << filesMD.size() << " remote service state files. "
             "last modified: " << m_lastProcessedModified;
 
         return processedFilesList;
@@ -833,11 +970,11 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
 
     if (!isSuccessful)
     {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files";
+        dbgDebug(D_WAAP_SERIALIZE) << "Failed to get the list of files";
     }
     else if (!processedFilesList.getFilesList().empty())
     {
-        dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "found state files";
+        dbgTrace(D_WAAP_SERIALIZE) << "found state files";
         return processedFilesList;
     }
     // backward compatibility - try to get backup file with the buggy prefix tenantID/assetID/instanceID/
@@ -846,7 +983,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
     pos = bcRemotePath.find('/', pos + 1);
     if (!Singleton::exists<I_InstanceAwareness>())
     {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "missing instance of instance awareness,"
+        dbgDebug(D_WAAP_SERIALIZE) << "missing instance of instance awareness,"
             " can't check backward compatibility";
         return processedFilesList;
     }
@@ -854,13 +991,13 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
     Maybe<string> id = instanceAwareness->getUniqueID();
     if (!id.ok())
     {
-        dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to get instance id err: " << id.getErr() <<
+        dbgDebug(D_WAAP_SERIALIZE) << "failed to get instance id err: " << id.getErr() <<
             ". can't check backward compatibility";
         return processedFilesList;
     }
     string idStr = id.unpack();
     bcRemotePath.insert(pos + 1, idStr + "/");
-    dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "List of files is empty - trying to get the file from " <<
+    dbgDebug(D_WAAP_SERIALIZE) << "List of files is empty - trying to get the file from " <<
         bcRemotePath;
 
     isSuccessful = sendObject(
@@ -870,16 +1007,16 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
 
     if (!isSuccessful)
     {
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files";
+        dbgWarning(D_WAAP_SERIALIZE) << "Failed to get the list of files";
     }
-    dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "backwards computability: got "
+    dbgDebug(D_WAAP_SERIALIZE) << "backwards computability: got "
         << processedFilesList.getFilesList().size() << " state files";
     return processedFilesList;
 }
 
 void SerializeToLocalAndRemoteSyncBase::mergeProcessedFromRemote()
 {
-    dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Merging processed data from remote. assetId='" << m_assetId <<
+    dbgDebug(D_WAAP_SERIALIZE) << "Merging processed data from remote. assetId='" << m_assetId <<
         "', owner='" << m_owner << "'";
     m_pMainLoop->addOneTimeRoutine(
         I_MainLoop::RoutineType::Offline,
@@ -903,7 +1040,7 @@ SerializeToLocalAndRemoteSyncBase::getLearningHost()
             m_learning_host = string(learningHost);
             return learningHost;
         }
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "learning host is not set. using default";
+        dbgWarning(D_WAAP_SERIALIZE) << "learning host is not set. using default";
     }
     return defaultLearningHost;
 }
@@ -919,7 +1056,7 @@ SerializeToLocalAndRemoteSyncBase::getSharedStorageHost()
             m_shared_storage_host = string(sharedStorageHost);
             return sharedStorageHost;
         }
-        dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "shared storage host is not set. using default";
+        dbgWarning(D_WAAP_SERIALIZE) << "shared storage host is not set. using default";
     }
     return defaultSharedStorageHost;
 }

components/security_apps/waap/waap_clib/Signatures.cc

@@ -116,7 +116,7 @@ Signatures::Signatures(const std::string& filepath) :
     ),
     allowed_text_re(sigsSource["allowed_text_re"].get<std::string>(), error, "allowed_text_re"),
     pipe_split_re(
-        "([\\w\\=\\-\\_\\.\\,\\(\\)\\[\\]\\/\\%\\s]+?)\\||([\\w\\=\\-\\_\\.\\,\\(\\)\\[\\]\\/\\%\\s]+)|\\|()",
+        "([^|]*)\\||([^|]+)|\\|()",
         error,
         "pipe_decode"),
     semicolon_split_re("([\\w\\=\\-\\_\\.\\,\\(\\)\\%]+?);|([\\w\\=\\-\\_\\.\\,\\(\\)\\%]+)|;()", error, "sem_decode"),

components/security_apps/waap/waap_clib/SingleDecision.cc

@@ -19,7 +19,10 @@ USE_DEBUG_FLAG(D_WAAP);
 SingleDecision::SingleDecision(DecisionType type):
     m_type(type),
     m_log(false),
-    m_block(false)
+    m_block(false),
+    m_ForceLog(false),
+    m_forceAllow(false),
+    m_forceBlock(false)
 {}
 
 SingleDecision::~SingleDecision()
@@ -35,11 +38,28 @@ bool SingleDecision::shouldLog() const
     return m_log;
 }
 
+bool SingleDecision::shouldForceLog() const
+{
+    return m_ForceLog;
+}
+
 bool SingleDecision::shouldBlock() const
 {
     return m_block;
 }
 
+bool SingleDecision::shouldForceAllow() const
+{
+    dbgTrace(D_WAAP) << "should force allow: " << m_forceAllow;
+    return m_forceAllow;
+}
+
+bool SingleDecision::shouldForceBlock() const
+{
+    dbgTrace(D_WAAP) << "should force block: " << m_forceBlock;
+    return m_forceBlock;
+}
+
 void SingleDecision::setLog(bool log)
 {
     dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes should log from " << m_log << " to " << log;
@@ -51,3 +71,22 @@ void SingleDecision::setBlock(bool block)
     dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes should block  from " << m_block << " to " << block;
     m_block = block;
 }
+
+void SingleDecision::setForceLog(bool overridesLog)
+{
+    dbgTrace(D_WAAP) << "Decision "<< getTypeStr() <<
+        " changes overrides log from " << m_ForceLog << " to " << overridesLog;
+    m_ForceLog = overridesLog;
+}
+
+void SingleDecision::setForceAllow(bool allow)
+{
+    dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes force allow from " << m_forceAllow << " to " << allow;
+    m_forceAllow = allow;
+}
+
+void SingleDecision::setForceBlock(bool block)
+{
+    dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes force block from " << m_forceBlock << " to " << block;
+    m_forceBlock = block;
+}

components/security_apps/waap/waap_clib/SingleDecision.h

@@ -25,15 +25,24 @@ public:
 
     void setLog(bool log);
     void setBlock(bool block);
+    void setForceLog(bool overridesLog);
+    void setForceAllow(bool allow);
+    void setForceBlock(bool block);
     DecisionType getType() const;
     bool shouldLog() const;
     bool shouldBlock() const;
+    bool shouldForceLog() const;
+    bool shouldForceAllow() const;
+    bool shouldForceBlock() const;
     virtual std::string getTypeStr() const = 0;
 
 protected:
     DecisionType m_type;
     bool m_log;
     bool m_block;
+    bool m_ForceLog;
+    bool m_forceAllow;
+    bool m_forceBlock;
 };
 
 #endif

components/security_apps/waap/waap_clib/Telemetry.cc

@@ -76,6 +76,7 @@ WaapTelemetrics::initMetrics()
     waf_blocked.report(0);
     force_and_block_exceptions.report(0);
 }
+
 void
 WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
 {
@@ -243,6 +244,46 @@ WaapAttackTypesMetrics::updateMetrics(const string &asset_id, const DecisionTele
     }
 }
 
+void
+WaapAdditionalTrafficTelemetrics::initMetrics()
+{
+    requests.report(0);
+    sources.report(0);
+    blocked.report(0);
+    temperature_count.report(0);
+    sources_seen.clear();
+}
+
+void
+WaapAdditionalTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
+{
+    initMetrics();
+
+    auto is_keep_alive_ctx = Singleton::Consume<I_Environment>::by<GenericMetric>()->get<bool>(
+        "keep_alive_request_ctx"
+    );
+    if (!is_keep_alive_ctx.ok() || !*is_keep_alive_ctx) {
+        requests.report(1);
+    } else {
+        dbgTrace(D_WAAP) << "Not increasing the number of requests due to keep alive";
+    }
+
+    if (!data.source.empty()) {
+        if (sources_seen.find(data.source) == sources_seen.end()) {
+            sources_seen.insert(data.source);
+            sources.report(1);
+        }
+    }
+
+    if (data.blockType == WAF_BLOCK) {
+        blocked.report(1);
+    }
+
+    if (data.temperatureDetected) {
+        temperature_count.report(1);
+    }
+}
+
 void
 WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
 {
@@ -268,10 +309,17 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
         attack_types_telemetries
     );
     initializeTelemetryData<WaapTrafficTelemetrics>(asset_id, data, "WAAP traffic telemetry", traffic_telemetries);
+    initializeTelemetryData<WaapAdditionalTrafficTelemetrics>(
+        asset_id,
+        data,
+        "WAAP Additional Traffic Telemetry",
+        additional_traffic_telemetries
+    );
 
     telemetries[asset_id]->updateMetrics(asset_id, data);
     attack_types_telemetries[asset_id]->updateMetrics(asset_id, data);
     traffic_telemetries[asset_id]->updateMetrics(asset_id, data);
+    additional_traffic_telemetries[asset_id]->updateMetrics(asset_id, data);
 
     auto agent_mode = Singleton::Consume<I_AgentDetails>::by<WaapMetricWrapper>()->getOrchestrationMode();
     string tenant_id = Singleton::Consume<I_AgentDetails>::by<WaapMetricWrapper>()->getTenantId();

components/security_apps/waap/waap_clib/TuningDecisions.h

@@ -60,6 +60,7 @@ private:
             MessageMetadata req_md(getSharedStorageHost(), 80);
             req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
             req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
+            req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
             auto req_status = messaging->sendSyncMessage(
                 method,
                 uri,
@@ -69,11 +70,14 @@ private:
             );
             return req_status.ok();
         }
+        MessageMetadata req_md;
+        req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN);
         auto req_status = messaging->sendSyncMessage(
             method,
             uri,
             obj,
-            MessageCategory::GENERIC
+            MessageCategory::GENERIC,
+            req_md
         );
         return req_status.ok();
     }

components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc

@@ -128,6 +128,8 @@ void TypeIndicatorFilter::loadParams(std::shared_ptr<Waap::Parameters::WaapParam
         std::to_string(TYPE_FILTER_CONFIDENCE_THRESHOLD)));
     std::string learnPermanentlyStr = pParams->getParamVal("typeIndicators.learnPermanently", "true");
     params.learnPermanently = !boost::iequals(learnPermanentlyStr, "false");
+    params.maxMemoryUsage = std::stoul(pParams->getParamVal("typeIndicators.maxMemoryUsage",
+        std::to_string(TYPE_FILTER_CONFIDENCE_MAX_MEMORY_USAGE)));
 
     std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true");
     bool syncEnabled = !boost::iequals(remoteSyncStr, "false");

components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h

@@ -24,6 +24,7 @@
 #define TYPE_FILTER_CONFIDENCE_MIN_INTERVALS 5
 #define TYPE_FILTER_CONFIDENCE_THRESHOLD 0.8
 #define TYPE_FILTER_INTERVAL_DURATION std::chrono::minutes(60)
+#define TYPE_FILTER_CONFIDENCE_MAX_MEMORY_USAGE (40 * 1024 * 1024) // 40MB
 
 class TypeIndicatorFilter : public IndicatorFilterBase
 {

components/security_apps/waap/waap_clib/WaapAssetState.cc

@@ -446,13 +446,14 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
         // update orig_size and orig_capacity after string copy
         orig_size = text.size();
         orig_capacity = text.capacity();
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (1) (after filterUTF7) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (1) (after filterUTF7) '" << text <<
+            "' size: " << text.size();
 
         // 2. Replace %xx sequences by their single-character equivalents.
         // Also replaces '+' symbol by space character.
         // Python equivalent: text = urllib.unquote_plus(text)
         text.erase(unquote_plus(text.begin(), text.end()), text.end());
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (2) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (2) '" << text << "' size: " << text.size();
 
         fixBreakingSpace(text);
 
@@ -460,38 +461,38 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
         // remove all characters whose ASCII code is >=128.
         // Python equivalent: text.encode('ascii',errors='ignore')
         filterUnicode(text);
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (3) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (3) '" << text << "' size: " << text.size();
 
         // 4. oh shi?... should I handle unicode html entities (python's htmlentitydefs module)???
         // Python equivalent: text = HTMLParser.HTMLParser().unescape(text)
         text.erase(escape_html(text.begin(), text.end()), text.end());
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (4) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (4) '" << text << "' size: " << text.size();
 
         // 5. Apply backslash escaping (like in C)
         // Python equivalent: text = text.decode('string_escape')
         text.erase(escape_backslashes(text.begin(), text.end()), text.end());
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (5) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (5) '" << text << "' size: " << text.size();
 
         // 6. remove all unicode characters from string. Basically,
         // remove all characters whose ASCII code is >=128.
         // Python equivalent: text.encode('ascii',errors='ignore')
         filterUnicode(text);
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (6) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (6) '" << text << "' size: " << text.size();
 
         // 7. Replace %xx sequences by their single-character equivalents.
         // Also replaces '+' symbol by space character.
         // Python equivalent: text = urllib.unquote_plus(text)
         text.erase(unquote_plus(text.begin(), text.end()), text.end());
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (7) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (7) '" << text << "' size: " << text.size();
 
         unescapeUnicode(text);
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "after unescapeUnicode '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "after unescapeUnicode '" << text << "' size: " << text.size();
 
         // 8. remove all unicode characters from string. Basically,
         // remove all characters whose ASCII code is >=128.
         // Python equivalent: text.encode('ascii',errors='ignore')
         filterUnicode(text);
-        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (8) '" << text << "'";
+        dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (8) '" << text << "' size: " << text.size();
 
         // 9. ???
         //
@@ -530,7 +531,7 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
             << AlertInfo(AlertTeam::CORE, "WAAP sample processing")
             << "unescape: original size=" << orig_size << " capacity=" << orig_capacity
             << " new size=" << text.size() << " capacity=" << text.capacity()
-            << " text='" << text << "'";
+            << " text='" << text << "'" << " orig text='" << s << "'";
 
         return text;
     }
@@ -1098,6 +1099,9 @@ WaapAssetState::apply(
     // Detect long text spans, and also any-length spans that end with file extensions such as ".jpg"
     bool longTextFound = m_Signatures->longtext_re.hasMatch(res.unescaped_line);
 
+    // When this flag remains false until the last evasion handling, a second unescape is performed
+    bool evasion_detected = false;
+
     if (longTextFound) {
         dbgTrace(D_WAAP_SAMPLE_SCAN) << "longtext found";
     }
@@ -1297,6 +1301,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1326,6 +1331,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1409,6 +1415,42 @@ WaapAssetState::apply(
         dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
     }
 
+    bool tab_evasion = (res.unescaped_line.find('\t') != std::string::npos);
+
+    if (tab_evasion) {
+        dbgTrace(D_WAAP_EVASIONS) << "Tab character evasion detected";
+
+        // Create normalized version with tabs removed
+        std::string unescaped = res.unescaped_line;
+
+        // Remove all tab characters to normalize the string
+        std::string::iterator end_pos = std::remove(unescaped.begin(), unescaped.end(), '\t');
+        unescaped.erase(end_pos, unescaped.end());
+
+        size_t kwCount = res.keyword_matches.size();
+
+        if (res.unescaped_line != unescaped) {
+            SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
+            checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
+                res.found_patterns, longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+        }
+
+        if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            // If new keywords were found, add a specific indication
+            res.keyword_matches.push_back("tab_character_evasion");
+
+            // Recalculate repetition and/or probing indicators
+            unsigned int newWordsCount = 0;
+            calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
+                newWordsCount);
+            // Take minimal words count because empirically it means evasion was probably successfully decoded
+            wordsCount = std::min(wordsCount, newWordsCount);
+        }
+    }
 
     bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
         "quotes_space_ev_fast_reg",
@@ -1471,6 +1513,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1499,7 +1542,7 @@ WaapAssetState::apply(
                     longTextFound, binaryDataFound);
         }
 
-
+        evasion_detected = true;
         // Recalculate repetition and/or probing indicators
         unsigned int newWordsCount = 0;
         calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1613,6 +1656,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1644,6 +1688,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1675,6 +1720,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1706,6 +1752,7 @@ WaapAssetState::apply(
         }
 
         if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1715,6 +1762,74 @@ WaapAssetState::apply(
         }
     }
 
+    // Detect evasion for command injection vulnerability using encoded characters (such as ^).
+    // example: "c^m^d /c d^i^r" -> "cmd /c dir"
+    // if '^' is found in the line, remove it and rescan
+    // search in scan result for any keyword with "os", "cmd", "exec" "command" or "shell" in it
+    // if not found, we don't want this evasion
+    // if found, we want to use its results
+    // Note: this is not a perfect solution, but it is better than nothing
+    if (line.find('^') != std::string::npos) {
+        dbgTrace(D_WAAP_EVASIONS) << "Windows command injection evasion suspected";
+
+        std::string unescaped = line;
+        bool is_win_cmd_evasion = false;
+        // remove all occurances of '^' character
+        unescaped.erase(std::remove(unescaped.begin(), unescaped.end(), '^'), unescaped.end());
+        dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'";
+        if (!unescaped.empty()) {
+            std::vector<std::string> evKeywordMatches(res.keyword_matches);
+            std::vector<std::string> evRegexMatches(res.regex_matches);
+            Waap::Util::map_of_stringlists_t evFoundPatterns(res.found_patterns);
+            bool evLongTextFound = longTextFound;
+            bool evBinaryDataFound = binaryDataFound;
+
+            if (line != unescaped) {
+                SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
+                checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, evKeywordMatches,
+                    res.found_patterns, evLongTextFound, evBinaryDataFound);
+                checkRegex(unescapedSample, m_Signatures->words_regex, evKeywordMatches, evFoundPatterns,
+                    evLongTextFound, evBinaryDataFound);
+                checkRegex(unescapedSample, m_Signatures->pattern_regex, evRegexMatches, evFoundPatterns,
+                    evLongTextFound, evBinaryDataFound);
+            }
+
+            if (evKeywordMatches.size() != res.keyword_matches.size()) {
+                if (Waap::Util::find_in_map_of_stringlists_keys("os_commands", evFoundPatterns)) {
+                    is_win_cmd_evasion = true;
+                } else {
+                    for (const auto &kw : evKeywordMatches) {
+                        if (kw.size() < 2 ||
+                                str_contains(kw, "cmd") ||
+                                str_contains(kw, "command") ||
+                                str_contains(kw, "os_") ||
+                                str_contains(kw, "exec")) {
+                            is_win_cmd_evasion = true;
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (is_win_cmd_evasion) {
+                dbgTrace(D_WAAP_EVASIONS) << "Evasion and relevant matches found, updating results";
+                // found relevant keywords after unescaping, set to new matches
+                res.keyword_matches = evKeywordMatches;
+                res.regex_matches = evRegexMatches;
+                res.found_patterns = evFoundPatterns;
+                longTextFound = evLongTextFound;
+                binaryDataFound = evBinaryDataFound;
+                // Recalculate repetition and/or probing indicators
+                unsigned int newWordsCount = 0;
+                calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
+                    newWordsCount);
+                // Take minimal words count because empirically it means evasion was probably successfully decoded
+                wordsCount = std::min(wordsCount, newWordsCount);
+            } else {
+                dbgTrace(D_WAAP_EVASIONS) << "Evasion not relevant, will not apply to results";
+            }
+        }
+    }
 
     // python: escape ='hi_acur_fast_reg_evasion' in found_patterns
     bool escape = Waap::Util::find_in_map_of_stringlists_keys("evasion", res.found_patterns);
@@ -1757,6 +1872,7 @@ WaapAssetState::apply(
             escape = false;
         }
         else if (!binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1833,6 +1949,7 @@ WaapAssetState::apply(
             escape = false;
         }
         else if (!binaryDataFound) {
+            evasion_detected = true;
             // Recalculate repetition and/or probing indicators
             unsigned int newWordsCount = 0;
             calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1841,6 +1958,42 @@ WaapAssetState::apply(
             wordsCount = std::min(wordsCount, newWordsCount);
         }
     }
+
+    // Handle second URL encoding evasion
+    if (!evasion_detected &&
+            !binaryDataFound &&
+            !longTextFound &&
+            Waap::Util::containsPercentEncoding(res.unescaped_line)) {
+        dbgTrace(D_WAAP_EVASIONS) << "Second URL encoding evasion detected";
+
+        std::string unescaped = unescape(res.unescaped_line);
+        size_t kwCount = res.keyword_matches.size();
+
+        if (res.unescaped_line != unescaped) {
+            SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
+            checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
+                res.found_patterns, longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+            checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
+                longTextFound, binaryDataFound);
+        }
+
+        if (kwCount == res.keyword_matches.size()) {
+            // Remove the evasion keyword if no real evasion found
+            keywordsToRemove.push_back("evasion");
+        }
+        else if (!binaryDataFound) {
+            evasion_detected = true;
+            // Recalculate repetition and/or probing indicators
+            unsigned int newWordsCount = 0;
+            calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
+                newWordsCount);
+            // Take minimal words count because empirically it means evasion was probably succesfully decoded
+            wordsCount = std::min(wordsCount, newWordsCount);
+        }
+    }
+
     dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
     // Remove evasion keywords that should not be reported because there's no real evasion found
     if (!keywordsToRemove.empty()) {

components/security_apps/waap/waap_clib/WaapConfigApplication.cc

@@ -83,7 +83,6 @@ void WaapConfigApplication::load(cereal::JSONInputArchive& ar)
     assets_ids_aggregation.insert(m_assetId);
 }
 
-
 bool WaapConfigApplication::operator==(const WaapConfigApplication& other) const
 {
     const WaapConfigBase* configBase = this;

components/security_apps/waap/waap_clib/WaapConfigBase.cc

@@ -132,7 +132,7 @@ void WaapConfigBase::loadOverridePolicy(cereal::JSONInputArchive& ar)
     try {
         m_overridePolicy = std::make_shared<Waap::Override::Policy>(ar);
     }
-    catch (std::runtime_error& e) {
+    catch (const cereal::Exception &e) {
         ar.setNextName(nullptr);
         dbgWarning(D_WAAP) << failMessage << e.what();
         m_overridePolicy = nullptr;
@@ -329,37 +329,38 @@ const std::string& WaapConfigBase::get_AssetName() const
     return m_assetName;
 }
 
-const std::string& WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const
+const string &
+WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const
 {
-
+    dbgTrace(D_WAAP) << "get practice ID of decision type: " << practiceType;
     switch (practiceType)
     {
-    case DecisionType::AUTONOMOUS_SECURITY_DECISION:
-        return m_practiceId;
+    case DecisionType::AUTONOMOUS_SECURITY_DECISION: break;
     default:
-        dbgError(D_WAAP)
-        << "Can't find practice type for practice ID by practice: "
-        << practiceType
-        << ", return web app practice ID";
-        return m_practiceId;
+        dbgDebug(D_WAAP)
+            << "Can't find practice type for practice ID by practice: "
+            << practiceType
+            << ", return web app practice ID";
     }
 
+    return m_practiceId;
 }
 
-const std::string& WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const
+const string &
+WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const
 {
+    dbgTrace(D_WAAP) << "get practice name of decision type: " << practiceType;
     switch (practiceType)
     {
-    case DecisionType::AUTONOMOUS_SECURITY_DECISION:
-        return m_practiceName;
+    case DecisionType::AUTONOMOUS_SECURITY_DECISION: break;
     default:
-        dbgError(D_WAAP)
-        << "Can't find practice type for practice name by practice: "
-        << practiceType
-        << ", return web app practice name";
-        return m_practiceName;
+        dbgDebug(D_WAAP)
+            << "Can't find practice type for practice name by practice: "
+            << practiceType
+            << ", return web app practice name";
     }
 
+    return m_practiceName;
 }
 
 const std::string& WaapConfigBase::get_RuleId() const

components/security_apps/waap/waap_clib/WaapOverride.cc

@@ -82,11 +82,48 @@ State::State() :
     forceBlockIds(),
     bForceException(false),
     forceExceptionIds(),
-    bIgnoreLog(false),
+    bSupressLog(false),
     bSourceIdentifierOverride(false),
     sSourceIdentifierMatch("")
 {
 }
 
+bool ExceptionsByPractice::operator==(const ExceptionsByPractice &other) const
+{
+    return m_web_app_ids == other.m_web_app_ids &&
+        m_api_protect_ids == other.m_api_protect_ids &&
+        m_anti_bot_ids == other.m_anti_bot_ids;
+}
+
+const std::vector<std::string>& ExceptionsByPractice::getExceptionsOfPractice(DecisionType practiceType) const
+{
+    switch (practiceType)
+    {
+
+    case DecisionType::AUTONOMOUS_SECURITY_DECISION:
+        return m_web_app_ids;
+    default:
+        dbgError(D_WAAP) <<
+        "Can't find practice type for exceptions by practice: " <<
+        practiceType <<
+        ", return web app exceptions";
+        return m_web_app_ids;
+    }
+}
+
+const std::set<std::string>& ExceptionsByPractice::getAllExceptions() const
+{
+    return m_all_ids;
+}
+
+bool ExceptionsByPractice::isIDInWebApp(const std::string &id) const
+{
+    auto it = std::find(m_web_app_ids.begin(), m_web_app_ids.end(), id);
+    if (it != m_web_app_ids.end()) {
+        dbgTrace(D_WAAP) << "rule id is in web application exceptions by practice: " << id;
+        return true;
+    }
+    return false;
+}
 }
 }

components/security_apps/waap/waap_clib/WaapOverride.h

@@ -21,6 +21,7 @@
 #include <memory>
 #include "debug.h"
 #include "CidrMatch.h"
+#include "DecisionType.h"
 #include "RegexComparator.h"
 
 USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
@@ -264,6 +265,15 @@ public:
 
     template <typename _A>
     void serialize(_A &ar) {
+        try {
+            ar(cereal::make_nvp("parsedMatch", m_match));
+        }
+        catch(const cereal::Exception &e)
+        {
+            dbgDebug(D_WAAP_OVERRIDE) << "An override rule was not loaded, parsedMatch error:" << e.what();
+            isValid = false;
+        }
+
         try {
             ar(cereal::make_nvp("id", m_id));
         }
@@ -272,7 +282,6 @@ public:
             dbgDebug(D_WAAP_OVERRIDE) << "An override rule has no id.";
             m_id.clear();
         }
-        ar(cereal::make_nvp("parsedMatch", m_match));
         if (!m_match.isValidMatch()) {
             dbgDebug(D_WAAP_OVERRIDE) << "An override rule was not load";
             isValid = false;
@@ -342,14 +351,50 @@ private:
     bool isValid;
 };
 
+class ExceptionsByPractice
+{
+public:
+    template <typename _A>
+    void serialize(_A& ar)
+    {
+        ar(
+            cereal::make_nvp("WebApplicationExceptions", m_web_app_ids),
+            cereal::make_nvp("APIProtectionExceptions", m_api_protect_ids),
+            cereal::make_nvp("AntiBotExceptions", m_anti_bot_ids)
+        );
+        m_all_ids.insert(m_web_app_ids.begin(), m_web_app_ids.end());
+        m_all_ids.insert(m_api_protect_ids.begin(), m_api_protect_ids.end());
+        m_all_ids.insert(m_anti_bot_ids.begin(), m_anti_bot_ids.end());
+    }
+
+    bool operator==(const ExceptionsByPractice &other) const;
+    const std::vector<std::string>& getExceptionsOfPractice(DecisionType practiceType) const;
+    const std::set<std::string>& getAllExceptions() const;
+    bool isIDInWebApp(const std::string &id) const;
+private:
+    std::vector<std::string> m_web_app_ids;
+    std::vector<std::string> m_api_protect_ids;
+    std::vector<std::string> m_anti_bot_ids;
+    std::set<std::string> m_all_ids;
+};
+
 class Policy {
 public:
     template <typename _A>
     Policy(_A &ar) {
+        try {
+            ar(
+                cereal::make_nvp("exceptionsPerPractice", m_exceptionsByPractice)
+            );
+        }
+        catch (std::runtime_error & e) {
+            ar.setNextName(nullptr);
+            dbgInfo(D_WAAP_OVERRIDE) << "Failed to load exceptions per practice, error: ", e.what();
+            m_exceptionsByPractice = ExceptionsByPractice();
+        }
         std::vector<Waap::Override::Rule> rules;
         ar(cereal::make_nvp("overrides", rules));
         m_isOverrideResponse = false;
-
         for (std::vector<Waap::Override::Rule>::const_iterator it = rules.begin(); it != rules.end(); ++it) {
             const Waap::Override::Rule& rule = *it;
             if (!rule.isValidRule()) {
@@ -379,6 +424,14 @@ public:
         const std::vector<Waap::Override::Rule>& rules = requestOverrides ? m_RequestOverrides : m_ResponseOverrides;
         dbgTrace(D_WAAP_OVERRIDE) << "Start matching override rules ...";
         for (const Waap::Override::Rule &rule : rules) {
+            if (m_exceptionsByPractice.getAllExceptions().size() > 0 &&
+                !m_exceptionsByPractice.isIDInWebApp(rule.getId())
+            ) {
+                dbgInfo(D_WAAP_OVERRIDE)
+                << "match rule id is not in web application exceptions by practice: "
+                << rule.getId();
+                continue;
+            }
             dbgTrace(D_WAAP_OVERRIDE) << "Matching override rule ...";
             rule.match(testFunctor, matchedBehaviors, matchedOverrideIds);
         }
@@ -389,9 +442,17 @@ public:
         return m_isOverrideResponse;
     }
 
+    bool isValidRules() {
+        return !m_RequestOverrides.empty() || !m_ResponseOverrides.empty();
+    }
+
+    const ExceptionsByPractice& getExceptionsByPractice() const {
+        return m_exceptionsByPractice;
+    }
 private:
     std::vector<Waap::Override::Rule> m_RequestOverrides; //overrides that change request data
     std::vector<Waap::Override::Rule> m_ResponseOverrides; //overrides that change response/log data
+    ExceptionsByPractice m_exceptionsByPractice;
     bool m_isOverrideResponse;
 };
 
@@ -403,7 +464,7 @@ struct State {
     bool bForceException;
     std::set<std::string> forceExceptionIds;
     // overrides decision in case log should be ignored
-    bool bIgnoreLog;
+    bool bSupressLog;
     // user identfier override to be applied
     bool bSourceIdentifierOverride;
     std::string sSourceIdentifierMatch;
@@ -437,8 +498,8 @@ struct State {
 
             if (matchedBehavior.getLog() == "ignore")
             {
-                dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bIgnoreLog due to override behavior.";
-                bIgnoreLog = true;
+                dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bSupressLog due to override behavior.";
+                bSupressLog = true;
             }
 
             sSourceIdentifierMatch = matchedBehavior.getSourceIdentifier();

components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc

@@ -26,8 +26,7 @@ namespace Waap {
     errorLimiter(false),
     rateLimiting(false),
     collectResponseForLog(false),
-    applyOverride(false),
-    triggerReport(false)
+    applyOverride(false)
     {
     }
 
@@ -40,12 +39,11 @@ namespace Waap {
             " RateLimiting=" << rateLimiting <<
             " ErrorLimiter=" << errorLimiter <<
             " collectResponseForLog=" << collectResponseForLog <<
-            " applyOverride=" << applyOverride <<
-            " triggerReport=" << triggerReport;
+            " applyOverride=" << applyOverride;
 
         return
             openRedirect || errorDisclosure || rateLimiting || errorLimiter ||
-            collectResponseForLog || applyOverride || triggerReport;
+            collectResponseForLog || applyOverride;
     }
 
     void
@@ -93,14 +91,6 @@ namespace Waap {
         applyOverride = flag;
     }
 
-    void
-    ResponseInspectReasons::setTriggerReport(bool flag)
-    {
-        dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setTriggerReport) " << triggerReport << " to " <<
-            flag;
-        triggerReport = flag;
-    }
-
     bool
     ResponseInspectReasons::getApplyOverride(void)
     {

components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h

@@ -25,7 +25,6 @@ public:
     void setErrorLimiter(bool flag);
     void setCollectResponseForLog(bool flag);
     void setApplyOverride(bool flag);
-    void setTriggerReport(bool flag);
 
     bool getApplyOverride(void);
 private:
@@ -35,7 +34,6 @@ private:
     bool rateLimiting;
     bool collectResponseForLog;
     bool applyOverride;
-    bool triggerReport;
 };
 
 }

components/security_apps/waap/waap_clib/WaapTrigger.cc

@@ -75,5 +75,53 @@ bool Policy::operator==(const Policy &other) const
     return triggers == other.triggers;
 }
 
+bool TriggersByPractice::operator==(const TriggersByPractice &other) const
+{
+    return m_web_app_ids == other.m_web_app_ids &&
+        m_api_protect_ids == other.m_api_protect_ids &&
+        m_anti_bot_ids == other.m_anti_bot_ids;
+}
+
+const std::vector<std::string>& TriggersByPractice::getTriggersByPractice(DecisionType practiceType) const
+{
+    switch (practiceType)
+    {
+    case DecisionType::AUTONOMOUS_SECURITY_DECISION:
+        return m_web_app_ids;
+    default:
+        dbgError(D_WAAP) <<
+        "Can't find practice type for triggers by practice: " <<
+        practiceType <<
+        ", return web app triggers";
+        return m_web_app_ids;
+    }
+}
+
+const std::vector<std::string>& TriggersByPractice::getAllTriggers() const
+{
+    return m_all_ids;
+}
+
+bool WebUserResponseByPractice::operator==(const WebUserResponseByPractice &other) const
+{
+    return m_web_app_ids == other.m_web_app_ids &&
+        m_api_protect_ids == other.m_api_protect_ids &&
+        m_anti_bot_ids == other.m_anti_bot_ids;
+}
+
+const std::vector<std::string>& WebUserResponseByPractice::getResponseByPractice(DecisionType practiceType) const
+{
+    switch (practiceType)
+    {
+    case DecisionType::AUTONOMOUS_SECURITY_DECISION:
+        return m_web_app_ids;
+    default:
+        dbgDebug(D_WAAP)
+            << "Can't find practice type for triggers by practice: "
+            << practiceType
+            << ", return web app triggers";
+        return m_web_app_ids;
+    }
+}
 }
 }

components/security_apps/waap/waap_clib/WaapTrigger.h

@@ -19,6 +19,7 @@
 #include <string>
 #include <memory>
 #include "debug.h"
+#include "DecisionType.h"
 
 USE_DEBUG_FLAG(D_WAAP);
 
@@ -143,15 +144,86 @@ struct Trigger {
     std::shared_ptr<Log> log;
 };
 
+class TriggersByPractice
+{
+public:
+    template <typename _A>
+    void serialize(_A& ar)
+    {
+        ar(
+            cereal::make_nvp("WebApplicationTriggers", m_web_app_ids),
+            cereal::make_nvp("APIProtectionTriggers", m_api_protect_ids),
+            cereal::make_nvp("AntiBotTriggers", m_anti_bot_ids)
+        );
+        m_all_ids.insert(m_all_ids.end(), m_web_app_ids.begin(), m_web_app_ids.end());
+        m_all_ids.insert(m_all_ids.end(), m_api_protect_ids.begin(), m_api_protect_ids.end());
+        m_all_ids.insert(m_all_ids.end(), m_anti_bot_ids.begin(), m_anti_bot_ids.end());
+    }
+
+    bool operator==(const TriggersByPractice &other) const;
+    const std::vector<std::string>& getTriggersByPractice(DecisionType practiceType) const;
+    const std::vector<std::string>& getAllTriggers() const;
+private:
+    std::vector<std::string> m_web_app_ids;
+    std::vector<std::string> m_api_protect_ids;
+    std::vector<std::string> m_anti_bot_ids;
+    std::vector<std::string> m_all_ids;
+};
+
+class WebUserResponseByPractice
+{
+public:
+    template <typename _A>
+    void serialize(_A& ar)
+    {
+        ar(
+            cereal::make_nvp("WebApplicationResponse", m_web_app_ids),
+            cereal::make_nvp("APIProtectionResponse", m_api_protect_ids),
+            cereal::make_nvp("AntiBotResponse", m_anti_bot_ids)
+        );
+    }
+
+    bool operator==(const WebUserResponseByPractice &other) const;
+    const std::vector<std::string>& getResponseByPractice(DecisionType practiceType) const;
+private:
+    std::vector<std::string> m_web_app_ids;
+    std::vector<std::string> m_api_protect_ids;
+    std::vector<std::string> m_anti_bot_ids;
+};
+
 struct Policy {
     template <typename _A>
     Policy(_A &ar) {
-        ar(cereal::make_nvp("triggers", triggers));
+        try {
+            ar(
+                cereal::make_nvp("triggersPerPractice", triggersByPractice)
+            );
+        }
+        catch (std::runtime_error &e) {
+            ar.setNextName(nullptr);
+            dbgInfo(D_WAAP) << "Failed to load triggers per practice, error: " << e.what();
+            triggersByPractice = TriggersByPractice();
+        }
+        try {
+            ar(
+                cereal::make_nvp("webUserResponsePerPractice", responseByPractice)
+            );
+        }
+        catch (std::runtime_error &e) {
+            ar.setNextName(nullptr);
+            dbgInfo(D_WAAP) << "Failed to load web user response per practice, error: " << e.what();
+            responseByPractice = WebUserResponseByPractice();
+        }
+        ar(
+            cereal::make_nvp("triggers", triggers)
+        );
     }
 
     bool operator==(const Policy &other) const;
 
     std::vector<Waap::Trigger::Trigger> triggers;
+    TriggersByPractice triggersByPractice;
+    WebUserResponseByPractice responseByPractice;
 };
 
 }

components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc

@@ -229,6 +229,7 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
                 case '(':
                 case ')':
                 case '|':
+                case '+':
                     break;
                 default:
                     // Only alphanumeric characters and characters listed above are allowed, anything else disables

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -45,8 +45,6 @@
 #include <iostream>
 #include "ParserDelimiter.h"
 #include "OpenRedirectDecision.h"
-#include "DecisionType.h"
-#include "generic_rulebase/triggers_config.h"
 #include "config.h"
 #include "LogGenWrapper.h"
 #include "reputation_features_events.h"
@@ -59,6 +57,7 @@ USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION);
 USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
 USE_DEBUG_FLAG(D_WAAP_HEADERS);
 
+
 using namespace ReportIS;
 using namespace std;
 
@@ -92,9 +91,6 @@ void Waf2Transaction::start_response(int response_status, int http_version)
     dbgTrace(D_WAAP) << "[transaction:" << this << "] start_response(response_status=" << response_status
         << "," << " http_version=" << http_version << ")";
     m_responseStatus = response_status;
-    if (m_triggerReport) {
-        m_responseInspectReasons.setTriggerReport(false);
-    }
 
     if(m_responseStatus == 404)
     {
@@ -324,11 +320,12 @@ Waf2Transaction::Waf2Transaction() :
     m_responseStatus(0),
     m_responseInspectReasons(),
     m_responseInjectReasons(),
+    m_practiceSubType("Web Application"),
     m_index(-1),
-    m_triggerLog(),
     m_triggerReport(false),
     is_schema_validation(false),
-    m_waf2TransactionFlags()
+    m_waf2TransactionFlags(),
+    m_temperature_detected(false)
 {
     m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = 0;
     I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
@@ -363,11 +360,12 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
     m_responseStatus(0),
     m_responseInspectReasons(),
     m_responseInjectReasons(),
+    m_practiceSubType("Web Application"),
     m_index(-1),
-    m_triggerLog(),
     m_triggerReport(false),
     is_schema_validation(false),
-    m_waf2TransactionFlags()
+    m_waf2TransactionFlags(),
+    m_temperature_detected(false)
 {
     I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
     m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
@@ -574,6 +572,8 @@ void Waf2Transaction::start() {
     hdrs_map.clear();
     m_request_body.clear();
     m_response_body.clear();
+    m_overrideStateByPractice.clear();
+    m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = Waap::Override::State();
 }
 
 void Waf2Transaction::set_transaction_time(const char* log_time) {
@@ -639,6 +639,7 @@ bool Waf2Transaction::checkIsScanningRequired()
             result = true;
         }
     }
+
     return result;
 }
 
@@ -828,7 +829,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
        }
         dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
 
-        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
+        ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode, false);
         up.push(p, buff_len);
         up.finish();
         m_deepParser.m_key.pop(tag.c_str());
@@ -875,7 +876,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len)
     if (value_len > 0) {
         dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the cookie value";
         m_deepParser.m_key.push("cookie", 6);
-        ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';', false);
+        ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';', false, false);
         cookieValueParser.push(value, value_len);
         cookieValueParser.finish();
         m_deepParser.m_key.pop("cookie");
@@ -969,10 +970,7 @@ void Waf2Transaction::scanSpecificHeader(const char* name, int name_len, const c
         parseUnknownHeaderName(name, name_len);
         // Scan unknown headers whose values do not match "clean generic header" pattern.
         // Note that we do want to process special header named x-chkp-csrf-token header - it is treated specially.
-        if (!m_pWaapAssetState->getSignatures()->good_header_value_re.hasMatch(std::string(value, value_len)) ||
-                headerName == "x-chkp-csrf-token" || headerType == HeaderType::OTHER_KNOWN_HEADERS) {
-            parseGenericHeaderValue(headerName, value, value_len);
-        }
+        parseGenericHeaderValue(headerName, value, value_len);
         break;
     }
     case HeaderType::USER_AGENT_HEADER: {
@@ -1104,9 +1102,6 @@ void Waf2Transaction::end_request_hdrs() {
     if (m_isScanningRequired) {
         createUserLimitsState();
         detectHeaders();
-        if (isUserLimitReached()) {
-            return;
-        }
     }
     // Scan URL and url query
     if (m_isScanningRequired && !m_processedUri) {
@@ -1117,6 +1112,16 @@ void Waf2Transaction::end_request_hdrs() {
         scanHeaders();
     }
 
+    // Chack after scanning the URL and headers so we have the valuse for state.
+    if (m_siteConfig != NULL)
+    {
+        m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(m_siteConfig);
+    }
+
+    if(m_isScanningRequired && isUserLimitReached()) {
+        return;
+    }
+
 
     if(m_siteConfig != NULL) {
         // Create rate limiting policy (lazy, on first request)
@@ -1275,12 +1280,6 @@ void Waf2Transaction::end_request() {
     // Enable response headers processing if response scanning is enabled in policy
     auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL;
     m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable);
-
-    auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL;
-    if (isTriggerReportExists(triggerPolicy)) {
-        m_responseInspectReasons.setTriggerReport(true);
-        dbgTrace(D_WAAP) << "setTriggerReport(true)";
-    }
 }
 
 void Waf2Transaction::extractEnvSourceIdentifier()
@@ -1360,7 +1359,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
     std::string body(data, data_len);
     if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
     {
-        dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
+        dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false. Html regex not matched.";
         return false;
     }
     dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: true";
@@ -1532,7 +1531,7 @@ Waf2Transaction::decideAfterHeaders()
     }
 
     m_isHeaderOverrideScanRequired = true;
-    m_overrideState = getOverrideState(sitePolicy);
+    m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy);
 
     // Select scores pool by location (but use forced pool when forced)
     std::string realPoolName =
@@ -1551,7 +1550,7 @@ Waf2Transaction::decideAfterHeaders()
         UNKNOWN_TYPE
     );
 
-    return finalizeDecision(sitePolicy, shouldBlock);
+    return shouldBlock;
 }
 
 
@@ -1584,7 +1583,7 @@ Waf2Transaction::decideFinal(
     if (WaapConfigAPI::getWaapAPIConfig(ngenAPIConfig)) {
         dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant API configuration from the I/S";
         sitePolicy = &ngenAPIConfig;
-        m_overrideState = getOverrideState(sitePolicy);
+        m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy);
 
         // User limits
         shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
@@ -1592,7 +1591,7 @@ Waf2Transaction::decideFinal(
     else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) {
         dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S";
         sitePolicy = &ngenSiteConfig;
-        m_overrideState = getOverrideState(sitePolicy);
+        m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy);
 
         shouldBlock = decideAutonomousSecurity(
             *sitePolicy,
@@ -1600,7 +1599,8 @@ Waf2Transaction::decideFinal(
             false,
             transactionResult,
             realPoolName,
-            fpClassification);
+            fpClassification
+        );
 
         // CSRF Protection
         auto csrfPolicy = m_siteConfig ? m_siteConfig->get_CsrfPolicy() : nullptr;
@@ -1613,62 +1613,23 @@ Waf2Transaction::decideFinal(
 
     if (mode == 2) {
         decide(
-            m_overrideState.bForceBlock,
-            m_overrideState.bForceException,
+            m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock,
+            m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException,
             mode
         );
         shouldBlock = isSuspicious();
     }
 
-    return finalizeDecision(sitePolicy, shouldBlock);
-}
-
-int
-Waf2Transaction::finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock)
-{
-    auto decision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
-        m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
-    // Send log
-    if (sitePolicy)
-    {
-        // auto reject should have default threat level info and above
-        if (m_overrideState.bForceBlock && decision->getThreatLevel() == ThreatLevel::NO_THREAT)
-        {
-            decision->setThreatLevel(ThreatLevel::THREAT_INFO);
-        }
-    }
-
-    if (m_overrideState.bForceBlock) {
-        dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): setting shouldBlock to true due to override";
-        shouldBlock = true; // BLOCK
-    }
-    else if (m_overrideState.bForceException) {
-        dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): setting shouldBlock to false due to override";
-        shouldBlock = false; // PASS
-    }
-
-    if (m_siteConfig) {
-        const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = m_siteConfig->get_TriggerPolicy();
-        if (triggerPolicy) {
-            const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
-            if (triggerLog && shouldSendExtendedLog(triggerLog))
-            {
-                m_responseInspectReasons.setCollectResponseForLog(true);
-            }
-        }
-    }
-
-    dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): returning shouldBlock: " << shouldBlock;
+    dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): returning shouldBlock: " << shouldBlock;
     return shouldBlock;
 }
 
 void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
-    const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
+    const LogTriggerConf &triggerLog,
     bool shouldBlock,
     const std::string& logOverride,
     const std::string& incidentType,
-    const std::string& practiceID,
-    const std::string& practiceName) const
+    DecisionType practiceType) const
 {
     auto env = Singleton::Consume<I_Environment>::by<WaapComponent>();
     auto active_id = env->get<std::string>("ActiveTenantId");
@@ -1686,10 +1647,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
     if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
     if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
 
-    const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
-        m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
-    bool send_extended_log = shouldSendExtendedLog(triggerLog);
-    if (triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) {
+    const auto& decision = m_waapDecision.getDecision(practiceType);
+    bool send_extended_log = shouldSendExtendedLog(triggerLog, practiceType);
+    if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webUrlPath) || decision->shouldForceLog()) {
         std::string httpUriPath = m_uriPath;
 
         if (httpUriPath.length() > MAX_LOG_FIELD_SIZE)
@@ -1699,7 +1659,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
 
         waapLog << LogField("httpUriPath", httpUriPath, LogFieldOption::XORANDB64);
     }
-    if (triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) {
+    if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webUrlQuery) || decision->shouldForceLog()) {
         std::string uriQuery = m_uriQuery;
         if (uriQuery.length() > MAX_LOG_FIELD_SIZE)
         {
@@ -1707,16 +1667,16 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
         }
         waapLog << LogField("httpUriQuery", uriQuery, LogFieldOption::XORANDB64);
     }
-    if (triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) {
+    if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webHeaders) || decision->shouldForceLog()) {
         waapLog << LogField("httpRequestHeaders", logHeadersStr(), LogFieldOption::XORANDB64);
     }
     // Log http response code if it is known
-    if (m_responseStatus != 0 && send_extended_log && triggerLog->responseCode) {
+    if (m_responseStatus != 0 && send_extended_log && triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode)) {
         waapLog << LogField("httpResponseCode", std::to_string(m_responseStatus));
     }
 
     // Count of bytes available to send to the log
-    std::string requestBodyToLog = (triggerLog->webBody) ?
+    std::string requestBodyToLog = (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webBody)) ?
         m_request_body : std::string();
     std::string responseBodyToLog = m_response_body;
     if (!shouldBlock && responseBodyToLog.empty())
@@ -1748,7 +1708,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
         waapLog << LogField("httpRequestBody", requestBodyToLog, LogFieldOption::XORANDB64);
     }
 
-    if (!responseBodyToLog.empty() && send_extended_log && triggerLog->responseBody)
+    if (!responseBodyToLog.empty() && send_extended_log && triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody))
     {
         waapLog << LogField("httpResponseBody", responseBodyToLog, LogFieldOption::XORANDB64);
     }
@@ -1757,10 +1717,10 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
     waapLog << LogField("securityAction", shouldBlock ? "Prevent" : "Detect");
     waapLog << LogField("waapOverride", logOverride);
     waapLog << LogField("practiceType", "Threat Prevention");
-    waapLog << LogField("practiceSubType", m_siteConfig->get_PracticeSubType());
+    waapLog << LogField("practiceSubType", m_practiceSubType);
     waapLog << LogField("ruleName", m_siteConfig->get_RuleName());
-    waapLog << LogField("practiceId", practiceID);
-    waapLog << LogField("practiceName", practiceName);
+    waapLog << LogField("practiceId", m_siteConfig->get_PracticeIdByPactice(practiceType));
+    waapLog << LogField("practiceName", m_siteConfig->get_PracticeNameByPactice(practiceType));
     waapLog << LogField("waapIncidentType", incidentType);
 
     // Registering this value would append the list of matched override IDs to the unified log
@@ -1786,19 +1746,24 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
 void
 Waf2Transaction::sendLog()
 {
-    dbgFlow(D_WAAP);
+    dbgFlow(D_WAAP) << "send log";
     m_waapDecision.orderDecisions();
     if (m_siteConfig == NULL) {
         dbgWarning(D_WAAP) <<
             "Waf2Transaction::sendLog: no site policy associated with transaction - not sending a log";
         return;
     }
+    DecisionType decision_type = m_waapDecision.getHighestPriorityDecisionToLog();
+    dbgTrace(D_WAAP) << "send log got decision type: " << decision_type;
+    auto final_decision = m_waapDecision.getDecision(decision_type);
+    if (!final_decision) {
+        final_decision = m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION);
+    }
+
     std::string attackTypes = buildAttackTypes();
     std::string logOverride = "None";
     DecisionTelemetryData telemetryData;
     std::string assetId = m_siteConfig->get_AssetId();
-    const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
-        m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
 
     I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
     auto finish = timeGet->getMonotonicTime();
@@ -1824,53 +1789,55 @@ Waf2Transaction::sendLog()
         telemetryData.responseCode = m_responseStatus;
     }
 
-
     telemetryData.source = getSourceIdentifier();
     telemetryData.assetName = m_siteConfig->get_AssetName();
     telemetryData.practiceId = m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION);
     telemetryData.practiceName = m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION);
-    if (m_scanResult) {
-        telemetryData.attackTypes = m_scanResult->attack_types;
-    }
-    telemetryData.threat = autonomousSecurityDecision->getThreatLevel();
-    if (m_overrideState.bForceBlock) {
-        telemetryData.blockType = FORCE_BLOCK;
-    }
-    else if (m_overrideState.bForceException) {
-        telemetryData.blockType = FORCE_EXCEPTION;
-    }
-    else if (m_waapDecision.getDecision(USER_LIMITS_DECISION)->shouldBlock()) {
+    telemetryData.temperatureDetected = wasTemperatureDetected();
+    switch (decision_type)
+    {
+    case USER_LIMITS_DECISION: {
         telemetryData.blockType = LIMIT_BLOCK;
+        break;
     }
-    else if (autonomousSecurityDecision->shouldBlock()) {
-        telemetryData.blockType = WAF_BLOCK;
-    }
-    else if (m_waapDecision.getDecision(CSRF_DECISION)->shouldBlock()) {
+    case CSRF_DECISION: {
         telemetryData.blockType = CSRF_BLOCK;
+        break;
     }
-    else {
+    case AUTONOMOUS_SECURITY_DECISION: {
+        telemetryData.blockType = WAF_BLOCK;
+        break;
+    }
+    default:
         telemetryData.blockType = NOT_BLOCKING;
-    }
-
-    WaapTelemetryEvent(assetId, telemetryData).notify();
-
-    if (m_overrideState.bIgnoreLog) {
-        dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: override is to ignore log - not sending a log";
-        return;
+        break;
     }
 
     bool shouldBlock = false;
-    if (m_overrideState.bForceBlock) {
+    if (final_decision->shouldForceBlock()) {
+        telemetryData.blockType = FORCE_BLOCK;
         // If override forces "reject" decision, mention it in the "override" log field.
         logOverride = OVERRIDE_DROP;
         shouldBlock = true;
-    } else if (m_overrideState.bForceException) {
+    }
+    else if (final_decision->shouldForceAllow()) {
+        telemetryData.blockType = FORCE_EXCEPTION;
         // If override forces "allow" decision, mention it in the "override" log field.
         logOverride = OVERRIDE_ACCEPT;
-    } else if (m_scanner.getIgnoreOverride()) {
+    } else if (decision_type == AUTONOMOUS_SECURITY_DECISION && m_scanner.getIgnoreOverride()) {
+        // skip exception detected by scanner
+        dbgTrace(D_WAAP) << "should ignore override";
         logOverride = OVERRIDE_IGNORE;
     }
 
+    WaapTelemetryEvent(assetId, telemetryData).notify();
+    auto it = m_overrideStateByPractice.find(decision_type);
+    if ((it != m_overrideStateByPractice.end() && it->second.bSupressLog) ||
+        (it == m_overrideStateByPractice.end() && m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog)) {
+        dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: override is to ignore log - not sending a log";
+        return;
+    }
+
     // Get triggers
     const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = m_siteConfig->get_TriggerPolicy();
 
@@ -1879,27 +1846,29 @@ Waf2Transaction::sendLog()
         return;
     }
 
-    const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
-
+    auto maybeTriggerLog = getTriggerLog(triggerPolicy, decision_type);
     // If there were no triggers of type Log - do not send log
-    if (!triggerLog) {
+    if (!maybeTriggerLog.ok()) {
         dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: found no triggers of type 'Log' - not sending a log";
         return;
     }
-
-    bool send_extended_log = shouldSendExtendedLog(triggerLog);
+    auto triggerLog = maybeTriggerLog.unpack();
+    bool send_extended_log = shouldSendExtendedLog(triggerLog, decision_type);
     shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision();
     // Do not send Detect log if trigger disallows it
-    if (!send_extended_log && shouldBlock == false && !triggerLog->tpDetect &&
-        !autonomousSecurityDecision->getOverridesLog())
+    // and we do not override log(for ignore exception)
+    if (!send_extended_log && shouldBlock == false &&
+        !triggerLog.isDetectLogActive(LogTriggerConf::SecurityType::ThreatPrevention) &&
+        !final_decision->shouldForceLog())
     {
         dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: not sending Detect log (triggers)";
         return;
     }
 
     // Do not send Prevent log if trigger disallows it
-    if (!send_extended_log && shouldBlock == true && !triggerLog->tpPrevent &&
-        !autonomousSecurityDecision->getOverridesLog())
+    if (!send_extended_log && shouldBlock == true &&
+        !triggerLog.isPreventLogActive(LogTriggerConf::SecurityType::ThreatPrevention) &&
+        !final_decision->shouldForceLog())
     {
         dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: not sending Prevent log (triggers)";
         return;
@@ -1908,7 +1877,7 @@ Waf2Transaction::sendLog()
     // In case no decision to block or log - send log if extend log or override
     if (!m_waapDecision.anyDecisionsToLogOrBlock())
     {
-        if (send_extended_log || autonomousSecurityDecision->getOverridesLog())
+        if (send_extended_log || final_decision->shouldForceLog())
         {
             sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes);
             dbgTrace(D_WAAP) << "Waf2Transaction::sendLog()::" <<
@@ -1921,24 +1890,14 @@ Waf2Transaction::sendLog()
         return;
     }
 
-    DecisionType decision_type = m_waapDecision.getHighestPriorityDecisionToLog();
     if (decision_type == DecisionType::NO_WAAP_DECISION) {
-        if (send_extended_log || autonomousSecurityDecision->getOverridesLog()) {
+        if (send_extended_log || final_decision->shouldForceLog()) {
             sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes);
         }
         dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: decisions marked for block only";
         return;
     }
 
-    std::set<std::string> triggers_set;
-    for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
-        triggers_set.insert(trigger.triggerId);
-        dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
-    }
-    ScopedContext ctx;
-    ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
-
-    auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
     switch (decision_type)
     {
     case USER_LIMITS_DECISION: {
@@ -1960,7 +1919,7 @@ Waf2Transaction::sendLog()
         }
 
         LogGenWrapper logGenWrapper(
-                                maybeLogTriggerConf,
+                                maybeTriggerLog,
                                 "Web Request",
                                 ReportIS::Audience::SECURITY,
                                 LogTriggerConf::SecurityType::ThreatPrevention,
@@ -1969,11 +1928,7 @@ Waf2Transaction::sendLog()
                                 shouldBlock);
 
         LogGen& waap_log = logGenWrapper.getLogGen();
-        appendCommonLogFields(
-            waap_log, triggerLog, shouldBlock, logOverride, incidentType,
-            m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
-            m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
-        );
+        appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType, decision_type);
         waap_log << LogField("waapIncidentDetails", incidentDetails);
         waap_log << LogField("eventConfidence", "High");
         break;
@@ -1983,7 +1938,7 @@ Waf2Transaction::sendLog()
     case RATE_LIMITING_DECISION:
     case ERROR_DISCLOSURE_DECISION: {
         LogGenWrapper logGenWrapper(
-                                maybeLogTriggerConf,
+                                maybeTriggerLog,
                                 "API Request",
                                 ReportIS::Audience::SECURITY,
                                 LogTriggerConf::SecurityType::ThreatPrevention,
@@ -2006,18 +1961,14 @@ Waf2Transaction::sendLog()
             waap_log << LogField("waapFoundIndicators", getKeywordMatchesStr(), LogFieldOption::XORANDB64);
         }
 
-        appendCommonLogFields(
-            waap_log, triggerLog, shouldBlock, logOverride, incidentType,
-            m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
-            m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
-        );
+        appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType, decision_type);
 
         waap_log << LogField("waapIncidentDetails", incidentDetails);
         break;
     }
     case CSRF_DECISION: {
         LogGenWrapper logGenWrapper(
-                                maybeLogTriggerConf,
+                                maybeTriggerLog,
                                 "CSRF Protection",
                                 ReportIS::Audience::SECURITY,
                                 LogTriggerConf::SecurityType::ThreatPrevention,
@@ -2027,18 +1978,17 @@ Waf2Transaction::sendLog()
 
         LogGen& waap_log = logGenWrapper.getLogGen();
         appendCommonLogFields(
-            waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery",
-            m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
-            m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
+            waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery", decision_type
         );
         waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered.");
         break;
     }
     case AUTONOMOUS_SECURITY_DECISION: {
-        if (triggerLog->webRequests ||
+        if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webRequests) ||
             send_extended_log ||
-            autonomousSecurityDecision->getThreatLevel() != ThreatLevel::NO_THREAT ||
-            autonomousSecurityDecision->getOverridesLog()) {
+            std::dynamic_pointer_cast<AutonomousSecurityDecision>(final_decision)
+                ->getThreatLevel() != ThreatLevel::NO_THREAT ||
+            final_decision->shouldForceLog()) {
             sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes);
         }
         break;
@@ -2080,7 +2030,8 @@ Waf2Transaction::decideAutonomousSecurity(
 
     // Do not call stage2 so it doesn't learn from exceptions.
     // Also do not call stage2 for attacks found in parameter name
-    if (!m_overrideState.bForceException && !(m_scanResult && m_scanResult->m_isAttackInParam)) {
+    if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException &&
+         !(m_scanResult && m_scanResult->m_isAttackInParam)) {
         if (!m_processedUri) {
             dbgWarning(D_WAAP) << "decideAutonomousSecurity(): processing URI although is was supposed "
                 "to be processed earlier ...";
@@ -2135,7 +2086,8 @@ Waf2Transaction::decideAutonomousSecurity(
     }
 
     // Fill attack details for attacks found in parameter names
-    if (!m_overrideState.bForceException && m_scanResult && m_scanResult->m_isAttackInParam) {
+    if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException &&
+         m_scanResult && m_scanResult->m_isAttackInParam) {
         // Since stage2 learning doesn't run in this case, assume stage1 score is the final score
         float finalScore = m_scanResult->score;
         ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(finalScore);
@@ -2153,13 +2105,18 @@ Waf2Transaction::decideAutonomousSecurity(
         transactionResult.d2Analysis.finalScore = finalScore;
         transactionResult.shouldBlock = shouldBlock;
         transactionResult.threatLevel = threat;
+    } else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock &&
+                decision->getThreatLevel() == ThreatLevel::NO_THREAT) {
+        // If override forces block, set threat level to INFO
+        decision->setThreatLevel(ThreatLevel::THREAT_INFO);
     }
 
     dbgTrace(D_WAAP_OVERRIDE) << "override ids count: " << m_matchedOverrideIds.size();
     // Apply overrides
     for (auto it = m_overridePostFilterMaxScore.begin(); it != m_overridePostFilterMaxScore.end(); it++) {
         const string id = it->first;
-        if (m_overrideState.forceBlockIds.find(id) != m_overrideState.forceBlockIds.end()) {
+        if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.find(id) !=
+            m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.end()) {
             // blocked effectivness is calculates later from the force block exception ids list
             continue;
         }
@@ -2179,34 +2136,41 @@ Waf2Transaction::decideAutonomousSecurity(
         }
     }
 
-    if (m_overrideState.bForceBlock) {
+    if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) {
         dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
             " and override forces REJECT ...";
         if (!decision->shouldBlock()) {
-            m_effectiveOverrideIds.insert(m_overrideState.forceBlockIds.begin(), m_overrideState.forceBlockIds.end());
+            m_effectiveOverrideIds.insert(
+                m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.begin(),
+                m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.end()
+            );
         }
         decision->setBlock(true);
-        if (!m_overrideState.bIgnoreLog)
+        decision->setForceBlock(true);
+        if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog)
         {
-            decision->setOverridesLog(true);
+            decision->setForceLog(true);
         }
     }
-    else if (m_overrideState.bForceException) {
-        dbgTrace(D_WAAP) << "de cideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
+    else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) {
+        dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
             " and override forces ALLOW ...";
         decision->setBlock(false);
-        if (!m_overrideState.bIgnoreLog)
+        decision->setForceAllow(true);
+        if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog)
         {
-            decision->setOverridesLog(true);
+            decision->setForceLog(true);
         }
     } else if (!m_matchedOverrideIds.empty()) {
-        if (!m_overrideState.bIgnoreLog)
+        if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog)
         {
-            decision->setOverridesLog(true);
+            decision->setForceLog(true);
         }
     }
-    dbgTrace(D_WAAP_OVERRIDE) << "force exception: " << m_overrideState.bForceException <<
-    " force block: " << m_overrideState.bForceBlock <<
+    dbgTrace(D_WAAP_OVERRIDE) <<
+    "force exception: " <<
+    m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException <<
+    " force block: " << m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock <<
     " matched overrides count: " << m_matchedOverrideIds.size() <<
     " effective overrides count: " << m_effectiveOverrideIds.size() <<
     " learned overrides count: " << m_exceptionLearned.size();
@@ -2214,8 +2178,9 @@ Waf2Transaction::decideAutonomousSecurity(
     bool log_all = false;
     const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = sitePolicy.get_TriggerPolicy();
     if (triggerPolicy) {
-        const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
-        if (triggerLog && triggerLog->webRequests) log_all = true;
+        auto triggerLog = getTriggerLog(triggerPolicy, AUTONOMOUS_SECURITY_DECISION);
+        if (triggerLog.ok() && triggerLog.unpack().isWebLogFieldActive(LogTriggerConf::WebLogFields::webRequests)) log_all = true;
+        if (triggerLog.ok() && shouldSendExtendedLog(triggerLog.unpack())) m_responseInspectReasons.setCollectResponseForLog(true);
     }
 
     if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) {
@@ -2223,7 +2188,6 @@ Waf2Transaction::decideAutonomousSecurity(
     } else {
         decision->setLog(true);
     }
-
     return decision->shouldBlock();
 }
 
@@ -2243,6 +2207,14 @@ bool Waf2Transaction::shouldInspectResponse()
 {
     return m_responseInspectReasons.shouldInspect() || m_responseInjectReasons.shouldInject();
 }
+bool
+Waf2Transaction::shouldLimitResponseHeadersInspection() {
+    auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL;
+    if (!shouldInspectResponse() && isTriggerReportExists(triggerPolicy)) {
+        return true;
+    }
+    return false;
+}
 bool Waf2Transaction::shouldInjectResponse()
 {
     return m_responseInjectReasons.shouldInject();
@@ -2276,14 +2248,14 @@ bool Waf2Transaction::decideResponse()
         if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) {
             dbgTrace(D_WAAP)
                     << "Waf2Transaction::decideResponse(): got relevant Application configuration from the I/S";
-            m_overrideState = getOverrideState(&ngenSiteConfig);
+            m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(&ngenSiteConfig);
             // Apply overrides
-            if (m_overrideState.bForceBlock) {
+            if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) {
                 dbgTrace(D_WAAP)
                         << "Waf2Transaction::decideResponse(): setting shouldBlock to true due to override";
                 return false; // BLOCK
             }
-            else if (m_overrideState.bForceException) {
+            else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) {
                 dbgTrace(D_WAAP)
                         << "Waf2Transaction::decideResponse(): setting shouldBlock to false due to override";
                 return true; // PASS
@@ -2297,25 +2269,25 @@ bool Waf2Transaction::decideResponse()
             dbgTrace(D_WAAP) << "Trigger policy was not found. Returning true (accept)";
             return true; // accept
         }
-
-        const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
-        if (!triggerLog) {
+        // response is only for WAF
+        auto maybeTriggerLog = getTriggerLog(triggerPolicy, AUTONOMOUS_SECURITY_DECISION);
+        if (!maybeTriggerLog.ok()) {
             dbgTrace(D_WAAP) << "Log trigger configuration was not found. Returning true (accept)";
             return true; // accept
         }
-
+        auto triggerLog = maybeTriggerLog.unpack();
         auto env = Singleton::Consume<I_Environment>::by<Waf2Transaction>();
         auto http_chunk_type = env->get<ngx_http_chunk_type_e>("HTTP Chunk type");
         bool should_send_extended_log = shouldSendExtendedLog(triggerLog) && http_chunk_type.ok();
         if (should_send_extended_log &&
             *http_chunk_type == ngx_http_chunk_type_e::RESPONSE_CODE &&
-            !triggerLog->responseBody
+            !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
         ) {
             should_send_extended_log = false;
         } else if (should_send_extended_log &&
             *http_chunk_type == ngx_http_chunk_type_e::REQUEST_END &&
-            !triggerLog->responseCode &&
-            !triggerLog->responseBody
+            !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode) &&
+            !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
         ) {
             should_send_extended_log = false;
         }
@@ -2347,83 +2319,116 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) {
 
 bool
 Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
-    auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
-    if (!exceptions.ok()) {
-        dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
-        return false;
-    }
-    dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
+
+    dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions for should ignore";
 
     std::unordered_map<std::string, std::set<std::string>> exceptions_dict;
+    std::set<ParameterBehavior> behaviors;
+    std::set<std::string> &ignored_keywords = getAssetState()->m_filtersMngr->getMatchedOverrideKeywords();
 
-    if (res.location != "referer") {
-        // collect param name
-        exceptions_dict["paramName"].insert(res.param_name);
-        exceptions_dict["paramName"].insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this));
+    // collect param name
+    exceptions_dict["paramName"].insert(res.param_name);
+    exceptions_dict["paramName"].insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this));
 
-        std::set<std::string> param_name_set;
-        param_name_set.insert(res.param_name);
-        param_name_set.insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this));
+    std::set<std::string> param_name_set;
+    param_name_set.insert(res.param_name);
+    param_name_set.insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this));
 
-        // collect param value
-        exceptions_dict["paramValue"].insert(res.unescaped_line);
+    // collect param value
+    exceptions_dict["paramValue"].insert(res.unescaped_line);
 
-        // collect param location
-        exceptions_dict["paramLocation"].insert(res.location);
+    // collect param location
+    exceptions_dict["paramLocation"].insert(res.location);
 
-        ScopedContext ctx;
-        ctx.registerValue<std::string>("paramValue", res.unescaped_line);
-        ctx.registerValue<std::set<std::string>>("paramName", param_name_set);
+    ScopedContext ctx;
+    ctx.registerValue<std::string>("paramValue", res.unescaped_line);
+    ctx.registerValue<std::set<std::string>>("paramName", param_name_set);
 
-        // collect sourceip, sourceIdentifier, url
-        exceptions_dict["sourceIP"].insert(m_remote_addr);
-        exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
-        exceptions_dict["url"].insert(getUriStr());
-        exceptions_dict["hostName"].insert(m_hostStr);
-        exceptions_dict["method"].insert(m_methodStr);
+    // collect sourceip, sourceIdentifier, url
+    exceptions_dict["sourceIP"].insert(m_remote_addr);
+    exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
+    exceptions_dict["url"].insert(getUriStr());
+    exceptions_dict["hostName"].insert(m_hostStr);
+    exceptions_dict["method"].insert(m_methodStr);
 
-        for (auto &keyword : res.keyword_matches) {
-            exceptions_dict["indicator"].insert(keyword);
-        }
-        for (auto &it : res.found_patterns) {
-            exceptions_dict["indicator"].insert(it.first);
-        }
+    for (auto &keyword : res.keyword_matches) {
+        exceptions_dict["indicator"].insert(keyword);
+    }
+    for (auto &it : res.found_patterns) {
+        exceptions_dict["indicator"].insert(it.first);
+    }
 
-        // calling behavior and check if there is a behavior that match to this specific param name.
-        auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
-                getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
-        for (const auto &behavior : behaviors) {
-            dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
-            if (!res.filtered_keywords.empty() || res.score > 0) {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
-                std::string overrideId = behavior.getId();
-                if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
-                    m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
-                    m_overridePostFilterMaxScore[overrideId] = res.score;
-                } else {
-                    if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) {
-                        m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
-                    }
-                    if (res.score > m_overridePostFilterMaxScore[overrideId]) {
-                        m_overridePostFilterMaxScore[overrideId] = res.score;
-                    }
-                }
-                if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) {
-                    m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter;
-                }
-            }
-            if (behavior == action_ignore)
-            {
-                dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
-                std::string overrideId = behavior.getId();
-                if (!overrideId.empty()) {
-                    m_matchedOverrideIds.insert(overrideId);
-                }
-                return true;
-            }
+    bool isConfigExist = false;
+    if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
+        dbgTrace(D_WAAP_OVERRIDE) << "waap api config found";
+        m_siteConfig = &m_ngenAPIConfig;
+        isConfigExist = true;
+    } else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) {
+        dbgTrace(D_WAAP_OVERRIDE) << "waap web application config found";
+        m_siteConfig = &m_ngenSiteConfig;
+        isConfigExist = true;
+    }
+    std::vector<std::string> site_exceptions;
+    if (isConfigExist) {
+        dbgTrace(D_WAAP_OVERRIDE) << "config exists, get override policy";
+        std::shared_ptr<Waap::Override::Policy> overridePolicy = m_siteConfig->get_OverridePolicy();
+        if (overridePolicy) {
+            site_exceptions = overridePolicy->getExceptionsByPractice()
+                    .getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION);
         }
     }
 
+    if (!site_exceptions.empty()) {
+        dbgTrace(D_WAAP_OVERRIDE) << "get behaviors by web app practice";
+        I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<Waf2Transaction>();
+        for (const auto &id : site_exceptions) {
+            dbgTrace(D_WAAP_OVERRIDE) << "get parameter exception: " << id;
+            auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict, ignored_keywords);
+            behaviors.insert(params.begin(), params.end());
+        }
+    } else {
+        auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
+        if (!exceptions.ok()) {
+            dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
+            return false;
+        }
+        // calling behavior and check if there is a behavior that match to this specific param name.
+        behaviors = exceptions.unpack().getBehavior(exceptions_dict, ignored_keywords);
+    }
+    dbgTrace(D_WAAP_OVERRIDE) << "got "<< behaviors.size() << " behaviors and " <<
+        ignored_keywords.size() << " ignored keywords";
+
+    for (const auto &behavior : behaviors) {
+        dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
+        if (!res.filtered_keywords.empty() || res.score > 0) {
+            dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
+            std::string overrideId = behavior.getId();
+            if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
+                m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
+                m_overridePostFilterMaxScore[overrideId] = res.score;
+            } else {
+                if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) {
+                    m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
+                }
+                if (res.score > m_overridePostFilterMaxScore[overrideId]) {
+                    m_overridePostFilterMaxScore[overrideId] = res.score;
+                }
+            }
+            if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) {
+                m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter;
+            }
+        }
+        if (behavior == action_ignore)
+        {
+            dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
+            std::string overrideId = behavior.getId();
+            if (!overrideId.empty()) {
+                m_matchedOverrideIds.insert(overrideId);
+            }
+            return true;
+        }
+    }
+    dbgTrace(D_WAAP_OVERRIDE) << "should not ignore";
     return false;
 }
 
@@ -2485,9 +2490,15 @@ void Waf2Transaction::collectFoundPatterns()
     }
 }
 
-bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger::Log> &trigger_log) const
+bool Waf2Transaction::shouldSendExtendedLog(const LogTriggerConf &trigger_log, DecisionType practiceType) const
 {
-    if (!trigger_log->extendLogging)
+    if (practiceType != AUTONOMOUS_SECURITY_DECISION)
+    {
+        dbgTrace(D_WAAP) << "Should not send extended logging. Practice type is: " << practiceType;
+        return false;
+    }
+    auto extend_logging_severity = trigger_log.getExtendLoggingSeverity();
+    if(extend_logging_severity == LogTriggerConf::extendLoggingSeverity::None)
     {
         dbgTrace(D_WAAP) << "Should not send extended log. Extended log is disabled.";
         return false;
@@ -2498,7 +2509,7 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger:
     ReportIS::Severity severity = Waap::Util::computeSeverityFromThreatLevel(
         autonomousSecurityDecision->getThreatLevel());
 
-    if (trigger_log->extendLoggingMinSeverity == "Critical" || trigger_log->extendLoggingMinSeverity == "critical")
+    if (extend_logging_severity == LogTriggerConf::extendLoggingSeverity::Critical)
     {
         if (severity == ReportIS::Severity::CRITICAL)
         {
@@ -2508,7 +2519,7 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger:
         dbgTrace(D_WAAP) << "Should not send extended logging. Min Severity Critical. Severity: " << (int) severity;
         return false;
     }
-    else if (trigger_log->extendLoggingMinSeverity == "High" || trigger_log->extendLoggingMinSeverity == "high")
+    else if (extend_logging_severity == LogTriggerConf::extendLoggingSeverity::High)
     {
         if (severity == ReportIS::Severity::CRITICAL || severity == ReportIS::Severity::HIGH)
         {
@@ -2519,6 +2530,19 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger:
         return false;
     }
 
-    dbgTrace(D_WAAP) << "Should not send extended logging. Min Severity: " << trigger_log->extendLoggingMinSeverity;
+    dbgTrace(D_WAAP) << "Should not send extended logging. Min Severity: " << (int) extend_logging_severity;
     return false;
 }
+
+
+void
+Waf2Transaction::setTemperatureDetected(bool detected)
+{
+    m_temperature_detected = detected;
+}
+
+bool
+Waf2Transaction::wasTemperatureDetected() const
+{
+    return m_temperature_detected;
+}

components/security_apps/waap/waap_clib/Waf2Engine.h

@@ -22,17 +22,22 @@
 #include "PatternMatcher.h"
 #include "generic_rulebase/rulebase_config.h"
 #include "generic_rulebase/evaluators/trigger_eval.h"
+#include "i_generic_rulebase.h"
+#include "generic_rulebase/parameters_config.h"
 #include "Waf2Util.h"
 #include "WaapConfigApplication.h"
 #include "WaapConfigApi.h"
 #include "WaapDecision.h"
+#include "DecisionType.h"
 #include "DeepAnalyzer.h"
 #include <vector>
+#include <set>
 #include <map>
 #include <string>
 #include <set>
 #include <utility>
 #include <memory>
+#include <unordered_map>
 #include <boost/uuid/uuid.hpp>            // uuid class
 #include <boost/uuid/uuid_generators.hpp> // uuid generators
 #include <boost/tokenizer.hpp>
@@ -65,7 +70,8 @@ class Waf2Transaction :
     private boost::noncopyable,
     Singleton::Consume<I_AgentDetails>,
     Singleton::Consume<I_TimeGet>,
-    Singleton::Consume<I_Environment>
+    Singleton::Consume<I_Environment>,
+    Singleton::Consume<I_GenericRulebase>
 {
 public:
     Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState);
@@ -131,6 +137,7 @@ public:
     ngx_http_cp_verdict_e getUserLimitVerdict();
     const std::string getUserLimitVerdictStr() const;
     const std::string getViolatedUserLimitTypeStr() const;
+    const std::string getCurrentWebUserResponse();
 
     virtual HeaderType detectHeaderType(const char* name, int name_len);
     HeaderType checkCleanHeader(const char* name, int name_len, const char* value, int value_len) const;
@@ -195,7 +202,9 @@ public:
     void handleSecurityHeadersInjection(std::vector<std::pair<std::string, std::string>>& injectHeaderStrs);
     void disableShouldInjectSecurityHeaders();
 
-    bool shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger::Log> &trigger_log) const;
+    bool shouldSendExtendedLog(
+        const LogTriggerConf &trigger_log, DecisionType practiceType = AUTONOMOUS_SECURITY_DECISION
+    ) const;
 
     // query
     virtual bool isSuspicious() const;
@@ -232,24 +241,26 @@ public:
     // LCOV_EXCL_START Reason: This function is tested in system tests
     bool checkIsHeaderOverrideScanRequired();
     // LCOV_EXCL_STOP
+    bool shouldLimitResponseHeadersInspection();
+
+    void setTemperatureDetected(bool detected);
+    bool wasTemperatureDetected() const;
 
 private:
-    int finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock);
-    const std::shared_ptr<Waap::Trigger::Log> getTriggerLog(const std::shared_ptr<Waap::Trigger::Policy>&
-        triggerPolicy) const;
+    const Maybe<LogTriggerConf, Config::Errors> getTriggerLog(const std::shared_ptr<Waap::Trigger::Policy>&
+        triggerPolicy, DecisionType practiceType) const;
     bool isTriggerReportExists(const std::shared_ptr<Waap::Trigger::Policy> &triggerPolicy);
     void sendAutonomousSecurityLog(
-        const std::shared_ptr<Waap::Trigger::Log>& triggerLog,
+        const LogTriggerConf& triggerLog,
         bool shouldBlock,
         const std::string& logOverride,
         const std::string& attackTypes) const;
     void appendCommonLogFields(LogGen& waapLog,
-        const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
+        const LogTriggerConf &triggerLog,
         bool shouldBlock,
         const std::string& logOverride,
         const std::string& incidentType,
-        const std::string& practiceID,
-        const std::string& practiceName) const;
+        DecisionType practiceType) const;
     std::string getUserReputationStr(double relativeReputation) const;
     bool isTrustedSource() const;
 
@@ -258,6 +269,12 @@ private:
     bool setCurrentAssetContext();
     bool checkIsScanningRequired();
     Waap::Override::State getOverrideState(IWaapConfig* sitePolicy);
+    std::set<ParameterBehavior> getBehaviors(
+        const std::unordered_map<std::string, std::set<std::string>> &exceptions_dict,
+        const std::vector<std::string>& exceptions, bool checkResponse);
+    std::unordered_map<std::string, std::set<std::string>> getExceptionsDict(DecisionType practiceType);
+    bool shouldEnforceByPracticeExceptions(DecisionType practiceType);
+    void setOverrideState(const std::set<ParameterBehavior>& behaviors, Waap::Override::State& state);
 
     // User limits functions
     void createUserLimitsState();
@@ -360,15 +377,16 @@ private:
     Waap::ResponseInspectReasons m_responseInspectReasons;
     Waap::ResponseInjectReasons m_responseInjectReasons;
     WaapDecision m_waapDecision;
-    Waap::Override::State m_overrideState;
+    std::unordered_map<int, Waap::Override::State> m_overrideStateByPractice;
+    std::string m_practiceSubType;
 
     uint64_t m_index;
 
-    // Cached pointer to const triggerLog (hence mutable)
-    mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
     bool m_triggerReport;
     bool is_schema_validation = false;
     Waf2TransactionFlags m_waf2TransactionFlags;
+
+    bool m_temperature_detected = false; // Tracks if temperature was detected
 };
 
 #endif // __WAF2_TRANSACTION_H__99e4201a

components/security_apps/waap/waap_clib/Waf2EngineGetters.cc

@@ -17,7 +17,6 @@
 #include <boost/uuid/uuid_generators.hpp> // uuid generators
 #include <boost/uuid/uuid_io.hpp>
 #include <boost/algorithm/string.hpp>
-#include "generic_rulebase/triggers_config.h"
 #include "config.h"
 #include "LogGenWrapper.h"
 #include <memory>
@@ -343,7 +342,7 @@ Waap::CSRF::State& Waf2Transaction::getCsrfState()
 }
 
 void Waf2Transaction::sendAutonomousSecurityLog(
-    const std::shared_ptr<Waap::Trigger::Log>& triggerLog,
+    const LogTriggerConf& triggerLog,
     bool shouldBlock,
     const std::string& logOverride,
     const std::string& attackTypes) const
@@ -352,11 +351,11 @@ void Waf2Transaction::sendAutonomousSecurityLog(
         m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
     ReportIS::Severity severity = Waap::Util::computeSeverityFromThreatLevel(
         autonomousSecurityDecision->getThreatLevel());
-    if (autonomousSecurityDecision->getOverridesLog() && logOverride == OVERRIDE_DROP)
+    if (autonomousSecurityDecision->shouldForceLog() && logOverride == OVERRIDE_DROP)
     {
         severity = ReportIS::Severity::MEDIUM;
     }
-    else if (autonomousSecurityDecision->getOverridesLog() && logOverride == OVERRIDE_ACCEPT)
+    else if (autonomousSecurityDecision->shouldForceLog() && logOverride == OVERRIDE_ACCEPT)
     {
         severity = ReportIS::Severity::INFO;
     }
@@ -381,11 +380,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
         waap_log << LogField("eventConfidence", confidence);
     }
 
-    appendCommonLogFields(
-        waap_log, triggerLog, shouldBlock, logOverride, attackTypes,
-        m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
-        m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
-    );
+    appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, attackTypes, AUTONOMOUS_SECURITY_DECISION);
 
     std::string sampleString = getSample();
     if (sampleString.length() > MAX_LOG_FIELD_SIZE) {
@@ -470,24 +465,36 @@ Waf2Transaction::getUserLimitVerdict()
     auto decision = m_waapDecision.getDecision(USER_LIMITS_DECISION);
     if (mode == AttackMitigationMode::LEARNING) {
         decision->setLog(true);
-        decision->setBlock(false);
-        if (isIllegalMethodViolation()) {
-            dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
-            verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
-        }
-        else {
-            dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason;
-            verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+        if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) {
+            // detect mode and no active drop exception
+            decision->setBlock(false);
+            if (isIllegalMethodViolation()) {
+                dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
+                verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
+            }
+            else {
+                dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason;
+                verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+            }
+        } else {
+            // detect mode and active drop exception
+            decision->setBlock(true);
+            decision->setForceBlock(true);
+            dbgInfo(D_WAAP_ULIMITS) << msg << "Override Block" << reason;
+            verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
         }
     }
     else if (mode == AttackMitigationMode::PREVENT) {
         decision->setLog(true);
-        if (!m_overrideState.bForceException) {
+        if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) {
+            // prevent mode and no active accept exception
             decision->setBlock(true);
             dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
             verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
         } else {
-            decision->setBlock(true);
+            // prevent mode and active accept exception
+            decision->setBlock(false);
+            decision->setForceAllow(true);
             dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
             verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
         }
@@ -585,20 +592,206 @@ bool Waf2Transaction::checkIsHeaderOverrideScanRequired()
     return m_isHeaderOverrideScanRequired;
 }
 
+const std::string Waf2Transaction::getCurrentWebUserResponse() {
+    dbgFlow(D_WAAP);
+    m_waapDecision.orderDecisions();
+    DecisionType practiceType = m_waapDecision.getHighestPriorityDecisionToLog();
+
+    dbgTrace(D_WAAP) << "set current web user response for practice: " << practiceType;
+
+    const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = m_siteConfig->get_TriggerPolicy();
+    if (!triggerPolicy) {
+        dbgTrace(D_WAAP) << "No trigger policy, can't set web user response for practice: " << practiceType;
+        return "";
+    }
+    auto responses = triggerPolicy->responseByPractice.getResponseByPractice(practiceType);
+    if (responses.empty()) {
+        dbgTrace(D_WAAP) << "No web user response for practice: " << practiceType;
+        return "";
+    }
+    dbgTrace(D_WAAP) << "Found web user response trigger by practice, ID: " << responses[0];
+    return responses[0];
+}
+
+std::unordered_map<std::string, std::set<std::string>>
+Waf2Transaction::getExceptionsDict(DecisionType practiceType) {
+    std::unordered_map<std::string, std::set<std::string>> exceptions_dict;
+    exceptions_dict["url"].insert(m_uriPath);
+    exceptions_dict["sourceIP"].insert(m_remote_addr);
+    exceptions_dict["method"].insert(m_methodStr);
+
+    extractEnvSourceIdentifier();
+    exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
+    if (
+        practiceType == DecisionType::AUTONOMOUS_SECURITY_DECISION
+    ) {
+        for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) {
+            exceptions_dict["paramName"].insert(keywordInfo.getName());
+        }
+    }
+    if (practiceType == DecisionType::AUTONOMOUS_SECURITY_DECISION) {
+        exceptions_dict["hostName"].insert(m_hostStr);
+        for (const std::string& keywordStr : getKeywordMatches()) {
+            exceptions_dict["indicator"].insert(keywordStr);
+        }
+        for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) {
+            exceptions_dict["paramValue"].insert(keywordInfo.getValue());
+        }
+        exceptions_dict["paramLocation"].insert(getLocation());
+        if (!checkIsHeaderOverrideScanRequired()) {
+            dbgDebug(D_WAAP) << "Header name override scan is not required";
+        } else {
+            for (auto& hdr_pair : getHdrPairs()) {
+                std::string name = hdr_pair.first;
+                std::transform(name.begin(), name.end(), name.begin(), ::tolower);
+                exceptions_dict["headerName"].insert(name);
+                std::string value = hdr_pair.second;
+                std::transform(value.begin(), value.end(), value.begin(), ::tolower);
+                exceptions_dict["headerValue"].insert(value);
+            }
+        }
+    }
+    return exceptions_dict;
+}
+
+std::set<ParameterBehavior>
+Waf2Transaction::getBehaviors(
+    const std::unordered_map<std::string, std::set<std::string>> &exceptions_dict,
+    const std::vector<std::string>& exceptions, bool checkResponse = false)
+{
+    std::set<ParameterBehavior> all_params;
+    I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<Waf2Transaction>();
+    for (const auto &id : exceptions) {
+        dbgTrace(D_WAAP) << "get parameter exception for: " << id;
+        auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict);
+
+        if (checkResponse && !getResponseBody().empty()) {
+            std::unordered_map<std::string, std::set<std::string>> response_dict = {
+                {"responseBody", {getResponseBody()}}
+            };
+            auto params = i_rulebase->getParameterException(id).getBehavior(response_dict);
+            if (params.size() > 0) {
+                dbgTrace(D_WAAP) << "got responseBody behavior, setApplyOverride(true)";
+                m_responseInspectReasons.setApplyOverride(true);
+                all_params.insert(params.begin(), params.end());
+                // once found, no need to check again
+                checkResponse = false;
+            }
+        }
+
+        dbgTrace(D_WAAP) << "got "<< params.size() << " behaviors";
+        all_params.insert(params.begin(), params.end());
+    }
+    return all_params;
+}
+
+
+bool Waf2Transaction::shouldEnforceByPracticeExceptions(DecisionType practiceType)
+{
+    dbgFlow(D_WAAP);
+    auto decision = m_waapDecision.getDecision(practiceType);
+    bool shouldEnforce = false;
+    std::shared_ptr<Waap::Override::Policy> overridePolicy = m_siteConfig->get_OverridePolicy();
+    if (overridePolicy) {
+        auto exceptions = overridePolicy->getExceptionsByPractice().getExceptionsOfPractice(practiceType);
+
+        if (!exceptions.empty()) {
+            dbgTrace(D_WAAP) << "get behaviors for practice: " << practiceType;
+
+            auto behaviors = getBehaviors(getExceptionsDict(practiceType), exceptions);
+            if (behaviors.size() > 0)
+            {
+                auto it = m_overrideStateByPractice.find(practiceType);
+                if (it == m_overrideStateByPractice.end()) {
+                    dbgWarning(D_WAAP) << "no override state for practice: " << practiceType;
+                    return false;
+                }
+                setOverrideState(behaviors, it->second);
+                if (it->second.bForceBlock) {
+                    dbgTrace(D_WAAP)
+                    << "should block by exceptions for practice: " << practiceType;
+                    decision->setBlock(true);
+                    decision->setForceBlock(true);
+                    shouldEnforce = true;
+                }
+                if (it->second.bForceException) {
+                    dbgTrace(D_WAAP)
+                    << "should not block by exceptions for practice: " << practiceType;
+                    decision->setBlock(false);
+                    decision->setForceAllow(true);
+                    shouldEnforce = true;
+                }
+            }
+        }
+    }
+    if (shouldEnforce) {
+        decision->setLog(true);
+        if(!m_overrideStateByPractice[practiceType].bSupressLog) {
+            decision->setForceLog(true);
+        }
+        std::shared_ptr<AutonomousSecurityDecision> autonomousDecision =
+            std::dynamic_pointer_cast<AutonomousSecurityDecision>(
+                m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)
+            );
+        if (autonomousDecision->getThreatLevel() <= ThreatLevel::THREAT_INFO) {
+            autonomousDecision->setLog(false);
+        }
+    } else if(!m_matchedOverrideIds.empty() && !m_overrideStateByPractice[practiceType].bSupressLog) {
+        decision->setForceLog(true);
+    }
+    return shouldEnforce;
+}
+
+void Waf2Transaction::setOverrideState(const std::set<ParameterBehavior>& behaviors, Waap::Override::State& state) {
+    dbgFlow(D_WAAP) << "setOverrideState(): from exceptions per practice";
+    for (auto const &behavior : behaviors) {
+        m_matchedOverrideIds.insert(behavior.getId());
+        if (behavior.getKey() == BehaviorKey::ACTION) {
+            if (behavior.getValue() == BehaviorValue::ACCEPT) {
+                dbgTrace(D_WAAP) << "setting bForceException due to override behavior: " << behavior.getId();
+                state.bForceException = true;
+                state.forceExceptionIds.insert(behavior.getId());
+            } else if (behavior.getValue() == BehaviorValue::REJECT) {
+                dbgTrace(D_WAAP) << "setting bForceBlock due to override behavior: " << behavior.getId();
+                state.bForceBlock = true;
+                state.forceBlockIds.insert(behavior.getId());
+            }
+        } else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) {
+            dbgTrace(D_WAAP) << "setting bSupressLog due to override behavior: " << behavior.getId();
+            state.bSupressLog = true;
+        }
+    }
+}
+
 Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
 {
     Waap::Override::State overrideState;
     std::shared_ptr<Waap::Override::Policy> overridePolicy = sitePolicy->get_OverridePolicy();
     if (overridePolicy) { // at first we will run request overrides (in order to set the source)
+        auto exceptions = overridePolicy->getExceptionsByPractice().
+            getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION);
+        if (!exceptions.empty()) {
+            dbgTrace(D_WAAP) << "get behaviors for override state";
+            m_responseInspectReasons.setApplyOverride(false);
+            auto behaviors = getBehaviors(getExceptionsDict(AUTONOMOUS_SECURITY_DECISION), exceptions, true);
+            if (behaviors.size() > 0) {
+                dbgTrace(D_WAAP) << "set override state by practice found behaviors";
+                setOverrideState(behaviors, m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]);
+            }
+            m_isHeaderOverrideScanRequired = false;
+            return m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION];
+        }
         m_responseInspectReasons.setApplyOverride(overridePolicy->isOverrideResponse());
         overrideState.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, true);
     }
 
     if (overridePolicy) { // later we will run response overrides
-        m_overrideState.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false);
+        m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].applyOverride(
+            *overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false
+        );
     }
     m_isHeaderOverrideScanRequired = false;
-    return m_overrideState;
+    return m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION];
 }
 
 Waf2TransactionFlags &Waf2Transaction::getTransactionFlags()
@@ -606,22 +799,32 @@ Waf2TransactionFlags &Waf2Transaction::getTransactionFlags()
     return m_waf2TransactionFlags;
 }
 
-const std::shared_ptr<Waap::Trigger::Log> Waf2Transaction::getTriggerLog(const std::shared_ptr<
-    Waap::Trigger::Policy> &triggerPolicy) const
+const Maybe<LogTriggerConf, Config::Errors> Waf2Transaction::getTriggerLog(const std::shared_ptr<
+    Waap::Trigger::Policy> &triggerPolicy, DecisionType practiceType) const
 {
-    // Trigger log already known (no need to extract it second time)
-    if (m_triggerLog) {
-        return m_triggerLog;
-    }
-
-    // Walk over trigger logs and choose the last one of type Log
-    for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
-        if (trigger.triggerType == "log") {
-            m_triggerLog = trigger.log;
+    dbgTrace(D_WAAP) << "Getting log trigger for practice type:" << practiceType;
+    std::set<std::string> triggers_set;
+    auto triggers = triggerPolicy->triggersByPractice.getTriggersByPractice(practiceType);
+    if (!triggers.empty()) {
+        for (const auto &id : triggers) {
+            triggers_set.insert(id);
+            dbgTrace(D_WAAP) << "Add log trigger by practice to triggers set:" << id;
+        }
+    } else {
+        for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
+            triggers_set.insert(trigger.triggerId);
+            dbgTrace(D_WAAP) << "Add log trigger waap to triggers set:" << trigger.triggerId;
         }
     }
 
-    return m_triggerLog;
+    ScopedContext ctx;
+    ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
+    auto trigger_config = getConfiguration<LogTriggerConf>("rulebase", "log");
+
+    if (!trigger_config.ok()) {
+        dbgError(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr();
+    }
+    return trigger_config;
 }
 
 bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr<
@@ -633,12 +836,31 @@ bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr<
     if (m_triggerReport) {
         return m_triggerReport;
     }
-    for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
-        if (trigger.triggerType == "report") {
-            return m_triggerReport = true;
+    std::set<std::string> triggers_set;
+    auto triggers = triggerPolicy->triggersByPractice.getAllTriggers();
+    if (!triggers.empty()) {
+        for (const auto &id : triggers) {
+            triggers_set.insert(id);
         }
     }
-    return m_triggerReport;
+    else {
+        for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
+            triggers_set.insert(trigger.triggerId);
+        }
+    }
+
+    ScopedContext ctx;
+    ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
+    auto trigger_config = getConfiguration<ReportTriggerConf>("rulebase", "report");
+    if (!trigger_config.ok()) {
+        dbgWarning(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr();
+        m_triggerReport = false;
+        return false;
+    }
+    auto triggerName = trigger_config.unpack().getName();
+    dbgTrace(D_WAAP) << "Got report trigger from rulbase: " << triggerName << ", m_triggerReport = true";
+    m_triggerReport = true;
+    return true;
 }
 
 ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -642,7 +642,7 @@ void unescapeUnicode(string& text) {
         STATE_ESCAPE_X
     } state = STATE_COPY;
 
-    for (; it != text.end(); ++it) {
+    for (; it != text.end() && result <= it; ++it) {
         const char ch = *it;
 
         switch (state) {
@@ -682,16 +682,10 @@ void unescapeUnicode(string& text) {
                 state = STATE_ESCAPE_U;
             }
             else if (ch == 'x') {
-#if 1
                 digitsAnticipated = 1; // anticipate at least one HEX digit after \x
                 code = 0;
                 nonZeroHexCounter = 0;
                 state = STATE_ESCAPE_X;
-#else
-                digitsAnticipated = 2; // parse/skip 2 hex digits
-                code = 0;
-                state = STATE_ESCAPE_U;
-#endif
             }
             else {
                 // this is invalid escape sequence: rollback and copy this character too
@@ -794,10 +788,12 @@ void unescapeUnicode(string& text) {
             *result++ = ch;
         }
 
+        dbgAssertOpt(!pAcc || size_t(pAcc - acc) < sizeof(acc))
+            << AlertInfo(AlertTeam::WAAP, "WAAP string unescape") <<
+            "Buffer overflow detected";
         // Accumulate
-        if (pAcc) {
+        if (pAcc && size_t(pAcc - acc) < sizeof(acc)) {
             // Ensure we don't have buffer overflow
-            assert(size_t(pAcc - acc) < sizeof(acc));
             *pAcc++ = ch;
         }
     }
@@ -806,7 +802,7 @@ void unescapeUnicode(string& text) {
         digitsAnticipated << ", acc='" << string(acc, pAcc ? (int)(pAcc - acc) : 0) << "'";
 
     // Output code if we just finished decoding an escape sequence succesully and reached end of string
-    if (state == STATE_ESCAPE_U && digitsAnticipated == 0) {
+    if (state == STATE_ESCAPE_U && digitsAnticipated == 0 && result < text.end()) {
         // only output ASCII codes <= 127. "swallow" all unicode.
         if (code <= 127) {
             *result++ = (char)code;
@@ -823,7 +819,7 @@ void unescapeUnicode(string& text) {
         if (isSpecialUnicode(code)) {
             *result++ = convertSpecialUnicode(code);
         }
-        else
+        else if (digitsAnticipated == 0)
         {
             *result++ = (char)code;
         }
@@ -831,7 +827,7 @@ void unescapeUnicode(string& text) {
 
     // flush any accumulated left-overs into output buffer
     if (pAcc) {
-        for (p = acc; p < pAcc; p++) {
+        for (p = acc; p < pAcc && result < text.end(); p++) {
             *result++ = *p;
         }
     }
@@ -1034,8 +1030,7 @@ base64_decode_status decideStatusBase64Decoded(
                     && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD
                     && !called_with_prefix
                     && decoded.size() > BASE64_MIN_SIZE_LIMIT
-                    && decoded.size() < BASE64_MAX_SIZE_LIMIT
-                    && terminatorCharsSeen != 0;
+                    && decoded.size() < BASE64_MAX_SIZE_LIMIT;
                 if (!empiric_condition) {
                     if (clear_on_error) decoded.clear();
                     return B64_DECODE_SUSPECTED;
@@ -1070,7 +1065,8 @@ base64_decode_status decideStatusBase64Decoded(
                 && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD
                 && !called_with_prefix
                 && decoded.size() > BASE64_MIN_SIZE_LIMIT
-                && decoded.size() < BASE64_MAX_SIZE_LIMIT;
+                && decoded.size() < BASE64_MAX_SIZE_LIMIT
+                && nonPrintableCharsCount != 0;
             if (empiric_condition) {
                 dbgTrace(D_WAAP_BASE64) << "Empiric test failed, non-base64 chunk, return B64_DECODE_INVALID";
                 decoded.clear();
@@ -1372,10 +1368,23 @@ static const SingleRegex base64_key_detector_re(
     "^[^<>{};,&\\?|=\\s]+={1}",
     err,
     "base64_key");
+// Matches valid Base64 prefixes: "data:" URIs with optional parameters or bare "base64,"
+// strings. Ensures correct MIME types, strict Base64 encoding with padding, and anchoring
+// at string-start or after '=' (with optional whitespace). Prevents invalid injections.
 static const SingleRegex base64_prefix_detector_re(
-    "data:\\S*;base64,\\S+|base64,\\S+",
+    "^data:[a-zA-Z0-9.+-]+/[a-zA-Z0-9.+-]+(?:;[a-zA-Z0-9.+-]+=[a-zA-Z0-9.+-]+)*;base64,[A-Za-z0-9+/]+"
+    "={0,2}|^base64,[A-Za-z0-9+/]+={0,2}|=\\s*data:[a-zA-Z0-9.+-]+/[a-zA-Z0-9.+-]+(?:;[a-zA-Z0-9.+-]+"
+    "=[a-zA-Z0-9.+-]+)*;base64,[A-Za-z0-9+/]+={0,2}|=\\s*base64,[A-Za-z0-9+/]+={0,2}",
     err,
     "base64_prefix");
+static const SingleRegex percent_encoding_re(
+    "%[A-Fa-f0-9]{2}",
+    err,
+    "percent_encoding");
+static const SingleRegex nosql_key_evasion_detector_re(
+    "\\w{1,48}\\s*[\\[\\{\\(]\\s*\\$\\w{1,48}\\s*[\\]\\}\\)]",
+    err,
+    "nosql_key_evasion_detector");
 
 // looks for combination <param>={<some text>*:<some text>*}
 //used to allow parsing param=JSON to reduce false positives
@@ -1909,6 +1918,16 @@ containsInvalidUtf8(const string &payload)
     return invalid_hex_evasion_re.hasMatch(payload);
 }
 
+bool
+containsPercentEncoding(const string &payload)
+{
+    static const size_t min_matches = 2;
+    vector<RegexMatch> regex_matches;
+    size_t counter = percent_encoding_re.findAllMatches(payload, regex_matches, min_matches);
+    // Check if there are at least two matches
+    return counter >= min_matches;
+}
+
 string
 unescapeInvalidUtf8(const string &payload)
 {
@@ -2183,6 +2202,10 @@ void decodeUtf16Value(const ValueStatsAnalyzer &valueStats, string &cur_val)
     cur_val = utf8Out;
 }
 
+bool testNoSQLKeySuspect(const string &key) {
+    return Waap::Util::nosql_key_evasion_detector_re.hasMatch(key);
+}
+
 bool testUrlBareUtf8Evasion(const string &line) {
     size_t percentPos = 0;
 

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -230,7 +230,7 @@ _IT escape_backslashes(_IT first, _IT last) {
     _IT src = first;
     _IT dst = first;
     _IT mark = first;
-    
+
     enum { STATE_COPY, STATE_ESCAPE, STATE_OCTAL, STATE_HEX } state = STATE_COPY;
     unsigned char accVal = 0;
     unsigned char digitsCount = 0;
@@ -1137,6 +1137,8 @@ namespace Util {
 
     bool containsInvalidUtf8(const std::string &payload);
 
+    bool containsPercentEncoding(const std::string &payload);
+
     // based on invalid utf-8 evasion from here: https://www.cgisecurity.com/lib/URLEmbeddedAttacks.html
     std::string unescapeInvalidUtf8(const std::string &text);
 
@@ -1145,6 +1147,8 @@ namespace Util {
 
     bool containsCspReportPolicy(const std::string &payload);
 
+    bool testNoSQLKeySuspect(const std::string &key);
+
     bool testUrlBareUtf8Evasion(const std::string &line);
     bool testUrlBadUtf8Evasion(const std::string &line);
 

components/security_apps/waap/waap_component_impl.cc

@@ -48,6 +48,7 @@ WaapComponent::Impl::Impl() :
     pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT),
     accept_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT),
     drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
+    limit_response_headers(ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS),
     waapStateTable(NULL),
     transactionsCount(0),
     deepAnalyzer()
@@ -246,7 +247,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event)
 
     // Delete state before returning any verdict which is not pending
     if ((verdict.getVerdict() != pending_response.getVerdict()) && waapStateTable->hasState<Waf2Transaction>()) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, verdict);
     } else {
     }
 
@@ -283,8 +284,9 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event)
     waf2Transaction.add_request_body_chunk(dataBuf, dataBufLen);
 
     ngx_http_cp_verdict_e verdict = waf2Transaction.getUserLimitVerdict();
+    EventVerdict eventVedict(verdict);
     if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, eventVedict);
     }
 
     return EventVerdict(verdict);
@@ -323,9 +325,10 @@ WaapComponent::Impl::respond(const EndRequestEvent &)
 
     // Delete state before returning any verdict which is not pending
     if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
+        verdict.getVerdict() != ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS &&
         waapStateTable->hasState<Waf2Transaction>()
     ) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, verdict);
     }
 
     return verdict;
@@ -374,7 +377,7 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event)
         verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
         waapStateTable->hasState<Waf2Transaction>()
     ) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, verdict);
     }
 
     return verdict;
@@ -412,6 +415,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
 
     ngx_http_cp_verdict_e verdict = pending_response.getVerdict();
     HttpHeaderModification modifications;
+    std::string webUserResponseByPractice;
     bool isSecurityHeadersInjected = false;
 
     if (waf2Transaction.shouldInjectSecurityHeaders()) {
@@ -477,16 +481,16 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
         // disable should inject security headers after injection to avoid response body scanning when it's unnecessary
         waf2Transaction.disableShouldInjectSecurityHeaders();
     }
-
+    EventVerdict eventVedict(move(modifications.getModificationList()), verdict);
     // Delete state before returning any verdict which is not pending
     if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
         verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
         waapStateTable->hasState<Waf2Transaction>()
     ) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, eventVedict);
     }
 
-    return EventVerdict(move(modifications.getModificationList()), verdict);
+    return eventVedict;
 }
 
 EventVerdict
@@ -528,6 +532,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
 
     ngx_http_cp_verdict_e verdict = pending_response.getVerdict();
     HttpBodyModification modifications;
+    std::string webUserResponseByPractice;
 
     // Set drop verdict if waap engine decides to drop response.
     if (!waf2Transaction.decideResponse()) {
@@ -582,16 +587,16 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
         dbgTrace(D_WAAP) << " * \e[32m HttpBodyResponse: shouldInspectResponse==false: ACCEPT\e[0m";
         verdict = accept_response.getVerdict();
     }
-
+    EventVerdict eventVedict(modifications.getModificationList(), verdict);
     // Delete state before returning any verdict which is not pending or inject
     if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
         verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
         waapStateTable->hasState<Waf2Transaction>()
     ) {
-        finishTransaction(waf2Transaction);
+        finishTransaction(waf2Transaction, eventVedict);
     }
 
-    return EventVerdict(modifications.getModificationList(), verdict);
+    return eventVedict;
 }
 
 EventVerdict
@@ -630,7 +635,7 @@ WaapComponent::Impl::respond(const EndTransactionEvent &)
     }
 
     // This is our last chance to delete the state. The verdict must not be "PENDING" at this point.
-    finishTransaction(waf2Transaction);
+    finishTransaction(waf2Transaction, verdict);
     return verdict;
 }
 
@@ -638,11 +643,13 @@ EventVerdict
 WaapComponent::Impl::waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction)
 {
     dbgTrace(D_WAAP) << "waapDecisionAfterHeaders() started";
+    EventVerdict verdict = pending_response;
     if (waf2Transaction.decideAfterHeaders()) {
         dbgTrace(D_WAAP) << "WaapComponent::Impl::waapDecisionAfterHeaders(): returning DROP response.";
-        return drop_response;
+        verdict = drop_response;
+        return verdict;
     }
-    return pending_response;
+    return verdict;
 }
 
 EventVerdict
@@ -660,8 +667,11 @@ WaapComponent::Impl::waapDecision(IWaf2Transaction& waf2Transaction)
     // (in the latter case - decision to drop/pass should be governed by failopen setting)
     if (verdictCode == 0) {
         waf2Transaction.checkShouldInject();
-
-        if (waf2Transaction.shouldInspectResponse()) {
+        if (waf2Transaction.shouldLimitResponseHeadersInspection()) {
+            dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mLIMIT RESPONSE HEADERS\e[0m)";
+            verdict = limit_response_headers;
+        } else if (waf2Transaction.shouldInspectResponse()) {
+            dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mPENDING RESPONSE\e[0m)";
             verdict = pending_response;
         } else {
             dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mPASS\e[0m)";
@@ -678,10 +688,12 @@ WaapComponent::Impl::waapDecision(IWaf2Transaction& waf2Transaction)
 }
 
 void
-WaapComponent::Impl::finishTransaction(IWaf2Transaction& waf2Transaction)
+WaapComponent::Impl::finishTransaction(IWaf2Transaction& waf2Transaction, EventVerdict& verdict)
 {
+    dbgTrace(D_WAAP) << "finishTransaction() started";
     waf2Transaction.collectFoundPatterns();
     waf2Transaction.sendLog();
+    verdict.setWebUserResponseByPractice(waf2Transaction.getCurrentWebUserResponse());
     ReportIS::Severity severity = waf2Transaction.computeEventSeverityFromDecision();
     validateFirstRequestForAsset(severity);
     waapStateTable->deleteState<Waf2Transaction>();

components/security_apps/waap/waap_component_impl.h

@@ -59,7 +59,7 @@ private:
     void init(const std::string &waapDataFileName);
     EventVerdict waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction);
     EventVerdict waapDecision(IWaf2Transaction& waf2Transaction);
-    void finishTransaction(IWaf2Transaction& waf2Transaction);
+    void finishTransaction(IWaf2Transaction& waf2Transaction, EventVerdict& verdict);
 
     bool waf2_proc_start(const std::string& waapDataFileName);
     void waf2_proc_exit();
@@ -73,6 +73,7 @@ private:
     EventVerdict pending_response;
     EventVerdict accept_response;
     EventVerdict drop_response;
+    EventVerdict limit_response_headers;
     WaapMetricWrapper waap_metric;
     AssetsMetric assets_metric;
     I_Table* waapStateTable;

components/utils/generic_rulebase/generic_rulebase.cc

@@ -87,6 +87,7 @@ GenericRulebase::Impl::preload()
     addMatcher<BeginWithUri>();
     BasicRuleConfig::preload();
     LogTriggerConf::preload();
+    ReportTriggerConf::preload();
     ParameterException::preload();
     registerExpectedConfiguration<Zone>("rulebase", "zones");
     registerExpectedConfigFile("zones", Config::ConfigFileType::Policy);

components/utils/generic_rulebase/match_query.cc

@@ -50,7 +50,7 @@ static const string ip_proto_type_name = "IP protocol";
 
 static const unordered_map<string, MatchQuery::StaticKeys> string_to_key = {
     { "sourceIP", MatchQuery::StaticKeys::SrcIpAddress },
-    { "sourceIpAddr", MatchQuery::StaticKeys::SrcIpAddress },
+    { "sourceIdentifier", MatchQuery::StaticKeys::SrcIpAddress },
     { "destinationIP", MatchQuery::StaticKeys::DstIpAddress },
     { "destinationIpAddr", MatchQuery::StaticKeys::DstIpAddress },
     { "ipAddress", MatchQuery::StaticKeys::IpAddress },
@@ -319,7 +319,7 @@ MatchQuery::matchAttributes(
         match = matchAttributesString(values);
         dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match;
     }
-
+    dbgTrace(D_RULEBASE_CONFIG) << "Should negate match? " << negate;
     return negate ? !match : match;
 }
 

components/utils/generic_rulebase/parameters_config.cc

@@ -125,9 +125,10 @@ ParameterException::getBehavior(
             // When matching indicators with action=ignore, we expect no behavior override.
             // Instead, a matched keywords list should be returned which will be later removed from score calculation
             if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
-                dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore";
                 matched_override_keywords.insert(match_res.matched_keywords->begin(),
                         match_res.matched_keywords->end());
+                dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore, found " <<
+                    matched_override_keywords.size() << "keywords";
             } else {
                 matched_behaviors.insert(match_behavior_pair.behavior);
             }
@@ -143,6 +144,8 @@ ParameterException::getBehavior(
             if (match_res.matched_keywords->size() > 0 && behavior == action_ignore) {
                 matched_override_keywords.insert(match_res.matched_keywords->begin(),
                         match_res.matched_keywords->end());
+                dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore, found " <<
+                    matched_override_keywords.size() << "keywords";
             } else {
                 matched_behaviors.insert(behavior);
             }
@@ -155,6 +158,6 @@ ParameterException::getBehavior(
 set<ParameterBehavior>
 ParameterException::getBehavior(const unordered_map<string, set<string>> &key_value_pairs) const
 {
-    set<string> keywords;
+    set<string> keywords; // placeholder only, this function will be used where there's no need for ignored keywords
     return getBehavior(key_value_pairs, keywords);
 }

components/utils/generic_rulebase/triggers_config.cc

@@ -241,3 +241,9 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in)
         archive_in.setNextName(nullptr);
     }
 }
+
+void
+ReportTriggerConf::load(cereal::JSONInputArchive& archive_in)
+{
+    parseJSONKey<string>("triggerName", name, archive_in);
+}

components/utils/http_transaction_data/http_transaction_data.cc

@@ -220,6 +220,18 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data)
         dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized parsed URI: " << ngx_parsed_uri.unpack();
     }
 
+    // Try to read waf_tag if available
+    string waf_tag;
+    if (cur_pos < transaction_raw_data.size()) {
+        Maybe<string> maybe_waf_tag = deserializeStrParam(transaction_raw_data, cur_pos);
+        if (maybe_waf_tag.ok()) {
+            waf_tag = maybe_waf_tag.unpackMove();
+            dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized waf_tag: " << waf_tag;
+        }
+    } else {
+        dbgTrace(D_NGINX_ATTACHMENT) << "No waf_tag to deserialize, using empty string";
+    }
+
     // Fail if after parsing exact number of items, we didn't exactly consume whole buffer
     if (cur_pos != transaction_raw_data.size()) {
         dbgWarning(D_NGINX_ATTACHMENT) << "Nothing to deserialize, but raw data still remain";
@@ -239,6 +251,7 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data)
         client_port.unpackMove()
     );
 
+    transaction.setWafTag(waf_tag);
     return transaction;
 }
 

components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc

@@ -12,7 +12,7 @@ Buffer
 encodeInt16(uint16_t val)
 {
     vector<u_char> raw_data(reinterpret_cast<u_char*>(&val), reinterpret_cast<u_char*>(&val) + sizeof(uint16_t));
-    return move(Buffer(raw_data));
+    return Buffer(raw_data);
 }
 
 class HttpTransactionTest : public Test

components/utils/ip_utilities/ip_utilities.cc

@@ -265,7 +265,7 @@ IpAddrToString(const IpAddress &address)
         sa6.sin6_addr = address.ip.ipv6;
 
         inet_ntop(AF_INET6, &(sa6.sin6_addr), ip_str, INET6_ADDRSTRLEN);
-        return move(string(ip_str));
+        return string(ip_str);
     }
 
     char ip_str[INET_ADDRSTRLEN];
@@ -275,7 +275,7 @@ IpAddrToString(const IpAddress &address)
     sa.sin_addr = address.ip.ipv4;
 
     inet_ntop(AF_INET, &(sa.sin_addr), ip_str, INET_ADDRSTRLEN);
-    return move(string(ip_str));
+    return string(ip_str);
 }
 
 IpAddress

components/utils/utilities/nginx_conf_collector/CMakeLists.txt

@@ -8,7 +8,7 @@ link_directories(${CMAKE_BINARY_DIR}/core)
 link_directories(${CMAKE_BINARY_DIR}/core/compression)
 
 SET(EXECUTABLE_NAME "nginx_conf_collector_bin")
-add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc)
+add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc fog_connection.cc)
 target_compile_definitions(${EXECUTABLE_NAME} PRIVATE "NGINX_CONF_COLLECTOR_VERSION=\"$ENV{CI_PIPELINE_ID}\"")
 
 target_link_libraries(${EXECUTABLE_NAME}
@@ -26,6 +26,7 @@ target_link_libraries(${EXECUTABLE_NAME}
         report
         config
         environment
+        curl_http_client
         singleton
         rest
         boost_context

components/utils/utilities/nginx_conf_collector/fog_connection.cc

@@ -0,0 +1,200 @@
+#include "fog_connection.h"
+
+#include <sstream>
+#include <fstream>
+#include <iostream>
+#include <cereal/archives/json.hpp>
+#include <cereal/types/vector.hpp>
+#include <cereal/types/map.hpp>
+#include <cereal/types/string.hpp>
+#include "debug.h"
+#include "internal/curl_http_client.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+// Helper function to check if HTTPResponse indicates success
+bool
+isSuccessfulResponse(const HTTPResponse& response)
+{
+    HTTPStatusCode code = response.getHTTPStatusCode();
+    return (code == HTTPStatusCode::HTTP_OK ||
+            code == HTTPStatusCode::HTTP_NO_CONTENT ||
+            code == HTTPStatusCode::HTTP_MULTI_STATUS);
+}
+
+FogConnection::FogConnection(const string& token, const string& fog_address)
+    : var_token(token), var_fog(fog_address), curl_client(std::make_unique<CurlHttpClient>()) {}
+
+void
+FogConnection::setProxy(const string& hosts)
+{
+    curl_client->setProxy(hosts);
+}
+
+Maybe<void>
+FogConnection::getCredentials()
+{
+    AgentRegistrationRequest request;
+    AgentRegistrationRequest::AuthData auth;
+
+    auth.authenticationMethod = "token";
+    auth.data = var_token;
+
+    request.authenticationData.push_back(auth);
+    request.metaData.agentName = "ConfCollector";
+    request.metaData.agentType = "Embedded";
+    request.metaData.platform = "linux";
+    request.metaData.architecture = "x86";
+    request.metaData.additionalMetaData["agentVendor"] = "nginx-conf-collector";
+
+    stringstream ss_req;
+    {
+        cereal::JSONOutputArchive ar(ss_req);
+        request.serialize(ar);
+    }
+
+    dbgTrace(D_NGINX_MANAGER) << "Registration JSON: " << ss_req.str();
+
+    string url = var_fog + "/agents";
+    map<string, string> headers = {{"Content-Type", "application/json"},
+                                    {"User-Agent", "Infinity Next (a7030abf93a4c13)"}};
+
+    auto response = curl_client->post(url, ss_req.str(), headers);
+
+    dbgTrace(D_NGINX_MANAGER)
+        << "Register agent response code: "
+        << static_cast<int>(response.getHTTPStatusCode())
+        << ", body: "
+        << response.getBody();
+
+    if (!isSuccessfulResponse(response)) {
+        return genError("Failed to register agent: HTTP "
+                        + to_string(static_cast<int>(response.getHTTPStatusCode()))
+                        + " - "
+                        + response.getBody());
+    }
+
+    if (response.getBody().find("referenceId") != string::npos) {
+        return genError("Registration failed: " + response.getBody());
+    }
+
+    try {
+        AgentRegistrationResponse reg_response;
+        stringstream ss_res(response.getBody());
+        cereal::JSONInputArchive ar(ss_res);
+        reg_response.serialize(ar);
+        
+        agent_id = reg_response.agentId;
+        clientId = reg_response.clientId;
+        clientSecret = reg_response.clientSecret;
+        tenant_id = reg_response.tenantId;
+        profile_id = reg_response.profileId;
+    } catch (const exception& e) {
+        dbgTrace(D_NGINX_MANAGER) << "Failed to parse registration response: " << response.getBody();
+        return genError("Failed to parse registration response: " + string(e.what()));
+    }
+
+    return {};
+}
+
+Maybe<void>
+FogConnection::getJWT()
+{
+    TokenRequest request;
+    request.login = clientId;
+    request.password = clientSecret;
+
+    stringstream ss_req;
+    {
+        cereal::JSONOutputArchive ar(ss_req);
+        ar(request);
+    }
+
+    string url = var_fog + "/oauth/token?grant_type=client_credentials";
+    map<string, string> headers = {{"Content-Type", "application/json"},
+                                    {"User-Agent", "Infinity Next (a7030abf93a4c13)"}};
+
+    dbgTrace(D_NGINX_MANAGER) << "get JWT JSON: " << ss_req.str();
+
+    curl_client->setBasicAuth(clientId, clientSecret);
+    curl_client->authEnabled(true);
+    auto response = curl_client->post(url, ss_req.str(), headers);
+
+    dbgTrace(D_NGINX_MANAGER)
+        << "get JWT response code: "
+        << static_cast<int>(response.getHTTPStatusCode())
+        << ", body: "
+        << response.getBody();
+
+    if (!isSuccessfulResponse(response)) {
+        return genError("Failed to get JWT: HTTP "
+                        + to_string(static_cast<int>(response.getHTTPStatusCode()))
+                        + " - "
+                        + response.getBody());
+    }
+
+    if (response.getBody().find("referenceId") != string::npos) {
+        return genError("JWT request failed: " + response.getBody());
+    }
+
+    try {
+        TokenResponse token_response;
+        stringstream ss_res(response.getBody());
+        cereal::JSONInputArchive ar(ss_res);
+        token_response.serialize(ar);
+        
+        ra_token = token_response.access_token;
+    } catch (const exception& e) {
+        dbgTrace(D_NGINX_MANAGER) << "Failed to parse JWT response: " << response.getBody();
+        return genError("Failed to parse JWT response: " + string(e.what()));
+    }
+
+    return {};
+}
+
+Maybe<void>
+FogConnection::uploadNginxConfig(const string& config_file_path)
+{
+    if (tenant_id.empty() || profile_id.empty() || ra_token.empty()) {
+        return genError("Missing required data for upload: tenant_id, profile_id, or ra_token");
+    }
+
+    ifstream file(config_file_path, ios::binary);
+    if (!file.is_open()) {
+        return genError("Cannot open file: " + config_file_path);
+    }
+
+    string file_content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
+    file.close();
+
+    if (file_content.empty()) {
+        dbgTrace(D_NGINX_MANAGER) << "Warning: Uploading empty file content from " << config_file_path;
+    }
+
+
+    string upload_url = var_fog + "/agents-core/storage/" + tenant_id + "/nginx/" + profile_id + "/1/nginx.conf";
+    map<string, string> headers = {
+        {"Authorization", "Bearer " + ra_token},
+        {"Content-Type", "text/plain"},
+        {"User-Agent", "Infinity Next (a7030abf93a4c13)"}
+    };
+
+    auto response = curl_client->put(upload_url, file_content, headers);
+
+    dbgTrace(D_NGINX_MANAGER)
+        << "Upload status code: "
+        << static_cast<int>(response.getHTTPStatusCode())
+        << ", body: "
+        << response.getBody();
+
+    if (!isSuccessfulResponse(response)) {
+        return genError("Upload failed: HTTP "
+                        + to_string(static_cast<int>(response.getHTTPStatusCode()))
+                        + " - "
+                        + response.getBody());
+    }
+
+    return {};
+}

components/utils/utilities/nginx_conf_collector/fog_connection.h

@@ -0,0 +1,33 @@
+#ifndef __FOG_CONNECTION_H__
+#define __FOG_CONNECTION_H__
+
+#include <string>
+#include <vector>
+#include <map>
+#include "services_sdk/interfaces/i_http_client.h"
+#include "req_res_objects.h"
+#include "maybe_res.h"
+
+class FogConnection
+{
+public:
+    FogConnection(const std::string& token, const std::string& fog_address);
+
+    void setProxy(const std::string& hosts);
+    Maybe<void> getCredentials();
+    Maybe<void> getJWT();
+    Maybe<void> uploadNginxConfig(const std::string& config_file_path);
+
+private:
+    std::string var_token;
+    std::string var_fog;
+    std::string agent_id;
+    std::string tenant_id;
+    std::string profile_id;
+    std::string ra_token;
+    std::string clientId;
+    std::string clientSecret;
+    std::unique_ptr<I_HttpClient> curl_client;
+};
+
+#endif // __FOG_CONNECTION_H__

components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc

@@ -11,8 +11,12 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <cstddef>
 #include <iostream>
 #include <unistd.h>
+#include <sstream>
+#include <fstream>
+#include <getopt.h>
 
 #include "agent_core_utilities.h"
 #include "debug.h"
@@ -20,6 +24,7 @@
 #include "mainloop.h"
 #include "nginx_utils.h"
 #include "time_proxy.h"
+#include "fog_connection.h"
 
 using namespace std;
 
@@ -43,6 +48,7 @@ public:
         environment.fini();
         time_proxy.fini();
     }
+
 private:
     ShellCmd shell_cmd;
     MainloopComponent mainloop;
@@ -63,12 +69,17 @@ printVersion()
 void
 printUsage(const char *prog_name)
 {
-    cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << '\n';
+    cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" <<
+        " [--upload --token <token> [--fog <address>]]" << '\n';
     cout << "  -V              Print version" << '\n';
     cout << "  -v              Enable verbose output" << '\n';
     cout << "  -i input_file   Specify input file (default is /etc/nginx/nginx.conf)" << '\n';
     cout << "  -o output_file  Specify output file (default is ./full_nginx.conf)" << '\n';
     cout << "  -h              Print this help message" << '\n';
+    cout << "  --upload        Upload configuration to FOG (requires --token)" << '\n';
+    cout << "  --token <token> profile token for FOG upload" << '\n';
+    cout << "  --fog <address> FOG server address (default: inext-agents.cloud.ngen.checkpoint.com)" << '\n';
+    cout << "  --proxy <address> Proxy server to send the request through" << '\n';
 }
 
 int
@@ -76,9 +87,21 @@ main(int argc, char *argv[])
 {
     string nginx_input_file = "/etc/nginx/nginx.conf";
     string nginx_output_file = "full_nginx.conf";
-
+    string fog_address = "inext-agents.cloud.ngen.checkpoint.com";
+    string token;
+    string proxy_host;
+    bool upload_flag = false;
     int opt;
-    while ((opt = getopt(argc, argv, "Vvhi:o:h")) != -1) {
+    
+    static struct option long_options[] = {
+        {"upload", no_argument, 0, 'u'},
+        {"token", required_argument, 0, 1001},
+        {"fog", required_argument, 0, 1002},
+        {"proxy", required_argument, 0, 1003},
+        {0, 0, 0, 0}
+    };
+    
+    while ((opt = getopt_long(argc, argv, "Vvhi:o:", long_options, nullptr)) != -1) {
         switch (opt) {
             case 'V':
                 printVersion();
@@ -95,18 +118,36 @@ main(int argc, char *argv[])
             case 'h':
                 printUsage(argv[0]);
                 return 0;
+            case 'u':
+                upload_flag = true;
+                break;
+            case 1001: // --token
+                token = optarg;
+                break;
+            case 1002: // --fog
+                fog_address = optarg;
+                break;
+            case 1003: // --proxy
+                proxy_host = optarg;
+                break;
             default:
                 printUsage(argv[0]);
                 return 1;
         }
     }
 
-    for (int i = optind; i < argc;) {
+    for (int i = optind; i < argc; ++i) {
         cerr << "Unknown argument: " << argv[i] << '\n';
         printUsage(argv[0]);
         return 1;
     }
 
+    if (upload_flag && token.empty()) {
+        cerr << "Error: --upload requires --token to be specified" << '\n';
+        printUsage(argv[0]);
+        return 1;
+    }
+
     dbgTrace(D_NGINX_MANAGER) << "Starting nginx configuration collector";
 
     MainComponent main_component;
@@ -144,5 +185,43 @@ main(int argc, char *argv[])
 
     cout << "Full nginx configuration file was successfully generated: " << result.unpack() << '\n';
 
+    if (upload_flag) {
+        cout << "Uploading configuration to FOG server: " << fog_address << '\n';
+        
+        string full_fog_url = fog_address;
+        if (fog_address.find("http://") != 0 && fog_address.find("https://") != 0) {
+            full_fog_url = "https://" + fog_address;
+        }
+        
+        FogConnection fog_connection(token, full_fog_url);
+
+        if (!proxy_host.empty()) {
+            fog_connection.setProxy(proxy_host);
+        }
+
+        auto credentials_result = fog_connection.getCredentials();
+        if (!credentials_result.ok()) {
+            cerr
+                << "Failed to register agent with the FOG. with error: "
+                << credentials_result.getErr()
+                << '\n';
+            return 1;
+        }
+
+        auto jwt_result = fog_connection.getJWT();
+        if (!jwt_result.ok()) {
+            cerr << "Failed to get JWT token. with error:" << jwt_result.getErr() << '\n';
+            return 1;
+        }
+
+        auto upload_result = fog_connection.uploadNginxConfig(result.unpack());
+        if (!upload_result.ok()) {
+            cerr << "Failed to upload nginx config file to FOG. with error:" << upload_result.getErr() << '\n';
+            return 1;
+        }
+
+        cout << "Successfully uploaded configuration to FOG server." << '\n';
+    }
+    
     return 0;
 }

components/utils/utilities/nginx_conf_collector/req_res_objects.h

@@ -0,0 +1,132 @@
+#ifndef __REQ_RES_OBJECTS_H__
+#define __REQ_RES_OBJECTS_H__
+
+#include "cereal/archives/json.hpp"
+#include "cereal/types/string.hpp"
+#include "cereal/types/vector.hpp"
+#include "cereal/types/map.hpp"
+
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+struct AgentRegistrationRequest
+{
+    struct AuthData
+    {
+        template<class Archive>
+        void serialize(Archive& ar)
+        {
+            try {
+                ar(cereal::make_nvp("authenticationMethod", authenticationMethod));
+                ar(cereal::make_nvp("data", data));
+            } catch (const cereal::Exception &e) {
+                dbgWarning(D_NGINX_MANAGER) << "Serialization error in AuthData: " << e.what();
+                ar.setNextName(nullptr);
+            }
+        }
+        
+        std::string authenticationMethod;
+        std::string data;
+    };
+    
+    struct MetaData
+    {
+        template<class Archive>
+        void serialize(Archive& ar)
+        {
+            try {
+                ar(cereal::make_nvp("agentName", agentName));
+                ar(cereal::make_nvp("agentType", agentType));
+                ar(cereal::make_nvp("platform", platform));
+                ar(cereal::make_nvp("architecture", architecture));
+                for (const auto& pair : additionalMetaData) {
+                    ar(cereal::make_nvp(pair.first.c_str(), pair.second));
+                }
+            } catch (const cereal::Exception &e) {
+                dbgWarning(D_NGINX_MANAGER) << "Serialization error in MetaData: " << e.what();
+                ar.setNextName(nullptr);
+            }
+        }
+        
+        std::string agentName;
+        std::string agentType;
+        std::string platform;
+        std::string architecture;
+        std::map<std::string, std::string> additionalMetaData;
+    };
+    
+    template<class Archive>
+    void serialize(Archive& ar)
+    {
+        try {
+            ar(cereal::make_nvp("authenticationData", authenticationData));
+            ar(cereal::make_nvp("metaData", metaData));
+        } catch (const cereal::Exception &e) {
+            dbgWarning(D_NGINX_MANAGER) << "Serialization error in AgentRegistrationRequest: " << e.what();
+            ar.setNextName(nullptr);
+        }
+    }
+    
+    std::vector<AuthData> authenticationData;
+    MetaData metaData;
+};
+
+struct TokenRequest
+{
+    template<class Archive>
+    void serialize(Archive& ar)
+    {
+        try {
+            ar(cereal::make_nvp("login", login));
+            ar(cereal::make_nvp("password", password));
+        } catch (const cereal::Exception &e) {
+            dbgWarning(D_NGINX_MANAGER) << "Serialization error in TokenRequest: " << e.what();
+            ar.setNextName(nullptr);
+        }
+    }
+    
+    std::string login;
+    std::string password;
+};
+
+struct AgentRegistrationResponse
+{
+    template<class Archive>
+    void serialize(Archive& ar)
+    {
+        try {
+            ar(cereal::make_nvp("agentId", agentId));
+            ar(cereal::make_nvp("clientId", clientId));
+            ar(cereal::make_nvp("clientSecret", clientSecret));
+            ar(cereal::make_nvp("tenantId", tenantId));
+            ar(cereal::make_nvp("profileId", profileId));
+        } catch (const cereal::Exception &e) {
+            dbgWarning(D_NGINX_MANAGER) << "Serialization error in AgentRegistrationResponse: " << e.what();
+            ar.setNextName(nullptr);
+        }
+    }
+    
+    std::string agentId;
+    std::string clientId;
+    std::string clientSecret;
+    std::string tenantId;
+    std::string profileId;
+};
+
+struct TokenResponse
+{
+    template<class Archive>
+    void serialize(Archive& ar)
+    {
+        try {
+            ar(cereal::make_nvp("access_token", access_token));
+        } catch (const cereal::Exception &e) {
+            dbgWarning(D_NGINX_MANAGER) << "Serialization error in TokenResponse: " << e.what();
+            ar.setNextName(nullptr);
+        }
+    }
+    
+    std::string access_token;
+};
+#endif

core/CMakeLists.txt

@@ -32,6 +32,7 @@ add_subdirectory(compression)
 add_subdirectory(attachments)
 add_subdirectory(report_messaging)
 add_subdirectory(env_details)
+add_subdirectory(curl_http_client)
 
 add_library(ngen_core SHARED ".")
 target_link_libraries(

core/agent_core_utilities/agent_core_utilities.cc

@@ -539,6 +539,14 @@ trim(string str)
     return removeLeadingWhitespaces(removeTrailingWhitespaces(str));
 }
 
+string
+toLower(string str)
+{
+    transform(str.begin(), str.end(), str.begin(), ::tolower);
+    return str;
+}
+
+
 } // namespace Strings
 
 } // namespace NGEN

core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc

@@ -197,6 +197,12 @@ TEST_F(AgentCoreUtilUT, trimTest)
     EXPECT_EQ(NGEN::Strings::trim(str_with_leading_and_trailing_whitespace), "str_with_whitespace");
 }
 
+TEST_F(AgentCoreUtilUT, toLowerTest)
+{
+    string str = "ThIS Is A 123 TEsT StRiNG";
+    EXPECT_EQ(NGEN::Strings::toLower(str), "this is a 123 test string");
+}
+
 TEST_F(AgentCoreUtilUT, resolveFullPathTest)
 {
     string working_dir = cptestFnameInExeDir("");
@@ -208,3 +214,43 @@ TEST_F(AgentCoreUtilUT, resolveFullPathTest)
     EXPECT_EQ(full_path, working_dir + "test.txt");
     ASSERT_TRUE(NGEN::Filesystem::deleteFile(working_dir + "test.txt"));
 }
+
+TEST_F(AgentCoreUtilUT, regexReplaceTest)
+{
+    struct TestCase {
+        std::string input;
+        std::string expected;
+    };
+
+    std::vector<TestCase> test_cases = {
+        {"my?invalid//:filename*test.txt", "my_invalid_filename_test.txt"},
+        {"hello///world", "hello_world"},
+        {"file@@name..txt", "file_name..txt"},
+        {"file--name", "file--name"},
+        {"some@@@file!!name.txt", "some_file_name.txt"},
+        {"https://some_file_name.txt", "https_some_file_name.txt"},
+        {"spaces in filename.txt", "spaces_in_filename.txt"},
+        {"trailing-dash-", "trailing-dash-"},
+        {"trailing.dot.", "trailing.dot."},
+        {"file name with (parens).txt", "file_name_with_parens_.txt"},
+        {"$pecial#Chars&here.txt", "_pecial_Chars_here.txt"},
+        {"___leading_underscores", "___leading_underscores"},
+        {"<<<<weird>>>filename", "_weird_filename"},
+        {"double..dots...txt", "double..dots...txt"},
+        {"a:b|c*d?e<f>g/h.txt", "a_b_c_d_e_f_g_h.txt"},
+        {"/leading/slash", "_leading_slash"},
+        {"back\\slash\\file", "back_slash_file"},
+        {"file.with..multiple.dots.txt", "file.with..multiple.dots.txt"},
+        {"CAPITAL&LETTERS^HERE", "CAPITAL_LETTERS_HERE"},
+        {"123_456-789.ok", "123_456-789.ok"},
+        {"__", "__"},
+        {"*.*", "_._"}
+    };
+
+    boost::regex regex("[^\\w.-]+"); // Matches one or more non-word, non-dot, non-hyphen characters
+
+    for (const auto& testCase : test_cases) {
+        std::string replaced = NGEN::Regex::regexReplace(__FILE__, __LINE__, testCase.input, regex, "_");
+        EXPECT_EQ(replaced, testCase.expected);
+    }
+}