Jan 06 2026 dev (#387)

* sync code * update code to support brotli * update code to support brotli * update code to support brotli * sync code * fix findBrotli * sync code * sync code * sync code * sync code --------- Co-authored-by: Ned Wright <nedwright@proton.me> Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
2026-01-17 16:00:26 +03:00 · 2026-01-13 17:17:52 +02:00
parent c1058db57d
commit e7b6e51b31
216 changed files with 12601 additions and 2825 deletions
--- a/components/utils/generic_rulebase/evaluators/parameter_eval.cc
+++ b/components/utils/generic_rulebase/evaluators/parameter_eval.cc
@@ -46,6 +46,6 @@ ParameterMatcher::evalVariable() const
    dbgTrace(D_RULEBASE_CONFIG)
        << "Did not find current parameter in context."
        << " Match parameter from current rule";
-    auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
+    auto rule = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
    return rule.ok() && rule.unpack().isParameterActive(parameter_id);
 }
--- a/components/utils/generic_rulebase/evaluators/practice_eval.cc
+++ b/components/utils/generic_rulebase/evaluators/practice_eval.cc
@@ -45,6 +45,6 @@ PracticeMatcher::evalVariable() const
        return bc_practice_id_ctx.unpack().count(practice_id) > 0;
    }

-    auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
+    auto rule = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
    return rule.ok() && rule.unpack().isPracticeActive(practice_id);
 }
--- a/components/utils/generic_rulebase/evaluators/trigger_eval.cc
+++ b/components/utils/generic_rulebase/evaluators/trigger_eval.cc
@@ -52,6 +52,6 @@ TriggerMatcher::evalVariable() const
        << makeSeparatedStr(bc_trigger_id_ctx.ok() ? *bc_trigger_id_ctx : set<GenericConfigId>(), ", ");
    if (bc_trigger_id_ctx.ok()) return bc_trigger_id_ctx.unpack().count(trigger_id) > 0;

-    auto rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
+    auto rule = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
    return rule.ok() && rule.unpack().isTriggerActive(trigger_id);
 }
--- a/components/utils/generic_rulebase/generic_rulebase.cc
+++ b/components/utils/generic_rulebase/generic_rulebase.cc
@@ -104,7 +104,8 @@ GenericRulebase::Impl::getLogTriggerConf(const string& trigger_Id) const
    ScopedContext ctx;
    set<string> triggers = {trigger_Id};
    ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers);
-    return getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log");
+    auto config = getConfigurationWithCache<LogTriggerConf>("rulebase", "log");
+    return config.ok() ? config.unpack() : LogTriggerConf();
 }

 ParameterException
@@ -113,19 +114,20 @@ GenericRulebase::Impl::getParameterException(const string& parameter_Id) const
    ScopedContext ctx;
    set<string> exceptions = {parameter_Id};
    ctx.registerValue<set<GenericConfigId>>(ParameterMatcher::ctx_key, exceptions);
-    return getConfigurationWithDefault(ParameterException(), "rulebase", "exception");
+    auto config = getConfigurationWithCache<ParameterException>("rulebase", "exception");
+    return config.ok() ? config.unpack() : ParameterException();
 }

 set<ParameterBehavior>
 GenericRulebase::Impl::getBehavior(const ParameterKeyValues &key_value_pairs) const
 {
-    auto &exceptions = getConfiguration<ParameterException>("rulebase", "exception");
+    auto exceptions = getConfigurationWithCache<ParameterException>("rulebase", "exception");

    if (!exceptions.ok()) {
        dbgTrace(D_RULEBASE_CONFIG) << "Could not find any exception with the current rule's context";
        return {};
    }
-    return (*exceptions).getBehavior(key_value_pairs);
+    return exceptions.unpack().getBehavior(key_value_pairs);
 }

 GenericRulebase::GenericRulebase() : Component("GenericRulebase"), pimpl(make_unique<Impl>()) {}
--- a/components/utils/generic_rulebase/generic_rulebase_context.cc
+++ b/components/utils/generic_rulebase/generic_rulebase_context.cc
@@ -64,10 +64,14 @@ GenericRulebaseContext::activate(const BasicRuleConfig &rule)
                ZoneMatcher::ctx_key,
                rule.getZoneId()
            );
-            ctx.registerValue<GenericConfigId>(
+            ctx.registerQuickAccessValue<GenericConfigId>(
                AssetMatcher::ctx_key,
                rule.getAssetId()
            );
+            ctx.registerQuickAccessValue<GenericConfigId>(
+                TriggerMatcher::ctx_key,
+                rule.getAssetId()
+            );
            ctx.activate();
            break;
        }
@@ -87,7 +91,7 @@ GenericRulebaseContext::activate()
 {
    switch(registration_state) {
        case RuleRegistrationState::UNINITIALIZED: {
-            auto maybe_rule = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
+            auto maybe_rule = setConfigurationInCache<BasicRuleConfig>("rulebase", "rulesConfig");
            if (!maybe_rule.ok()) {
                registration_state = RuleRegistrationState::UNREGISTERED;
                return;
--- a/components/utils/generic_rulebase/match_query.cc
+++ b/components/utils/generic_rulebase/match_query.cc
@@ -244,7 +244,8 @@ MatchQuery::getAllKeys() const
 bool
 MatchQuery::matchAttributes(
        const unordered_map<string, set<string>> &key_value_pairs,
-        set<string> &matched_override_keywords) const
+        set<string> &matched_override_keywords,
+        bool skip_irrelevant_key) const
 {

    dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes";
@@ -256,25 +257,47 @@ MatchQuery::matchAttributes(
        }
        return matchAttributes(key_value_pair->second, matched_override_keywords);
    } else if (type == MatchType::Operator && operator_type == Operators::And) {
-        for (const MatchQuery &inner_match: items) {
-            if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
+        bool has_relevant_keys = false;
+        for (const MatchQuery &inner_match : items) {
+            // Skip irrelevant keys when skip_irrelevant_key is false
+            if (skip_irrelevant_key && key_value_pairs.find(inner_match.getKey()) == key_value_pairs.end()) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Skipping irrelevant key in AND operator";
+                continue;
+            }
+            has_relevant_keys = true;
+            if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords, skip_irrelevant_key)) {
                dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator";
                return false;
            }
        }
+        if (!has_relevant_keys) {
+            dbgTrace(D_RULEBASE_CONFIG) << "No relevant keys found for AND operator";
+            return false;
+        }
        dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator";
        return true;
    } else if (type == MatchType::Operator && operator_type == Operators::Or) {
        // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
        set<string> inner_override_keywords;
        bool res = false;
+        bool has_relevant_keys = false;
        for (const MatchQuery &inner_match: items) {
+            // Skip irrelevant keys when skip_irrelevant_key is false
+            if (skip_irrelevant_key && key_value_pairs.find(inner_match.getKey()) == key_value_pairs.end()) {
+                dbgTrace(D_RULEBASE_CONFIG) << "Skipping irrelevant key in OR operator";
+                continue;
+            }
+            has_relevant_keys = true;
            inner_override_keywords.clear();
-            if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) {
+            if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords, skip_irrelevant_key)) {
                matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end());
                res = true;
            }
        }
+        if (!has_relevant_keys) {
+            dbgTrace(D_RULEBASE_CONFIG) << "No relevant keys found for OR operator";
+            return false;
+        }
        dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res;
        return res;
    } else {
@@ -284,20 +307,23 @@ MatchQuery::matchAttributes(
 }

 MatchQuery::MatchResult
-MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs) const
+MatchQuery::getMatch(
+    const unordered_map<string, set<string>> &key_value_pairs,
+    bool skip_irrelevant_key) const
 {
    MatchQuery::MatchResult matches;
    matches.matched_keywords = make_shared<set<string>>();
-    matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
+    matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords, skip_irrelevant_key);
    dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match;
    return matches;
 }

 bool
 MatchQuery::matchAttributes(
-        const unordered_map<string, set<string>> &key_value_pairs) const
+        const unordered_map<string, set<string>> &key_value_pairs,
+        bool skip_irrelevant_key) const
 {
-    return getMatch(key_value_pairs).is_match;
+    return getMatch(key_value_pairs, skip_irrelevant_key).is_match;
 }

 bool
--- a/components/utils/generic_rulebase/parameters_config.cc
+++ b/components/utils/generic_rulebase/parameters_config.cc
@@ -105,19 +105,22 @@ ParameterException::load(cereal::JSONInputArchive &archive_in)
    for (const MatchBehaviorPair &match_query : match_queries) {
        if (isGeoLocationExists(match_query.match)) return;
    }
+
+    is_containing_kv_pair = checkKVPair();
 }

 set<ParameterBehavior>
 ParameterException::getBehavior(
-        const unordered_map<string, set<string>> &key_value_pairs,
-        set<string> &matched_override_keywords) const
+    const unordered_map<string, set<string>> &key_value_pairs,
+    set<string> &matched_override_keywords,
+    bool skip_irrelevant_key) const
 {
    set<ParameterBehavior> matched_behaviors;

    matched_override_keywords.clear();
    dbgTrace(D_RULEBASE_CONFIG) << "Matching exception";
    for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
-        MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
+        MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs, skip_irrelevant_key);
        if (match_res.is_match) {
            dbgTrace(D_RULEBASE_CONFIG)
                << "Successfully matched an exception from a list of matches, behavior: "
@@ -136,7 +139,7 @@ ParameterException::getBehavior(
    }

    if (match_queries.empty()) {
-        MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs);
+        MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs, skip_irrelevant_key);
        if (match_res.is_match) {
            dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception.";
            // When matching indicators with action=ignore, we expect no behavior override.
@@ -156,8 +159,70 @@ ParameterException::getBehavior(
 }

 set<ParameterBehavior>
-ParameterException::getBehavior(const unordered_map<string, set<string>> &key_value_pairs) const
+ParameterException::getBehavior(
+    const unordered_map<string, set<string>> &key_value_pairs,
+    bool skip_irrelevant_key) const
 {
    set<string> keywords; // placeholder only, this function will be used where there's no need for ignored keywords
-    return getBehavior(key_value_pairs, keywords);
+    return getBehavior(key_value_pairs, keywords, skip_irrelevant_key);
+}
+
+static bool
+checkMatchQueryForKVPair(const MatchQuery &query)
+{
+    if (query.getType() == MatchQuery::MatchType::Condition) {
+        return false;
+    }
+
+    if (query.getType() == MatchQuery::MatchType::Operator) {
+        if (query.getOperatorType() == MatchQuery::Operators::And) {
+            set<string> found_keys;
+
+            for (const MatchQuery &item : query.getItems()) {
+                if (item.getType() == MatchQuery::MatchType::Condition) {
+                    found_keys.insert(item.getKey());
+                }
+            }
+
+            bool hasParamName = found_keys.find("paramName") != found_keys.end();
+            bool hasParamValue = found_keys.find("paramValue") != found_keys.end();
+
+            if (hasParamName && hasParamValue) {
+                return true;
+            }
+
+            bool hasHeaderName = found_keys.find("headerName") != found_keys.end();
+            bool hasHeaderValue = found_keys.find("headerValue") != found_keys.end();
+
+            if (hasHeaderName && hasHeaderValue) {
+                return true;
+            }
+        }
+
+        for (const MatchQuery &item : query.getItems()) {
+            if (item.getType() == MatchQuery::MatchType::Operator) {
+                if (checkMatchQueryForKVPair(item)) {
+                    return true;
+                }
+            }
+        }
+    }
+
+    return false;
+}
+
+bool
+ParameterException::checkKVPair() const
+{
+    if (checkMatchQueryForKVPair(match)) {
+        return true;
+    }
+
+    for (const MatchBehaviorPair &match_behavior_pair : match_queries) {
+        if (checkMatchQueryForKVPair(match_behavior_pair.match)) {
+            return true;
+        }
+    }
+
+    return false;
 }
--- a/components/utils/generic_rulebase/triggers_config.cc
+++ b/components/utils/generic_rulebase/triggers_config.cc
@@ -59,6 +59,7 @@ WebTriggerConf::load(cereal::JSONInputArchive &archive_in)

        parseJSONKey<string>("response body", response_body, archive_in);
        parseJSONKey<string>("response title", response_title, archive_in);
+        parseJSONKey<string>("content type", content_type, archive_in);
    } catch (const exception &e) {
        dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the web trigger configuration: '" << e.what() << "'";
        archive_in.setNextName(nullptr);
@@ -185,6 +186,7 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in)
        setTriggersFlag("acLogGeoLocation", archive_in, SecurityType::AccessControl, log_geo_location);
        setTriggersFlag("tpLogGeoLocation", archive_in, SecurityType::ThreatPrevention, log_geo_location);
        setTriggersFlag("complianceLogGeoLocation", archive_in, SecurityType::Compliance, log_geo_location);
+        setTriggersFlag("shouldIgnoreExceptionLog", archive_in, SecurityType::ThreatPrevention, should_log_exception);

        bool extend_logging = false;
        parseJSONKey<bool>("extendLogging", extend_logging, archive_in);
--- a/components/utils/geo_location/geo_location.cc
+++ b/components/utils/geo_location/geo_location.cc
@@ -78,7 +78,7 @@ public:

        MMDB_lookup_result_s result = MMDB_lookup_sockaddr(&max_mind_db_obj, &sockaddr_to_search, &maxminddb_error);
        if (maxminddb_error != MMDB_SUCCESS) {
-            dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(maxminddb_error);
+            dbgDebug(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(maxminddb_error);
            return genError("maxMindDB error: " + string(MMDB_strerror(maxminddb_error)));
        }
        if (result.found_entry) {
@@ -153,9 +153,9 @@ private:
            }
        }
        if (status != MMDB_SUCCESS) {
-            dbgWarning(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(status);
+            dbgDebug(D_GEO_DB) << "maxMindDB error: " << MMDB_strerror(status);
        } else if (!entry_data.has_data) {
-            dbgWarning(D_GEO_DB) << "maxMindDB Entry has no data";
+            dbgDebug(D_GEO_DB) << "maxMindDB Entry has no data";
        } else {
            string search_result(entry_data.utf8_string, entry_data.data_size);
            return search_result;
--- a/components/utils/http_transaction_data/http_transaction_data.cc
+++ b/components/utils/http_transaction_data/http_transaction_data.cc
@@ -19,7 +19,7 @@

 #include "enum_array.h"
 #include "buffer.h"
-#include "nginx_attachment_common.h"
+#include "nano_attachment_common.h"

 using namespace std;

--- a/components/utils/pm/CMakeLists.txt
+++ b/components/utils/pm/CMakeLists.txt
@@ -1,3 +1,3 @@
-add_library(pm general_adaptor.cc kiss_hash.cc kiss_patterns.cc kiss_pm_stats.cc kiss_thin_nfa.cc kiss_thin_nfa_analyze.cc kiss_thin_nfa_build.cc kiss_thin_nfa_compile.cc pm_adaptor.cc pm_hook.cc debugpm.cc)
+add_library(pm general_adaptor.cc kiss_hash.cc kiss_patterns.cc kiss_pm_stats.cc kiss_thin_nfa.cc kiss_thin_nfa_analyze.cc kiss_thin_nfa_build.cc kiss_thin_nfa_compile.cc pm_adaptor.cc pm_hook.cc debugpm.cc hyperscan_hook.cc)

 add_subdirectory(pm_ut)
--- a/components/utils/pm/hyperscan_hook.cc
+++ b/components/utils/pm/hyperscan_hook.cc
@@ -0,0 +1,135 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifdef USE_HYPERSCAN
+#include "hyperscan_hook.h"
+#include <algorithm>
+#include <cctype>
+
+// Helper function to escape regex special characters for literal matching
+static std::string escapeRegexChars(const std::string& input) {
+    std::string escaped;
+    for (char c : input) {
+        switch (c) {
+            case '.':
+            case '^':
+            case '$':
+            case '*':
+            case '+':
+            case '?':
+            case '(':
+            case ')':
+            case '[':
+            case ']':
+            case '{':
+            case '}':
+            case '\\':
+            case '|':
+                escaped += '\\';
+                escaped += c;
+                break;
+            default:
+                escaped += c;
+                break;
+        }
+    }
+    return escaped;
+}
+
+HyperscanHook::HyperscanHook() : m_hsDatabase(nullptr), m_hsScratch(nullptr), m_hsReady(false) {}
+
+HyperscanHook::~HyperscanHook() {
+    if (m_hsScratch) hs_free_scratch(m_hsScratch);
+    if (m_hsDatabase) hs_free_database(m_hsDatabase);
+}
+
+Maybe<void> HyperscanHook::prepare(const std::set<PMPattern> &patterns) {
+    m_hsPatterns.clear();
+    m_idToPattern.clear();    for (const auto &pat : patterns) {
+        if (pat.empty()) continue; // Use pat.empty() instead of pat.pattern().empty()
+
+        // Convert pattern data to string using the public interface
+        std::string pattern_str(reinterpret_cast<const char*>(pat.data()), pat.size());
+
+        // Escape regex special characters for literal matching
+        std::string escaped_pattern = escapeRegexChars(pattern_str);
+
+        m_hsPatterns.push_back(escaped_pattern);
+        m_idToPattern.push_back(pat);
+    }
+
+    std::vector<const char*> c_patterns;
+    std::vector<unsigned int> flags;
+    std::vector<unsigned int> ids;
+
+    for (size_t i = 0; i < m_hsPatterns.size(); ++i) {
+        c_patterns.push_back(m_hsPatterns[i].c_str());
+        flags.push_back(HS_FLAG_CASELESS); // adjust as needed
+        ids.push_back((unsigned int)i);
+    }    hs_compile_error_t *compile_err = nullptr;
+    hs_error_t result = hs_compile_multi(c_patterns.data(), flags.data(), ids.data(),
+                        (unsigned int)c_patterns.size(), HS_MODE_BLOCK, nullptr,
+                        &m_hsDatabase, &compile_err);
+
+    if (result != HS_SUCCESS) {
+        std::string error_msg = "Failed to compile Hyperscan database";
+        if (compile_err) {
+            error_msg += ": ";
+            error_msg += compile_err->message;
+            hs_free_compile_error(compile_err);
+        }
+        return genError(error_msg);
+    }if (hs_alloc_scratch(m_hsDatabase, &m_hsScratch) != HS_SUCCESS) {
+        return genError("Failed to allocate Hyperscan scratch space");
+    }
+
+    m_hsReady = true;
+    return Maybe<void>();
+}
+
+// TODO - No need for HS hook, scanning is done by WaapHyperscanEngine::scanSample()
+std::set<PMPattern> HyperscanHook::scanBuf(const Buffer &buf) const {
+    std::set<PMPattern> results;
+    scanBufWithOffsetLambda(buf, [&results](uint, const PMPattern &pattern, bool) {
+        results.insert(pattern);
+    });
+    return results;
+}
+
+std::set<std::pair<uint, uint>> HyperscanHook::scanBufWithOffset(const Buffer &buf) const {
+    std::set<std::pair<uint, uint>> results;
+    scanBufWithOffsetLambda(buf, [&results](uint endMatchOffset, const PMPattern &pattern, bool) {
+        uint startOffset = endMatchOffset + 1 - pattern.size();
+        results.insert(std::make_pair(startOffset, endMatchOffset));
+    });
+    return results;
+}
+
+void HyperscanHook::scanBufWithOffsetLambda(const Buffer &buf, I_PMScan::CBFunction cb) const {
+    if (!m_hsReady) return;
+    struct HyperScanContext {
+        const HyperscanHook *self;
+        const Buffer *buffer;
+        I_PMScan::CBFunction cb;
+    };
+    auto onMatch = [](unsigned int id, unsigned long long, unsigned long long to, unsigned int,
+                    void *ctx) -> int {
+        HyperScanContext *hctx = (HyperScanContext*)ctx;
+        const HyperscanHook *self = hctx->self;
+        const PMPattern &pat = self->m_idToPattern[id];
+        uint endMatchOffset = (uint)to - 1;
+        hctx->cb(endMatchOffset, pat, false); // matchAll logic can be extended if needed
+        return 0;
+    };
+    HyperScanContext ctx{this, &buf, cb};
+    hs_scan(m_hsDatabase, (const char*)buf.data(), (unsigned int)buf.size(), 0, m_hsScratch, onMatch, &ctx);
+}
+
+#endif // USE_HYPERSCAN
--- a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt
+++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt
@@ -32,6 +32,7 @@ target_link_libraries(${EXECUTABLE_NAME}
        boost_context
        boost_regex
        pthread
+        ${Brotli_LIBRARIES}
 )

 install(TARGETS ${EXECUTABLE_NAME} DESTINATION bin)