My 11th 2023 update

CMakeLists.txt

@@ -28,6 +28,7 @@ include_directories(core/include/services_sdk/interfaces)
 include_directories(core/include/services_sdk/resources)
 include_directories(core/include/services_sdk/utilities)
 include_directories(core/include/attachments)
+include_directories(events/include)
 include_directories(components/include)
 
 add_subdirectory(build_system)

components/include/i_downloader.h

@@ -39,6 +39,8 @@ public:
         Package::ChecksumTypes checksum_type,
         const std::string &service_name
     ) const = 0;
+
+    virtual std::string getProfileFromMap(const std::string &tenant_id) const = 0;
 };
 
 #endif // __I_DOWNLOADER_H__

components/include/i_keywords_rule.h

@@ -0,0 +1,30 @@
+#ifndef ___I_KEYWORDS_RULE_H__
+#define ___I_KEYWORDS_RULE_H__
+
+#include <memory>
+#include <string>
+
+#include "maybe_res.h"
+
+class I_KeywordsRule
+{
+public:
+    class VirtualRule
+    {
+    public:
+        virtual ~VirtualRule() {};
+        virtual bool isMatch() const = 0;
+    };
+
+    virtual Maybe<std::shared_ptr<VirtualRule>> genRule(const std::string &rule) = 0;
+
+    static const std::string & getKeywordsRuleTag() { return keywords_tag; }
+
+protected:
+    virtual ~I_KeywordsRule() {}
+
+private:
+    static std::string keywords_tag;
+};
+
+#endif // ___I_KEYWORDS_RULE_H__

components/include/i_orchestration_tools.h

@@ -111,6 +111,10 @@ public:
     virtual bool removeFile(const std::string &path) const = 0;
     virtual bool copyFile(const std::string &src_path, const std::string &dst_path) const = 0;
     virtual bool doesFileExist(const std::string &file_path) const = 0;
+    virtual void fillKeyInJson(
+        const std::string &filename,
+        const std::string &_key,
+        const std::string &_val) const = 0;
     virtual bool createDirectory(const std::string &directory_path) const = 0;
     virtual bool doesDirectoryExist(const std::string &dir_path) const = 0;
     virtual bool executeCmd(const std::string &cmd) const = 0;

components/include/ips_comp.h

@@ -0,0 +1,35 @@
+#ifndef __IPS_COMP_H__
+#define __IPS_COMP_H__
+
+#include "singleton.h"
+#include "i_keywords_rule.h"
+#include "i_table.h"
+#include "i_http_manager.h"
+#include "i_environment.h"
+#include "http_inspection_events.h"
+#include "i_generic_rulebase.h"
+#include "component.h"
+
+class IPSComp
+        :
+    public Component,
+    Singleton::Consume<I_KeywordsRule>,
+    Singleton::Consume<I_Table>,
+    Singleton::Consume<I_Environment>,
+    Singleton::Consume<I_GenericRulebase>
+{
+public:
+    IPSComp();
+    ~IPSComp();
+
+    void preload();
+
+    void init();
+    void fini();
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif // __IPS_COMP_H__

components/include/keyword_comp.h

@@ -0,0 +1,28 @@
+#ifndef __KEYWORD_COMP__
+#define __KEYWORD_COMP__
+
+#include <memory>
+
+#include "singleton.h"
+#include "i_environment.h"
+#include "i_table.h"
+#include "i_keywords_rule.h"
+#include "component.h"
+
+class KeywordComp
+        :
+    public Component,
+    Singleton::Provide<I_KeywordsRule>,
+    Singleton::Consume<I_Table>,
+    Singleton::Consume<I_Environment>
+{
+public:
+    KeywordComp();
+    ~KeywordComp();
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif // __KEYWORD_COMP__

components/include/output.h

@@ -0,0 +1,13 @@
+#ifndef __OUTPUT_H__
+#define __OUTPUT_H__
+
+#include <string>
+
+namespace infra
+{
+
+std::string printChar(char ch);
+
+} // namespace infra
+
+#endif // __OUTPUT_H__

components/security_apps/CMakeLists.txt

@@ -1,3 +1,4 @@
+add_subdirectory(ips)
 add_subdirectory(layer_7_access_control)
 add_subdirectory(orchestration)
 add_subdirectory(waap)

components/security_apps/ips/CMakeLists.txt

@@ -0,0 +1,17 @@
+include_directories(include)
+
+add_library(ips
+    ips_comp.cc
+    ips_entry.cc
+    ips_signatures.cc
+    simple_protection.cc
+    compound_protection.cc
+    ips_configuration.cc
+    helper_open_source.cc
+    ips_basic_policy.cc
+    snort_basic_policy.cc
+    ips_metric.cc
+    ips_common_types.cc
+)
+
+add_subdirectory(ips_ut)

components/security_apps/ips/compound_protection.cc

@@ -0,0 +1,216 @@
+#include "compound_protection.h"
+
+#include <algorithm>
+
+#include "rule_detection.h"
+#include "ips_entry.h"
+#include "ips_comp.h"
+#include "debug.h"
+
+using namespace std;
+using MatchType = IPSSignatureSubTypes::BaseSignature::MatchType;
+
+USE_DEBUG_FLAG(D_IPS);
+
+CompoundProtection::Impl::Impl(const string &_sig_name, SignaturesVector &&sig_vec, Operation oper)
+        :
+    sig_name(_sig_name),
+    sub_signatures(move(sig_vec)),
+    operation(oper),
+    table(Singleton::Consume<I_Table>::by<IPSComp>())
+{
+    for (const auto &sig : sub_signatures) {
+        const auto &sub_sig_context = sig->getContext();
+        for (auto &new_context : sub_sig_context) {
+            bool is_new_context = true;
+            for (auto &old_context : contexts) {
+                if (new_context == old_context) {
+                    is_new_context = false;
+                    break;
+                }
+            }
+            if (is_new_context) contexts.push_back(new_context);
+        }
+    }
+}
+
+MatchType
+CompoundProtection::Impl::getMatch(const set<PMPattern> &matched) const
+{
+    switch (operation) {
+        case Operation::OR: return getMatchOr(matched);
+        case Operation::AND: return getMatchAnd(matched);
+        case Operation::ORDERED_AND: return getMatchOrderedAnd(matched);
+    }
+
+    dbgAssert(false) << "Unknown compound operation: " << static_cast<uint>(operation);
+    return MatchType::NO_MATCH;
+}
+
+set<PMPattern>
+CompoundProtection::Impl::patternsInSignature() const
+{
+    set<PMPattern> res;
+
+    for (auto &sig : sub_signatures) {
+        const auto &sub_sig_patterns = sig->patternsInSignature();
+        for (auto &pat : sub_sig_patterns) {
+            res.insert(pat);
+        }
+    }
+
+    return res;
+}
+
+MatchType
+CompoundProtection::Impl::getMatchOr(const set<PMPattern> &matched) const
+{
+    MatchType res = MatchType::NO_MATCH;
+    for (auto &sig : sub_signatures) {
+        switch (getSubMatch(sig, matched)) {
+            case MatchType::NO_MATCH: break;
+            case MatchType::CACHE_MATCH: {
+                res = MatchType::CACHE_MATCH;
+                break;
+            }
+            case MatchType::MATCH: return MatchType::MATCH;
+        }
+    }
+    return res;
+}
+
+MatchType
+CompoundProtection::Impl::getMatchAnd(const set<PMPattern> &matched) const
+{
+    MatchType res = MatchType::CACHE_MATCH;
+    for (auto &sig : sub_signatures) {
+        switch (getSubMatch(sig, matched)) {
+            case MatchType::NO_MATCH: {
+                res = MatchType::NO_MATCH;
+                break;
+            }
+            case MatchType::CACHE_MATCH: break;
+            case MatchType::MATCH: {
+                if (res == MatchType::CACHE_MATCH) res = MatchType::MATCH;
+                break;
+            }
+        }
+    }
+    return res;
+}
+
+MatchType
+CompoundProtection::Impl::getMatchOrderedAnd(const set<PMPattern> &matched) const
+{
+    MatchType res = MatchType::CACHE_MATCH;
+    for (auto &sig : sub_signatures) {
+        switch (getSubMatch(sig, matched)) {
+            case MatchType::NO_MATCH: return MatchType::NO_MATCH;
+            case MatchType::CACHE_MATCH: break;
+            case MatchType::MATCH: {
+                res = MatchType::MATCH;
+                break;
+            }
+        }
+    }
+    return res;
+}
+
+static bool
+isStringInVector(const Maybe<string, Context::Error> &str, const vector<string> &vec)
+{
+    if (!str.ok()) return false;
+    return find(vec.begin(), vec.end(), *str) != vec.end();
+}
+
+MatchType
+CompoundProtection::Impl::getSubMatch(
+    const std::shared_ptr<IPSSignatureSubTypes::BaseSignature> &sub_sig,
+    const set<PMPattern> &matched
+) const
+{
+    if (isFlagSet(sub_sig->getSigId())) return MatchType::CACHE_MATCH;
+
+    auto env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    auto curr_ctx = env->get<string>(I_KeywordsRule::getKeywordsRuleTag());
+    if (!isStringInVector(curr_ctx, sub_sig->getContext())) return MatchType::NO_MATCH;
+
+    auto res = sub_sig->getMatch(matched);
+    if (res != MatchType::NO_MATCH) setFlag(sub_sig->getSigId());
+    return res;
+}
+
+bool
+CompoundProtection::Impl::isFlagSet(const std::string &id) const
+{
+    if (!table->hasState<IPSEntry>()) {
+        dbgWarning(D_IPS) << "No entry was found, limited compound functionality";
+        return false;
+    }
+    return table->getState<IPSEntry>().isFlagSet(id);
+}
+
+void
+CompoundProtection::Impl::setFlag(const std::string &id) const
+{
+    if (!table->hasState<IPSEntry>()) {
+        dbgWarning(D_IPS) << "No entry was found, limited compound functionality";
+        return;
+    }
+    table->getState<IPSEntry>().setFlag(id);
+}
+
+class OperandsReader
+{
+public:
+    OperandsReader(const string &sig_name) : base_sig_name(sig_name) {}
+
+    void
+    load(cereal::JSONInputArchive &ar)
+    {
+        cereal::size_type cereal_size;
+        ar(cereal::make_size_tag(cereal_size));
+        rules.resize(static_cast<size_t>(cereal_size));
+
+        uint index = 0;
+        for (auto &rule : rules) {
+            stringstream ss;
+            ss << base_sig_name << "##" << index;
+            ++index;
+            RuleDetection detection(ss.str());
+            ar(detection);
+            rule = detection.getRule();
+        }
+    }
+
+    vector<shared_ptr<IPSSignatureSubTypes::BaseSignature>> && extrackRules() { return move(rules); }
+
+private:
+    string base_sig_name;
+    vector<shared_ptr<IPSSignatureSubTypes::BaseSignature>> rules;
+};
+
+shared_ptr<IPSSignatureSubTypes::BaseSignature>
+CompoundProtection::get(const string &sig_name, cereal::JSONInputArchive &ar)
+{
+    string operation;
+    OperandsReader operands(sig_name);
+
+    ar(
+        cereal::make_nvp("operation", operation),
+        cereal::make_nvp("operands", operands)
+    );
+
+    return make_shared<Impl>(sig_name, operands.extrackRules(), getOperation(operation));
+}
+
+CompoundProtection::Operation
+CompoundProtection::getOperation(const string &operation)
+{
+    if (operation == "or") return Operation::OR;
+    if (operation == "and") return Operation::AND;
+    if (operation == "ordered_and") return Operation::ORDERED_AND;
+
+    reportConfigurationError("Unknown compound operation: " + operation);
+    return Operation::OR;
+}

components/security_apps/ips/helper_open_source.cc

@@ -0,0 +1,24 @@
+#include "helper.h"
+
+#include "config.h"
+
+using namespace std;
+
+namespace IPSHelper
+{
+
+string
+deobfuscateString(const string &str)
+{
+    if (str.substr(0, 7) == "M^AGI$C") reportConfigurationError("Deobfuscation isn't available in open-source mode");
+    return str;
+}
+
+string
+deobfuscateKeyword(const string &str)
+{
+    if (str.substr(0, 7) == "M^AGI$C") reportConfigurationError("Deobfuscation isn't available in open-source mode");
+    return str;
+}
+
+}  // IPSHelper

components/security_apps/ips/include/compound_protection.h

@@ -0,0 +1,49 @@
+#ifndef __COMPOUND_PROTECTION_H__
+#define __COMPOUND_PROTECTION_H__
+
+#include <memory>
+
+#include "ips_signatures.h"
+#include "i_table.h"
+
+class CompoundProtection
+{
+    enum class Operation { OR, AND, ORDERED_AND };
+    using BaseSignature = IPSSignatureSubTypes::BaseSignature;
+
+    class Impl : public IPSSignatureSubTypes::BaseSignature
+    {
+        using SignaturesVector = std::vector<std::shared_ptr<BaseSignature>>;
+
+    public:
+        Impl(const std::string &sig_name, SignaturesVector &&sig_vec, Operation oper);
+
+        const std::string & getSigId() const override { return sig_name; }
+        MatchType getMatch(const std::set<PMPattern> &matched) const override;
+        std::set<PMPattern> patternsInSignature() const override;
+        const std::vector<std::string> & getContext() const override { return contexts; }
+
+    private:
+        MatchType getMatchOr(const std::set<PMPattern> &matched) const;
+        MatchType getMatchAnd(const std::set<PMPattern> &matched) const;
+        MatchType getMatchOrderedAnd(const std::set<PMPattern> &matched) const;
+
+        MatchType getSubMatch(const std::shared_ptr<BaseSignature> &sub_sig, const std::set<PMPattern> &matched) const;
+        bool isFlagSet(const std::string &id) const;
+        void setFlag(const std::string &id) const;
+
+        std::string sig_name;
+        SignaturesVector sub_signatures;
+        std::vector<std::string> contexts;
+        Operation operation;
+        I_Table *table;
+    };
+
+public:
+    static std::shared_ptr<BaseSignature> get(const std::string &sig_name, cereal::JSONInputArchive &ar);
+
+private:
+    static Operation getOperation(const std::string &operation);
+};
+
+#endif // __COMPOUND_PROTECTION_H__

components/security_apps/ips/include/helper.h

@@ -0,0 +1,14 @@
+#ifndef __HELPER_H__
+#define __HELPER_H__
+
+#include <string>
+
+namespace IPSHelper
+{
+
+std::string deobfuscateString(const std::string &str);
+std::string deobfuscateKeyword(const std::string &str);
+
+} // IPSHelper
+
+#endif // __HELPER_H__

components/security_apps/ips/include/i_first_tier_agg.h

@@ -0,0 +1,19 @@
+#ifndef __I_FIRST_TIER_AGG_H__
+#define __I_FIRST_TIER_AGG_H__
+
+#include <memory>
+#include <set>
+#include <string>
+
+#include "pm_hook.h"
+
+class I_FirstTierAgg
+{
+public:
+    virtual std::shared_ptr<PMHook> getHook(const std::string &context_name, const std::set<PMPattern> &patterns) = 0;
+
+protected:
+    virtual ~I_FirstTierAgg() {}
+};
+
+#endif // __I_FIRST_TIER_AGG_H__

components/security_apps/ips/include/ips_basic_policy.h

@@ -0,0 +1,58 @@
+#ifndef __IPS_BASIC_POLICY_H__
+#define __IPS_BASIC_POLICY_H__
+
+#include <cereal/archives/json.hpp>
+#include <string>
+#include <vector>
+#include "ips_enums.h"
+#include "debug.h"
+#include "maybe_res.h"
+#include "ips_signatures.h"
+
+class RuleSelector
+{
+public:
+    class Rule
+    {
+    public:
+        void serialize(cereal::JSONInputArchive &ar);
+
+        bool isSignaturedMatched(const IPSSignatureSubTypes::CompleteSignature &signature) const;
+        const IPSSignatureSubTypes::SignatureAction & getAction() const { return action; };
+        void readAction(cereal::JSONInputArchive &ar, const std::string &action_type);
+        void print(std::ostream &os) const;
+
+    private:
+        void readPerformanceImpact(cereal::JSONInputArchive &ar);
+        void readSeverityLevel(cereal::JSONInputArchive &ar);
+        void readConfidenceLevel(cereal::JSONInputArchive &ar);
+        void readServerProtections(cereal::JSONInputArchive &ar);
+        void readClientProtections(cereal::JSONInputArchive &ar);
+        void readProtectionsFromYear(cereal::JSONInputArchive &ar);
+        void readProtectionTags(cereal::JSONInputArchive &ar);
+        void readProtectionIds(cereal::JSONInputArchive &ar);
+
+        IPSSignatureSubTypes::SignatureAction action = IPSSignatureSubTypes::SignatureAction::IGNORE;
+        Maybe<IPSSignatureSubTypes::IPSLevel> performance_impact = genError("undefined");
+        Maybe<IPSSignatureSubTypes::IPSLevel> severity_level = genError("undefined");
+        Maybe<IPSSignatureSubTypes::IPSLevel> confidence_level = genError("undefined");
+        Maybe<bool> server_protections = genError("undefined");
+        Maybe<bool> client_protections = genError("undefined");
+        Maybe<int> protections_from_year = genError("undefined");
+        Maybe<std::vector<std::string>> protection_tags = genError("undefined");
+        Maybe<std::vector<std::string>> protection_ids = genError("undefined");
+    };
+
+public:
+    std::vector<IPSSignatureSubTypes::SignatureAndAction> selectSignatures() const;
+    void print(std::ostream &os) const;
+    void load(cereal::JSONInputArchive &ar);
+
+private:
+    void readRules(cereal::JSONInputArchive &ar);
+    void readDefaultAction(cereal::JSONInputArchive &ar);
+
+    std::vector<Rule> rules;
+};
+
+#endif // __IPS_BASIC_POLICY_H__

components/security_apps/ips/include/ips_common_types.h

@@ -0,0 +1,11 @@
+#ifndef __IPS_COMMON_TYPES__
+#define __IPS_COMMON_TYPES__
+
+#include "buffer.h"
+
+struct IPSCommonTypes
+{
+    static const Buffer requests_header_for_log;
+};
+
+#endif //__IPS_COMMON_TYPES__

components/security_apps/ips/include/ips_configuration.h

@@ -0,0 +1,36 @@
+#ifndef __IPS_CONFIGURATION_H__
+#define __IPS_CONFIGURATION_H__
+
+#include "config.h"
+
+class IPSConfiguration
+{
+public:
+    enum class ContextType { NORMAL, KEEP, HISTORY };
+
+    class Context {
+    public:
+        Context() : type(ContextType::NORMAL), history_size(0) {}
+        Context(ContextType type, uint history);
+
+        ContextType getType() const { return type; }
+        uint getHistorySize() const;
+
+    private:
+        ContextType type;
+        uint history_size;
+    };
+
+    IPSConfiguration() {}
+    IPSConfiguration(const std::map<std::string, Context> &initial_conf) : context_config(initial_conf) {}
+
+    void load(cereal::JSONInputArchive &ar);
+
+    Context getContext(const std::string &name) const;
+    uint getHistorySize(const std::string &name) const;
+
+private:
+    std::map<std::string, Context> context_config;
+};
+
+#endif // __IPS_CONFIGURATION_H__

components/security_apps/ips/include/ips_entry.h

@@ -0,0 +1,56 @@
+#ifndef __IPS_ENTRY_H__
+#define __IPS_ENTRY_H__
+
+#include <map>
+#include <set>
+
+#include "table_opaque.h"
+#include "parsed_context.h"
+#include "buffer.h"
+#include "context.h"
+
+class IPSEntry : public TableOpaqueSerialize<IPSEntry>, public Listener<ParsedContext>
+{
+public:
+    IPSEntry();
+
+    void upon(const ParsedContext &) override;
+    ParsedContextReply respond(const ParsedContext &ctx) override;
+    std::string getListenerName() const override { return name(); }
+
+    template <typename T>
+    void serialize(T &, uint32_t) {}
+    static std::string name();
+    static std::unique_ptr<TableOpaqueBase> prototype();
+    static uint currVer();
+    static uint minVer();
+
+    void uponEnteringContext() override { registerListener(); }
+    void uponLeavingContext() override { unregisterListener(); }
+
+    void setFlag(const std::string &flag) { flags.insert(flag); }
+    void unsetFlag(const std::string &flag) { flags.erase(flag); }
+    bool isFlagSet(const std::string &flag) const { return flags.count(flag) != 0; }
+
+    Buffer getBuffer(const std::string &name) const;
+    void setTransactionData(const Buffer &key, const Buffer &value);
+    Maybe<Buffer> getTransactionData(const Buffer &key) const;
+
+    void addPendingContext(const std::string &name, const Buffer &buffer);
+    const std::vector<std::pair<std::string, Buffer>> getPendingContexts() const { return pending_contexts; }
+    void clearPendingContexts() { pending_contexts.clear(); }
+
+    void setDrop() { is_drop = true; }
+    bool isDrop() const { return is_drop; }
+
+private:
+    std::map<std::string, Buffer> past_contexts;
+    std::set<std::string> flags;
+    Context ctx;
+    std::map<Buffer, Buffer> transaction_data;
+    std::vector<std::pair<std::string, Buffer>> pending_contexts;
+
+    bool is_drop = false;
+};
+
+#endif // __IPS_ENTRY_H__

components/security_apps/ips/include/ips_enums.h

@@ -0,0 +1,26 @@
+#ifndef __IPS_ENUMS_H__
+#define __IPS_ENUMS_H__
+
+namespace IPSSignatureSubTypes
+{
+
+enum class SignatureAction
+{
+    PREVENT,
+    DETECT,
+    IGNORE
+};
+
+enum class IPSLevel
+{
+    VERY_LOW,
+    LOW,
+    MEDIUM_LOW,
+    MEDIUM,
+    MEDIUM_HIGH,
+    HIGH,
+    CRITICAL
+};
+
+}  // IPSSignatureSubTypes
+#endif // __IPS_ENUMS_H__

components/security_apps/ips/include/ips_metric.h

@@ -0,0 +1,35 @@
+#ifndef __IPS_METRIC_H__
+#define __IPS_METRIC_H__
+
+#include "ips_signatures.h"
+#include "generic_metric.h"
+
+namespace IPSSignatureSubTypes
+{
+
+class MatchEvent : public Event<MatchEvent>
+{
+public:
+    MatchEvent(const std::shared_ptr<CompleteSignature> &sig, SignatureAction act) : signature(sig), action(act) {}
+
+    const SignatureAction & getAction() const { return action; }
+
+private:
+    std::shared_ptr<CompleteSignature> signature;
+    SignatureAction action;
+};
+
+class IPSMetric : public GenericMetric, public Listener<MatchEvent>
+{
+public:
+    void upon(const MatchEvent &event) override;
+
+private:
+    MetricCalculations::Counter prevented{this, "preventEngineMatchesSample"};
+    MetricCalculations::Counter detected{this, "detectEngineMatchesSample"};
+    MetricCalculations::Counter ignored{this, "ignoreEngineMatchesSample"};
+};
+
+} // IPSSignatureSubTypes
+
+#endif // __IPS_METRIC_H__

components/security_apps/ips/include/ips_signatures.h

@@ -0,0 +1,238 @@
+#ifndef __IPS_SIGNATURES_H__
+#define __IPS_SIGNATURES_H__
+
+#include <vector>
+
+#include "config.h"
+#include "parsed_context.h"
+#include "log_generator.h"
+#include "pm_hook.h"
+#include "ips_enums.h"
+#include "ips_entry.h"
+#include "i_first_tier_agg.h"
+
+namespace IPSSignatureSubTypes
+{
+using ActionResults = std::tuple<IPSSignatureSubTypes::SignatureAction, std::string, std::vector<std::string>>;
+
+class BaseSignature
+{
+public:
+    enum class MatchType { NO_MATCH, CACHE_MATCH, MATCH };
+
+    virtual const std::string & getSigId() const = 0;
+    virtual MatchType getMatch(const std::set<PMPattern> &matched) const = 0;
+    virtual std::set<PMPattern> patternsInSignature() const = 0;
+    virtual const std::vector<std::string> & getContext() const = 0;
+};
+
+class IPSSignatureMetaData
+{
+public:
+    void load(cereal::JSONInputArchive &ar);
+    void setIndicators(const std::string &source, const std::string &version);
+
+    const std::string & getId()                   const { return protection_id; }
+    const std::string & getName()                 const { return sig_name; }
+    const std::string & getUpdateVersion()        const { return update; }
+    const std::string & getLogTitle()             const { return event_log; }
+    const std::string & getSource()               const { return source; }
+    const std::string & getFeedVersion()          const { return version; }
+    const std::vector<std::string> & getCveList() const { return cve_list; }
+    IPSLevel getSeverity()                        const { return severity; }
+    std::string getSeverityString()               const;
+    IPSLevel getConfidence()                      const { return confidence; }
+    std::string getConfidenceString()             const;
+    IPSLevel getPerformance()                     const { return performance; }
+    std::string getPerformanceString()            const;
+    bool isSilent()                               const { return is_silent; }
+    std::string getIncidentType()                 const;
+    bool isYearAtLeast(const Maybe<int> &year)    const;
+    Maybe<int> getYear()                          const;
+
+private:
+    std::string protection_id;
+    std::string sig_name;
+    std::string event_log;
+    std::string update;
+    std::string source;
+    std::string version;
+    std::vector<std::string> cve_list;
+    std::vector<std::string> tag_list;
+    IPSLevel severity;
+    IPSLevel confidence;
+    IPSLevel performance;
+    bool is_silent = false;
+};
+
+class CompleteSignature
+{
+public:
+    void load(cereal::JSONInputArchive &ar);
+    BaseSignature::MatchType getMatch(const std::set<PMPattern> &matches) const;
+    std::set<PMPattern> patternsInSignature() const;
+    void setIndicators(const std::string &source, const std::string &version);
+
+    const std::vector<std::string> & getContext() const { return rule->getContext(); }
+    const std::string & getId()                   const { return metadata.getId(); }
+    const std::string & getLogTitle()             const { return metadata.getLogTitle(); }
+    const std::string & getName()                 const { return metadata.getName(); }
+    const std::string & getUpdateVersion()        const { return metadata.getUpdateVersion(); }
+    const std::string & getSource()               const { return metadata.getSource(); }
+    const std::string & getFeedVersion()          const { return metadata.getFeedVersion(); }
+    const std::vector<std::string> & getCveList() const { return metadata.getCveList(); }
+    IPSLevel getSeverity()                        const { return metadata.getSeverity(); }
+    std::string getSeverityString()               const { return metadata.getSeverityString(); }
+    IPSLevel getConfidence()                      const { return metadata.getConfidence(); }
+    std::string getConfidenceString()             const { return metadata.getConfidenceString(); }
+    IPSLevel getPerformance()                     const { return metadata.getPerformance(); }
+    std::string getPerformanceString()            const { return metadata.getPerformanceString(); }
+    bool isSilent()                               const { return metadata.isSilent(); }
+    std::string getIncidentType()                 const { return metadata.getIncidentType(); }
+
+    bool isYearAtLeast(const Maybe<int> &year)    const { return metadata.isYearAtLeast(year); }
+    Maybe<int> getYear()                          const { return metadata.getYear(); }
+
+private:
+    IPSSignatureMetaData metadata;
+    std::shared_ptr<BaseSignature> rule;
+};
+
+class SignatureAndAction
+{
+public:
+    SignatureAndAction(std::shared_ptr<CompleteSignature> _signature, SignatureAction _action)
+            :
+        signature(_signature),
+        action(_action)
+    {
+    }
+
+    bool isMatchedPrevent(const Buffer &context_buffer, const std::set<PMPattern> &pattern) const;
+    bool matchSilent(const Buffer &context_buffer) const;
+    std::set<PMPattern> patternsInSignature() const { return signature->patternsInSignature(); }
+    const std::vector<std::string> & getContext() const { return signature->getContext(); }
+
+private:
+    ActionResults getAction(const IPSEntry &ips_state) const;
+    std::shared_ptr<CompleteSignature> signature;
+    SignatureAction action;
+};
+} // IPSSignatureSubTypes
+
+class IPSSignaturesPerContext : public Singleton::Consume<I_FirstTierAgg>
+{
+public:
+    void addSignature(const IPSSignatureSubTypes::SignatureAndAction &sig);
+    bool isMatchedPrevent(const Buffer &context_buffer) const;
+    void calcFirstTier(const std::string &ctx_name);
+
+private:
+    std::set<PMPattern> getFirstTierMatches(const Buffer &buffer) const;
+
+    std::map<PMPattern, std::vector<IPSSignatureSubTypes::SignatureAndAction>> signatures_per_lss;
+    std::vector<IPSSignatureSubTypes::SignatureAndAction> signatures_without_lss;
+    std::shared_ptr<PMHook> first_tier;
+};
+
+class IPSSignaturesResource
+{
+public:
+    void load(cereal::JSONInputArchive &ar);
+
+    const std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> &
+    getSignatures() const
+    {
+        return all_signatures;
+    }
+
+private:
+    std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> all_signatures;
+};
+
+class SnortSignaturesResourceFile
+{
+public:
+    void load(cereal::JSONInputArchive &ar);
+    bool isFile(const std::string &file_name) const { return file_name == name; }
+    const std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> &
+    getSignatures() const
+    {
+        return all_signatures;
+    }
+
+private:
+    std::string name;
+    std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> all_signatures;
+};
+
+class SnortSignaturesResource
+{
+public:
+    void load(cereal::JSONInputArchive &ar);
+
+    const std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> &
+    getSignatures(const std::string &file_name) const
+    {
+        for (auto &file : files) {
+            if (file.isFile(file_name)) return file.getSignatures();
+        }
+        return empty;
+    }
+
+private:
+    std::vector<std::shared_ptr<IPSSignatureSubTypes::CompleteSignature>> empty;
+    std::vector<SnortSignaturesResourceFile> files;
+};
+
+class IPSSignatures
+{
+    std::set<PMPattern> getFirstTier(const ParsedContext &context);
+
+public:
+    void load(cereal::JSONInputArchive &ar);
+    bool isMatchedPrevent(const std::string &context_name, const Buffer &context_buffer) const;
+    bool isEmpty() const { return signatures_per_context.empty(); }
+    bool isEmpty(const std::string &context) const;
+
+    const std::string & getAsset() const { return asset_name; }
+    const std::string & getAssetId() const { return asset_id; }
+    const std::string & getPractice() const { return practice_name; }
+    const std::string & getPracticeId() const { return practice_id; }
+    const std::string & getSourceIdentifier() const { return source_id; }
+
+private:
+    std::map<std::string, IPSSignaturesPerContext> signatures_per_context;
+    std::string asset_name;
+    std::string asset_id;
+    std::string practice_name;
+    std::string practice_id;
+    std::string source_id;
+};
+
+class SnortSignatures
+{
+    std::set<PMPattern> getFirstTier(const ParsedContext &context);
+
+public:
+    void load(cereal::JSONInputArchive &ar);
+    bool isMatchedPrevent(const std::string &context_name, const Buffer &context_buffer) const;
+    bool isEmpty() const { return signatures_per_context.empty(); }
+    bool isEmpty(const std::string &context) const;
+
+    const std::string & getAsset() const { return asset_name; }
+    const std::string & getAssetId() const { return asset_id; }
+    const std::string & getPractice() const { return practice_name; }
+    const std::string & getPracticeId() const { return practice_id; }
+    const std::string & getSourceIdentifier() const { return source_id; }
+
+private:
+    std::map<std::string, IPSSignaturesPerContext> signatures_per_context;
+    std::string asset_name;
+    std::string asset_id;
+    std::string practice_name;
+    std::string practice_id;
+    std::string source_id;
+};
+
+#endif // __IPS_SIGNATURES_H__

components/security_apps/ips/include/rule_detection.h

@@ -0,0 +1,30 @@
+#ifndef __RULE_DETECTION_H__
+#define __RULE_DETECTION_H__
+
+#include "simple_protection.h"
+#include "compound_protection.h"
+
+class RuleDetection
+{
+public:
+    RuleDetection(const std::string &_sig_name) : sig_name(_sig_name) {}
+
+    template <typename T>
+    void serialize(T &ar)
+    {
+        std::string type;
+        ar(cereal::make_nvp("type", type));
+
+        if (type == "simple") rule = SimpleProtection::get(sig_name, ar);
+        else if (type == "compound") rule = CompoundProtection::get(sig_name, ar);
+        else reportConfigurationError("Unknown rule type: " + type);
+    };
+
+    std::shared_ptr<IPSSignatureSubTypes::BaseSignature> getRule() { return rule; }
+
+private:
+    std::shared_ptr<IPSSignatureSubTypes::BaseSignature> rule;
+    std::string sig_name;
+};
+
+#endif // __RULE_DETECTION_H__

components/security_apps/ips/include/simple_protection.h

@@ -0,0 +1,49 @@
+#ifndef __SIMPLE_PROTECTION_H__
+#define __SIMPLE_PROTECTION_H__
+
+#include <memory>
+
+#include "ips_signatures.h"
+#include "i_keywords_rule.h"
+
+class SimpleProtection
+{
+    class Impl : public IPSSignatureSubTypes::BaseSignature
+    {
+    public:
+        Impl(
+            const std::string &sig_name,
+            const std::string &ssm,
+            const std::string &keyword,
+            const std::vector<std::string> &context
+        );
+
+        const std::string & getSigId() const override { return sig_name; }
+        MatchType getMatch(const std::set<PMPattern> &matched) const override;
+        std::set<PMPattern> patternsInSignature() const override;
+        const std::vector<std::string> & getContext() const override { return context; }
+
+    private:
+        std::string sig_name;
+        std::vector<std::string> context;
+        std::shared_ptr<I_KeywordsRule::VirtualRule> rule;
+        PMPattern pattern;
+    };
+
+public:
+    template <typename Archive>
+    static std::shared_ptr<IPSSignatureSubTypes::BaseSignature> get(const std::string &sig_name, Archive &ar)
+    {
+        std::string ssm, keyword;
+        std::vector<std::string> context;
+
+        ar(
+            cereal::make_nvp("SSM",      ssm),
+            cereal::make_nvp("keywords", keyword),
+            cereal::make_nvp("context",  context)
+        );
+
+        return std::make_shared<Impl>(sig_name, ssm, keyword, context);
+    }
+};
+#endif // __SIMPLE_PROTECTION_H__

components/security_apps/ips/include/snort_basic_policy.h

@@ -0,0 +1,22 @@
+#ifndef __SNORT_BASIC_POLICY_H__
+#define __SNORT_BASIC_POLICY_H__
+
+#include <cereal/archives/json.hpp>
+#include <vector>
+#include <string>
+
+#include "ips_enums.h"
+#include "ips_signatures.h"
+
+class SnortRuleSelector
+{
+public:
+    std::vector<IPSSignatureSubTypes::SignatureAndAction> selectSignatures() const;
+    void load(cereal::JSONInputArchive &ar);
+
+private:
+    IPSSignatureSubTypes::SignatureAction action = IPSSignatureSubTypes::SignatureAction::IGNORE;
+    std::vector<std::string> file_names;
+};
+
+#endif // __SNORT_BASIC_POLICY_H__

components/security_apps/ips/ips_basic_policy.cc

@@ -0,0 +1,254 @@
+#include "ips_basic_policy.h"
+#include <cereal/types/string.hpp>
+#include <cereal/types/vector.hpp>
+#include <ostream>
+#include <stdexcept>
+#include <iomanip>
+#include "ips_signatures.h"
+#include "helper.h"
+#include "config.h"
+#include "common.h"
+
+USE_DEBUG_FLAG(D_IPS);
+
+using namespace std;
+
+void
+RuleSelector::load(cereal::JSONInputArchive &ar)
+{
+    readRules(ar);
+    readDefaultAction(ar);
+}
+
+vector<IPSSignatureSubTypes::SignatureAndAction>
+RuleSelector::selectSignatures() const
+{
+    vector<IPSSignatureSubTypes::SignatureAndAction> res;
+
+    auto all_signatures = getResource<IPSSignaturesResource>("IPS", "protections");
+    if (!all_signatures.ok()) return res;
+    auto signatures_version = getResourceWithDefault<string>("", "IPS", "VersionId");
+
+    for (auto &signature : (*all_signatures).getSignatures()) {
+        for (auto &rule : rules) {
+            if (rule.isSignaturedMatched(*signature)) {
+                if (rule.getAction() != IPSSignatureSubTypes::SignatureAction::IGNORE) {
+                    signature->setIndicators("Check Point", signatures_version);
+                    res.emplace_back(signature, rule.getAction());
+                }
+                break;
+            }
+        }
+    }
+
+    return res;
+}
+
+void
+RuleSelector::readRules(cereal::JSONInputArchive &ar)
+{
+    ar(cereal::make_nvp("rules", rules));
+}
+
+void
+RuleSelector::readDefaultAction(cereal::JSONInputArchive &ar)
+{
+    Rule rule;
+    rule.readAction(ar, "defaultAction");
+    rules.push_back(rule);
+}
+
+void
+RuleSelector::Rule::serialize(cereal::JSONInputArchive &ar)
+{
+    readAction(ar, "action");
+    readPerformanceImpact(ar);
+    readSeverityLevel(ar);
+    readConfidenceLevel(ar);
+    readServerProtections(ar);
+    readClientProtections(ar);
+    readProtectionsFromYear(ar);
+    readProtectionTags(ar);
+    readProtectionIds(ar);
+}
+
+bool
+RuleSelector::Rule::isSignaturedMatched(const IPSSignatureSubTypes::CompleteSignature &signature) const
+{
+    if (confidence_level.ok() && signature.getConfidence() != confidence_level.unpack()) return false;
+    if (severity_level.ok() && signature.getSeverity() < severity_level.unpack()) return false;
+    if (performance_impact.ok() && signature.getPerformance() > performance_impact.unpack()) return false;
+    if (!signature.isYearAtLeast(protections_from_year)) return false;
+
+    return true;
+}
+
+static ostream &
+operator<<(ostream &os, const RuleSelector::Rule & rule)
+{
+    rule.print(os);
+    return os;
+}
+
+void
+RuleSelector::Rule::print(ostream &os) const
+{
+    os << "[Rule] " << "action: " << static_cast<int>(action);
+    if (performance_impact.ok()) {
+        os << " performanceImpact: " << static_cast<int>(performance_impact.unpack());
+    }
+    if (severity_level.ok()) {
+        os << " severityLevel: " << static_cast<int>(severity_level.unpack());
+    }
+    if (confidence_level.ok()) {
+        os << " confidenceLevel: " << static_cast<int>(confidence_level.unpack());
+    }
+    if (server_protections.ok()) {
+        os << boolalpha << " serverProtections: " << server_protections.unpack();
+    }
+    if (client_protections.ok()) {
+        os << boolalpha << " clientProtections: " << client_protections.unpack();
+    }
+    if (protections_from_year.ok()) {
+        os << " protectionsFromYear: " << protections_from_year.unpack();
+    }
+    if (protection_ids.ok()) {
+        os << " protectionIds: " << makeSeparatedStr(protection_ids.unpack(), ", ");
+    }
+    if (protection_tags.ok()) {
+        os << " protectionTags: " << makeSeparatedStr(protection_tags.unpack(), ", ");
+    }
+}
+
+void
+RuleSelector::Rule::readAction(cereal::JSONInputArchive &ar, const string &action_type)
+{
+    string str;
+    ar(cereal::make_nvp(action_type, str));
+
+    if (str == "Inactive") action = IPSSignatureSubTypes::SignatureAction::IGNORE;
+    else if (str == "Detect") action = IPSSignatureSubTypes::SignatureAction::DETECT;
+    else if (str == "Prevent") action = IPSSignatureSubTypes::SignatureAction::PREVENT;
+    else reportConfigurationError("invalid action value " + str);
+}
+
+void
+RuleSelector::Rule::readPerformanceImpact(cereal::JSONInputArchive &ar)
+{
+    try {
+        string str;
+        ar(cereal::make_nvp("performanceImpact", str));
+
+        if (str == "Very low") performance_impact = IPSSignatureSubTypes::IPSLevel::VERY_LOW;
+        else if (str == "Low or lower") performance_impact =  IPSSignatureSubTypes::IPSLevel::LOW;
+        else if (str == "Medium or lower") performance_impact =  IPSSignatureSubTypes::IPSLevel::MEDIUM;
+        else if (str == "High or lower") performance_impact =  IPSSignatureSubTypes::IPSLevel::HIGH;
+        else reportConfigurationError("invalid performanceImpact value " + str);
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readSeverityLevel(cereal::JSONInputArchive &ar)
+{
+    try {
+        string str;
+        ar(cereal::make_nvp("severityLevel", str));
+
+        if (str == "Critical") severity_level = IPSSignatureSubTypes::IPSLevel::CRITICAL;
+        else if (str == "High or above") severity_level = IPSSignatureSubTypes::IPSLevel::HIGH;
+        else if (str == "Medium or above") severity_level = IPSSignatureSubTypes::IPSLevel::MEDIUM;
+        else if (str == "Low or above") severity_level = IPSSignatureSubTypes::IPSLevel::LOW;
+        else reportConfigurationError("invalid severityLevel value " + str);
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readConfidenceLevel(cereal::JSONInputArchive &ar)
+{
+    try {
+        string str;
+        ar(cereal::make_nvp("confidenceLevel", str));
+
+        if (str == "Low") confidence_level  = IPSSignatureSubTypes::IPSLevel::LOW;
+        else if (str == "Medium") confidence_level = IPSSignatureSubTypes::IPSLevel::MEDIUM;
+        else if (str == "High") confidence_level = IPSSignatureSubTypes::IPSLevel::HIGH;
+        else reportConfigurationError("invalid confidenceLevel value " + str);
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readServerProtections(cereal::JSONInputArchive &ar)
+{
+    try {
+        bool _server_protections;
+        ar(cereal::make_nvp("serverProtections", _server_protections));
+        server_protections = _server_protections;
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readClientProtections(cereal::JSONInputArchive &ar)
+{
+    try {
+        bool _client_protections;
+        ar(cereal::make_nvp("clientProtections", _client_protections));
+        client_protections = _client_protections;
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readProtectionsFromYear(cereal::JSONInputArchive &ar)
+{
+    try {
+        int year;
+        ar(cereal::make_nvp("protectionsFromYear", year));
+        if (year < 1999 || year > 2021) {
+            reportConfigurationError("invalid protectionsFromYear value " + to_string(year));
+        }
+        else {
+            protections_from_year = year;
+        }
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readProtectionTags(cereal::JSONInputArchive &ar)
+{
+    try {
+        vector<string> _protection_tags;
+        ar(cereal::make_nvp("protectionTags", _protection_tags));
+        protection_tags = _protection_tags;
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::Rule::readProtectionIds(cereal::JSONInputArchive &ar)
+{
+    try {
+        vector<string> _protection_ids;
+        ar(cereal::make_nvp("protectionIds", _protection_ids));
+        protection_ids = _protection_ids;
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+    }
+}
+
+void
+RuleSelector::print(ostream &os) const
+{
+    os << makeSeparatedStr(rules, ";");
+}

components/security_apps/ips/ips_common_types.cc

@@ -0,0 +1,4 @@
+#include "ips_common_types.h"
+
+const Buffer IPSCommonTypes::requests_header_for_log =
+    Buffer("agent-found-Requests-header-for-log", 35, Buffer::MemoryType::STATIC);

components/security_apps/ips/ips_comp.cc

@@ -0,0 +1,392 @@
+#include "ips_comp.h"
+
+#include <algorithm>
+
+#include "debug.h"
+#include "new_table_entry.h"
+#include "ips_entry.h"
+#include "ips_configuration.h"
+#include "ips_signatures.h"
+#include "ips_metric.h"
+#include "generic_rulebase/parameters_config.h"
+#include "config.h"
+#include "virtual_modifiers.h"
+#include "helper.h"
+#include "ips_common_types.h"
+#include "nginx_attachment_common.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_IPS);
+
+static const Buffer header_sep(": ", 2, Buffer::MemoryType::STATIC);
+static const Buffer line_sep("\r\n", 2, Buffer::MemoryType::STATIC);
+static const Buffer space(" ", 1, Buffer::MemoryType::STATIC);
+static const Buffer x_forworded_for_key("x-forworded-for", 15, Buffer::MemoryType::STATIC);
+static const Buffer log_sep(", ", 2, Buffer::MemoryType::STATIC);
+static const Buffer empty_buffer("", 0, Buffer::MemoryType::STATIC);
+
+static const string cookie("cookie");
+static const string oauth("_oauth2_proxy");
+static const string jsessionid("jsessionid");
+static const string xff("x-forwarded-for");
+static const string header("header");
+static const string source_ip("source ip");
+
+class IPSComp::Impl
+        :
+    public Singleton::Provide<I_FirstTierAgg>::SelfInterface,
+    public Listener<NewTableEntry>,
+    public Listener<NewHttpTransactionEvent>,
+    public Listener<HttpRequestHeaderEvent>,
+    public Listener<HttpRequestBodyEvent>,
+    public Listener<EndRequestEvent>,
+    public Listener<ResponseCodeEvent>,
+    public Listener<HttpResponseHeaderEvent>,
+    public Listener<HttpResponseBodyEvent>,
+    public Listener<EndTransactionEvent>
+{
+    static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+    static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+    static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
+
+    class SigsFirstTierAgg
+    {
+    public:
+        const shared_ptr<PMHook> &
+        getHook(const set<PMPattern> &new_pat)
+        {
+            auto old_size = pats.size();
+            pats.insert(new_pat.begin(), new_pat.end());
+
+            if (pats.size() != old_size) {
+                if (!hook->prepare(pats).ok()) {
+                    reportConfigurationError("failed to compile first tier");
+                }
+            }
+
+            return hook;
+        }
+
+    private:
+        set<PMPattern> pats;
+        shared_ptr<PMHook> hook = make_shared<PMHook>();
+    };
+
+public:
+    void
+    preload()
+    {
+        function<void()> cb = [&](){ clearAggCache(); };
+        registerConfigPrepareCb(cb);
+        registerConfigLoadCb(cb);
+        registerConfigAbortCb(cb);
+    }
+
+    void
+    init()
+    {
+        ips_metric.init(
+            "IPS Stats",
+            ReportIS::AudienceTeam::AGENT_CORE,
+            ReportIS::IssuingEngine::AGENT_CORE,
+            std::chrono::minutes(10),
+            true,
+            ReportIS::Audience::SECURITY
+        );
+        ips_metric.registerListener();
+        registerListener();
+        table = Singleton::Consume<I_Table>::by<IPSComp>();
+        env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    }
+
+    void
+    fini()
+    {
+        unregisterListener();
+    }
+
+    void
+    upon(const NewTableEntry &) override
+    {
+        if (isSignatureListsEmpty()) return;
+        auto table = Singleton::Consume<I_Table>::by<IPSComp>();
+        table->createState<IPSEntry>();
+        table->getState<IPSEntry>().uponEnteringContext();
+    }
+
+    string getListenerName() const override { return "ips application"; }
+
+    EventVerdict
+    respond(const NewHttpTransactionEvent &event) override
+    {
+        if (isSignatureListsEmpty()) return ACCEPT;
+        table->createState<IPSEntry>();
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        Buffer method(event.getHttpMethod());
+        ips_state.addPendingContext("HTTP_METHOD", method);
+
+        Buffer uri(event.getURI());
+        ips_state.addPendingContext("HTTP_COMPLETE_URL_ENCODED", uri);
+
+        auto decoder = makeVirtualContainer<HexDecoder<'%'>>(event.getURI());
+        vector<u_char> decoded_url(decoder.begin(), decoder.end());
+        auto start = find(decoded_url.begin(), decoded_url.end(), static_cast<u_char>('?'));
+
+        if (start != decoded_url.end()) {
+            vector<u_char> query(start + 1, decoded_url.end());
+            ips_state.addPendingContext("HTTP_QUERY_DECODED", Buffer(move(query)));
+        }
+        vector<u_char> path(decoded_url.begin(), start);
+        ips_state.addPendingContext("HTTP_PATH_DECODED", Buffer(move(path)));
+        ips_state.addPendingContext("HTTP_COMPLETE_URL_DECODED", Buffer(move(decoded_url)));
+
+        Buffer protocol(event.getHttpProtocol());
+        ips_state.addPendingContext("HTTP_PROTOCOL", protocol);
+
+        auto full_line = method + space + uri + space + protocol + line_sep;
+        ips_state.addPendingContext("HTTP_RAW", full_line);
+
+        return INSPECT;
+    }
+
+    static string
+    getHeaderContextName(const Buffer &name)
+    {
+        string name_str = name;
+        transform(name_str.begin(), name_str.end(), name_str.begin(), ::toupper);
+        return "HTTP_REQUEST_HEADER_" + name_str;
+    }
+
+    EventVerdict
+    respond(const HttpRequestHeaderEvent &event) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        auto header_value = event.getKey() + header_sep + event.getValue();
+        ips_state.addPendingContext("HTTP_REQUEST_ONE_HEADER", header_value);
+        auto full_header = header_value + line_sep;
+        ips_state.addPendingContext("HTTP_REQUEST_HEADER", full_header);
+        ips_state.addPendingContext(getHeaderContextName(event.getKey()), event.getValue());
+        ips_state.addPendingContext("HTTP_RAW", full_header);
+
+        auto max_size = getConfigurationWithDefault<uint>(1536, "IPS", "Max Field Size");
+
+        // Add request header for log
+        auto maybe_req_headers_for_log = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log);
+        if (!maybe_req_headers_for_log.ok()) {
+            ips_state.setTransactionData(IPSCommonTypes::requests_header_for_log, header_value);
+        } else if ((maybe_req_headers_for_log.unpack()).size() + log_sep.size() + header_value.size() < max_size) {
+            Buffer request_headers_for_log = maybe_req_headers_for_log.unpack() + log_sep;
+            request_headers_for_log += header_value;
+            ips_state.setTransactionData(IPSCommonTypes::requests_header_for_log, request_headers_for_log);
+        }
+
+        addRequestHdr(event.getKey(), event.getValue());
+        if (event.isLastHeader()) {
+            for (auto &context : ips_state.getPendingContexts()) {
+                if (isDropContext(context.first, context.second)) setDrop(ips_state);
+            }
+            ips_state.clearPendingContexts();
+            if (isDrop(ips_state)) return DROP;
+        }
+
+        return INSPECT;
+    }
+
+    EventVerdict
+    respond(const HttpRequestBodyEvent &event) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        if (isDropContext("HTTP_REQUEST_BODY", event.getData())) setDrop(ips_state);
+
+        if (!ips_state.isFlagSet("HttpRequestData")) {
+            ips_state.setFlag("HttpRequestData");
+            auto data =
+                ips_state.getBuffer("HTTP_METHOD") +
+                space +
+                ips_state.getBuffer("HTTP_COMPLETE_URL_DECODED") +
+                space +
+                ips_state.getBuffer("HTTP_PROTOCOL") +
+                line_sep +
+                ips_state.getBuffer("HTTP_REQUEST_HEADER") +
+                line_sep +
+                event.getData();
+            if (isDropContext("HTTP_REQUEST_DATA", data)) setDrop(ips_state);
+        }
+
+        if (isDropContext("HTTP_RAW", event.getData())) setDrop(ips_state);
+
+        return INSPECT;
+    }
+
+    EventVerdict
+    respond(const EndRequestEvent &) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        if (!ips_state.isFlagSet("HttpRequestData")) {
+            ips_state.setFlag("HttpRequestData");
+            auto data =
+                ips_state.getBuffer("HTTP_METHOD") +
+                space +
+                ips_state.getBuffer("HTTP_COMPLETE_URL_DECODED") +
+                space +
+                ips_state.getBuffer("HTTP_PROTOCOL") +
+                line_sep +
+                ips_state.getBuffer("HTTP_REQUEST_HEADER") +
+                line_sep;
+            if (isDropContext("HTTP_REQUEST_DATA", data)) return DROP;
+        }
+
+        if (isDrop(ips_state)) return DROP;
+
+        if (isContextActive("HTTP_RESPONSE_HEADER")) return INSPECT;
+        if (isContextActive("HTTP_RESPONSE_BODY")) return INSPECT;
+        return ACCEPT;
+    }
+
+    EventVerdict
+    respond(const ResponseCodeEvent &event) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        Buffer buf(reinterpret_cast<const char *>(&event), sizeof(event), Buffer::MemoryType::VOLATILE);
+        if (isDropContext("HTTP_RESPONSE_CODE", buf)) return DROP;
+
+        return INSPECT;
+    }
+
+    EventVerdict
+    respond(const HttpResponseHeaderEvent &event) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        if (isDropContext("HTTP_RESPONSE_HEADER", event.getValue())) return DROP;
+
+        return INSPECT;
+    }
+
+    EventVerdict
+    respond(const HttpResponseBodyEvent &event) override
+    {
+        if (!table->hasState<IPSEntry>()) return ACCEPT;
+
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.uponEnteringContext();
+        auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); });
+
+        if (isDropContext("HTTP_RESPONSE_BODY", event.getData())) return DROP;
+
+        return event.isLastChunk() ? ACCEPT : INSPECT;
+    }
+
+    EventVerdict respond (const EndTransactionEvent &) override { return ACCEPT; }
+
+private:
+    static void setDrop(IPSEntry &state) { state.setDrop(); }
+    static bool isDrop(const IPSEntry &state) { return state.isDrop(); }
+
+    bool
+    isDropContext(const string &name, const Buffer &buf)
+    {
+        auto responeses = ParsedContext(buf, name, 0).query();
+        for (auto &reponse : responeses) {
+            if (reponse == ParsedContextReply::DROP) return true;
+        }
+        return false;
+    }
+
+    static bool
+    isContextActive(const string &context)
+    {
+        return !getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections").isEmpty(context) ||
+            !getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections").isEmpty(context);
+    }
+
+    static bool
+    isSignatureListsEmpty()
+    {
+        return getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections").isEmpty() &&
+            getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections").isEmpty();
+    }
+
+    void
+    addRequestHdr(const Buffer &name, const Buffer &value)
+    {
+        auto &ips_state = table->getState<IPSEntry>();
+        ips_state.setTransactionData(name, value);
+    }
+
+    shared_ptr<PMHook>
+    getHook(const string &context_name, const set<PMPattern> &patterns) override
+    {
+        return tier_aggs[context_name].getHook(patterns);
+    }
+
+    void clearAggCache() { tier_aggs.clear(); }
+
+    I_Table *table = nullptr;
+    I_Environment *env = nullptr;
+    IPSSignatureSubTypes::IPSMetric ips_metric;
+    map<string, SigsFirstTierAgg> tier_aggs;
+};
+
+IPSComp::IPSComp() : Component("IPSComp"), pimpl(make_unique<Impl>()) {}
+
+IPSComp::~IPSComp() {}
+
+void
+IPSComp::preload()
+{
+    registerExpectedResource<IPSSignaturesResource>("IPS", "protections");
+    registerExpectedResource<string>("IPS", "VersionId");
+    registerExpectedResource<SnortSignaturesResource>("IPSSnortSigs", "protections");
+    registerExpectedConfiguration<IPSConfiguration>("IPS", "IpsConfigurations");
+    registerExpectedConfiguration<uint>("IPS", "Max Field Size");
+    registerExpectedConfiguration<IPSSignatures>("IPS", "IpsProtections");
+    registerExpectedConfiguration<SnortSignatures>("IPSSnortSigs", "SnortProtections");
+    registerExpectedConfigFile("ips", Config::ConfigFileType::Policy);
+    registerExpectedConfigFile("ips", Config::ConfigFileType::Data);
+    registerExpectedConfigFile("snort", Config::ConfigFileType::Policy);
+
+    ParameterException::preload();
+
+    pimpl->preload();
+}
+
+void
+IPSComp::init()
+{
+    pimpl->init();
+}
+
+void
+IPSComp::fini()
+{
+    pimpl->fini();
+}

components/security_apps/ips/ips_configuration.cc

@@ -0,0 +1,74 @@
+#include "ips_configuration.h"
+#include "debug.h"
+
+using namespace std;
+
+IPSConfiguration::Context::Context(ContextType _type, uint history) : type(_type), history_size(history) {}
+
+uint
+IPSConfiguration::Context::getHistorySize() const
+{
+    dbgAssert(type == ContextType::HISTORY) << "Try to access history size for non-history context";
+    return history_size;
+}
+
+static const map<string, IPSConfiguration::ContextType> type_convertor = {
+    { "normal",  IPSConfiguration::ContextType::NORMAL },
+    { "keep",    IPSConfiguration::ContextType::KEEP },
+    { "history", IPSConfiguration::ContextType::HISTORY }
+};
+
+class ContextConfigurationJSON
+{
+public:
+    void
+    load(cereal::JSONInputArchive &ar)
+    {
+        string type_name;
+        ar(
+            cereal::make_nvp("name", name),
+            cereal::make_nvp("type", type_name)
+        );
+
+        auto type_pointer = type_convertor.find(type_name);
+        if (type_pointer == type_convertor.end()) reportConfigurationError("Unknown IPS context type: " + type_name);
+        type = type_pointer->second;
+
+        if (type == IPSConfiguration::ContextType::HISTORY) ar(cereal::make_nvp("historySize", size));
+    }
+
+    string getName() const { return name; }
+    IPSConfiguration::Context getContext() const { return IPSConfiguration::Context(type, size); }
+
+private:
+    string name;
+    IPSConfiguration::ContextType type;
+    uint size = 0;
+};
+
+void
+IPSConfiguration::load(cereal::JSONInputArchive &ar)
+{
+    vector<ContextConfigurationJSON> config;
+    ar(cereal::make_nvp("contextsConfiguration", config));
+
+    for (auto &context : config) {
+        context_config.emplace(context.getName(), context.getContext());
+    }
+}
+
+IPSConfiguration::Context
+IPSConfiguration::getContext(const string &name) const
+{
+    auto context = context_config.find(name);
+    if (context == context_config.end()) return IPSConfiguration::Context();
+    return context->second;
+}
+
+uint
+IPSConfiguration::getHistorySize(const string &name) const
+{
+    auto context = context_config.find(name);
+    dbgAssert(context != context_config.end()) << "Try to access history size for non-exiting context";
+    return context->second.getHistorySize();
+}

components/security_apps/ips/ips_entry.cc

@@ -0,0 +1,143 @@
+#include "ips_entry.h"
+#include "ips_signatures.h"
+#include "ips_configuration.h"
+#include "config.h"
+#include "debug.h"
+#include "common.h"
+#include "i_keywords_rule.h"
+#include "helper.h"
+
+using namespace std;
+using namespace IPSHelper;
+
+USE_DEBUG_FLAG(D_IPS);
+
+static const map<string, IPSConfiguration::Context> default_conf_mapping = {
+    { "HTTP_METHOD",               IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_COMPLETE_URL_DECODED", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_PATH_DECODED",         IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_QUERY_DECODED",        IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_PROTOCOL",             IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_REQUEST_HEADER",       IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_REQUEST_BODY",         IPSConfiguration::Context(IPSConfiguration::ContextType::HISTORY, 1000) },
+    { "HTTP_RESPONSE_CODE",        IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_RESPONSE_HEADER",      IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) },
+    { "HTTP_RESPONSE_BODY",        IPSConfiguration::Context(IPSConfiguration::ContextType::HISTORY, 1000) }
+};
+
+static const IPSConfiguration default_conf(default_conf_mapping);
+
+IPSEntry::IPSEntry() : TableOpaqueSerialize<IPSEntry>(this) {}
+
+void
+IPSEntry::upon(const ParsedContext &)
+{
+}
+
+ParsedContextReply
+IPSEntry::respond(const ParsedContext &parsed)
+{
+    const auto &name = parsed.getName();
+    auto buf = parsed.getBuffer();
+
+    dbgDebug(D_IPS) << "Entrying context " << name;
+    dbgTrace(D_IPS) << "Context Content " << dumpHex(buf);
+
+    auto config = getConfigurationWithDefault(default_conf, "IPS", "IpsConfigurations").getContext(name);
+    if (config.getType() == IPSConfiguration::ContextType::HISTORY) {
+        buf = past_contexts[name] + buf;
+    }
+    ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), name);
+    ctx.registerValue(name, buf);
+
+    ctx.activate();
+    auto &signatures = getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections");
+    bool should_drop = signatures.isMatchedPrevent(parsed.getName(), buf);
+    auto &snort_signatures = getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections");
+    should_drop |= snort_signatures.isMatchedPrevent(parsed.getName(), buf);
+    ctx.deactivate();
+
+    switch(config.getType()) {
+        case IPSConfiguration::ContextType::NORMAL: {
+            ctx.unregisterKey<Buffer>(name);
+            break;
+        }
+        case IPSConfiguration::ContextType::KEEP: {
+            past_contexts[name] += buf;
+            ctx.registerValue(name, past_contexts[name]);
+            break;
+        }
+        case IPSConfiguration::ContextType::HISTORY: {
+            if (buf.size() > config.getHistorySize()) buf.keepTail(config.getHistorySize());
+            ctx.registerValue(name, buf);
+            past_contexts[name] = buf;
+            break;
+        }
+    }
+
+    dbgDebug(D_IPS) << "Return " << (should_drop ? "drop" : "continue");
+
+    return should_drop ? ParsedContextReply::DROP : ParsedContextReply::ACCEPT;
+}
+
+Buffer
+IPSEntry::getBuffer(const string &name) const
+{
+    auto elem = past_contexts.find(name);
+    if (elem != past_contexts.end()) return elem->second;
+
+    for (auto &p : pending_contexts) {
+        if (p.first == name) return p.second;
+    }
+
+    return Buffer();
+}
+
+void
+IPSEntry::setTransactionData(const Buffer &key, const Buffer &value)
+{
+    transaction_data[key] = value;
+}
+
+Maybe<Buffer>
+IPSEntry::getTransactionData(const Buffer &key) const
+{
+    map<Buffer, Buffer>::const_iterator iter = transaction_data.find(key);
+    
+    if (iter == transaction_data.end()) {
+        return genError("Http header value not found");
+    }
+
+    return iter->second;
+
+}
+
+string
+IPSEntry::name()
+{
+    return "IPS";
+}
+
+unique_ptr<TableOpaqueBase>
+IPSEntry::prototype()
+{
+    return make_unique<IPSEntry>();
+}
+
+uint
+IPSEntry::currVer()
+{
+    return 0;
+}
+
+uint
+IPSEntry::minVer()
+{
+    return 0;
+}
+
+void
+IPSEntry::addPendingContext(const std::string &name, const Buffer &buffer)
+{
+    pending_contexts.emplace_back(name, buffer);
+}

components/security_apps/ips/ips_metric.cc

@@ -0,0 +1,20 @@
+#include "ips_metric.h"
+
+void
+IPSSignatureSubTypes::IPSMetric::upon(const MatchEvent &event)
+{
+    switch (event.getAction()) {
+        case SignatureAction::PREVENT: {
+            prevented.report(1);
+            break;
+        }
+        case SignatureAction::DETECT: {
+            detected.report(1);
+            break;
+        }
+        case SignatureAction::IGNORE: {
+            ignored.report(1);
+            break;
+        }
+    }
+}

components/security_apps/ips/ips_signatures.cc

@@ -0,0 +1,809 @@
+#include "ips_signatures.h"
+
+#include <sstream>
+#include <algorithm>
+
+#include "ips_comp.h"
+#include "ips_basic_policy.h"
+#include "snort_basic_policy.h"
+#include "generic_rulebase/parameters_config.h"
+#include "generic_rulebase/triggers_config.h"
+#include "rule_detection.h"
+#include "helper.h"
+#include "config.h"
+#include "context.h"
+#include "ips_entry.h"
+#include "ips_metric.h"
+#include "ips_common_types.h"
+
+USE_DEBUG_FLAG(D_IPS);
+
+using namespace IPSSignatureSubTypes;
+using namespace ReportIS;
+using namespace std;
+using MatchType = BaseSignature::MatchType;
+
+static const map<IPSLevel, Severity> severities = {
+    { IPSLevel::CRITICAL,   Severity::CRITICAL },
+    { IPSLevel::HIGH,        Severity::HIGH },
+    { IPSLevel::MEDIUM_HIGH, Severity::HIGH },
+    { IPSLevel::MEDIUM,      Severity::MEDIUM },
+    { IPSLevel::MEDIUM_LOW,  Severity::LOW },
+    { IPSLevel::LOW,         Severity::LOW },
+    { IPSLevel::VERY_LOW,    Severity::INFO }
+};
+
+static const map<string, IPSLevel> levels = {
+    { "Critical",    IPSLevel::CRITICAL },
+    { "High",        IPSLevel::HIGH },
+    { "Medium High", IPSLevel::MEDIUM_HIGH },
+    { "Medium",      IPSLevel::MEDIUM },
+    { "Medium Low",  IPSLevel::MEDIUM_LOW },
+    { "Low",         IPSLevel::LOW },
+    { "Very Low",    IPSLevel::VERY_LOW }
+};
+
+static IPSLevel
+getLevel(const string &level_string, const string &attr_name)
+{
+    auto index = levels.find(level_string);
+    if (index == levels.end()) {
+        reportConfigurationError(
+            "Unknown level: '" + level_string + "' in attribute " + attr_name
+        );
+    }
+    return index->second;
+}
+
+void
+IPSSignatureMetaData::setIndicators(const string &_source, const string &_version)
+{
+    source = _source;
+    version = _version;
+}
+
+string
+IPSSignatureMetaData::getSeverityString() const
+{
+    switch (severity) {
+        case IPSLevel::VERY_LOW:
+            return "Very Low";
+        case IPSLevel::LOW:
+            return "Low";
+        case IPSLevel::MEDIUM_LOW:
+            return "Medium Low";
+        case IPSLevel::MEDIUM:
+            return "Medium";
+        case IPSLevel::MEDIUM_HIGH:
+            return "Medium High";
+        case IPSLevel::HIGH:
+            return "High";
+        case IPSLevel::CRITICAL:
+            return "Critical";
+    }
+
+    dbgAssert(false) << "Illegal severity value: " << static_cast<uint>(severity);
+    return "Critical";
+}
+
+string
+IPSSignatureMetaData::getConfidenceString() const
+{
+    if (confidence <= IPSLevel::LOW) return "Low";
+    if (confidence >= IPSLevel::HIGH) return "High";
+    return "Medium";
+}
+
+string
+IPSSignatureMetaData::getPerformanceString() const
+{
+    switch (performance) {
+        case IPSLevel::VERY_LOW:
+            return "Very Low";
+        case IPSLevel::LOW:
+            return "Low";
+        case IPSLevel::MEDIUM_LOW:
+            return "Medium Low";
+        case IPSLevel::MEDIUM:
+            return "Medium";
+        case IPSLevel::MEDIUM_HIGH:
+            return "Medium High";
+        case IPSLevel::HIGH:
+            return "High";
+        case IPSLevel::CRITICAL:
+            return "Critical";
+    }
+
+    dbgAssert(false) << "Illegal performance value: " << static_cast<uint>(performance);
+    return "Critical";
+}
+
+void
+IPSSignatureMetaData::load(cereal::JSONInputArchive &ar)
+{
+    string severity_string, confidence_string, performance_string;
+    ar(
+        cereal::make_nvp("maintrainId",       protection_id),
+        cereal::make_nvp("protectionName",    sig_name),
+        cereal::make_nvp("severity",          severity_string),
+        cereal::make_nvp("lastUpdate",        update),
+        cereal::make_nvp("confidenceLevel",   confidence_string),
+        cereal::make_nvp("performanceImpact", performance_string),
+        cereal::make_nvp("cveList",           cve_list),
+        cereal::make_nvp("tags",              tag_list)
+    );
+
+    severity = getLevel(severity_string, "severity");
+    confidence = getLevel(confidence_string, "confidence");
+    performance = getLevel(performance_string, "performance");
+
+    try {
+        ar(cereal::make_nvp("logAttackName", event_log));
+    } catch (cereal::Exception &) {
+        event_log = "IPS Signature '" + sig_name + "' Found";
+        ar.setNextName(nullptr);
+    }
+
+    try {
+        ar(cereal::make_nvp("silent", is_silent));
+    } catch (cereal::Exception &) {
+        ar.setNextName(nullptr);
+    }
+}
+
+static const size_t protection_type_pos = strlen("Protection_Type_");
+static const size_t vul_type_pos = strlen("Vul_Type_");
+
+string
+IPSSignatureSubTypes::IPSSignatureMetaData::getIncidentType() const
+{
+    for (auto &tag : tag_list) {
+        if (tag.compare(0, vul_type_pos, "Vul_Type_") == 0) {
+            auto incident_type = tag.substr(vul_type_pos);
+            replace(incident_type.begin(), incident_type.end(), '_', ' ');
+            if (incident_type == "Vulnerability") return "Vulnerability exploit attempt";
+            return incident_type;
+        }
+    }
+
+    for (auto &tag : tag_list) {
+        if (tag.compare(0, protection_type_pos, "Protection_Type_") == 0) {
+            auto incident_type = tag.substr(protection_type_pos);
+            replace(incident_type.begin(), incident_type.end(), '_', ' ');
+            if (incident_type == "Vulnerability") return "Vulnerability exploit attempt";
+            return incident_type;
+        }
+    }
+
+    return "";
+}
+
+static const size_t year_start_pos = strlen("Threat_Year_");
+
+bool
+IPSSignatureMetaData::isYearAtLeast(const Maybe<int> &year) const
+{
+    if (!year.ok()) return true;
+    auto protection_year = getYear();
+    if (!protection_year.ok()) return true;
+
+    return *protection_year >= *year;
+}
+
+Maybe<int>
+IPSSignatureMetaData::getYear() const
+{
+    for (auto &tag : tag_list) {
+        if (tag.compare(0, year_start_pos, "Threat_Year_") == 0) {
+            if (tag.size() != year_start_pos + 4) {
+                dbgWarning(D_IPS) << "Threat year tag (" << tag << ") doen't meet expected format";
+                return false;
+            }
+            int protection_year =
+                (tag[year_start_pos] - '0') * 1000 +
+                (tag[year_start_pos + 1] - '0') * 100 +
+                (tag[year_start_pos + 2] - '0') * 10 +
+                (tag[year_start_pos + 3] - '0');
+            return protection_year;
+        }
+    }
+
+    return genError("Year not found");
+}
+
+void
+CompleteSignature::load(cereal::JSONInputArchive &ar)
+{
+    ar(cereal::make_nvp("protectionMetadata", metadata));
+    RuleDetection rule_detection(metadata.getName());
+    ar(cereal::make_nvp("detectionRules", rule_detection));
+    rule = rule_detection.getRule();
+}
+
+MatchType
+CompleteSignature::getMatch(const set<PMPattern> &matches) const
+{
+    return rule->getMatch(matches);
+}
+
+set<PMPattern>
+CompleteSignature::patternsInSignature() const
+{
+    return rule->patternsInSignature();
+}
+
+void
+CompleteSignature::setIndicators(const string &source, const string &version)
+{
+    metadata.setIndicators(source, version);
+}
+
+template <typename ErrorType>
+static string
+getSubString(const Maybe<Buffer, ErrorType> &buf, uint max_size = 0)
+{
+    if (max_size == 0) max_size = buf.unpack().size();
+    const Buffer &real_buf = buf.unpack();
+    auto res = real_buf.size() <= max_size ? real_buf : real_buf.getSubBuffer(0, max_size);
+    return static_cast<string>(res);
+}
+
+ActionResults
+SignatureAndAction::getAction(const IPSEntry &ips_state) const
+{
+    dbgDebug(D_IPS) << "matching exceptions";
+
+    unordered_map<string, set<string>> exceptions_dict;
+    exceptions_dict["protectionName"].insert(signature->getName());
+
+    ScopedContext ctx;
+    ctx.registerValue<string>("protectionName", signature->getName());
+
+    auto env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    auto host = env->get<string>(HttpTransactionData::host_name_ctx);
+    if (host.ok()) exceptions_dict["hostName"].insert(*host);
+
+    auto client_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+    if (client_ip.ok()) {
+        stringstream client_ip_str;
+        client_ip_str << client_ip.unpack();
+        exceptions_dict["sourceIP"].insert(client_ip_str.str());
+    }
+
+    auto path = ips_state.getBuffer("HTTP_PATH_DECODED");
+    if (path.size()) exceptions_dict["url"].insert(static_cast<string>(path));
+
+    auto env_source_identifier = env->get<string>(HttpTransactionData::source_identifier);
+    if (env_source_identifier.ok()) {
+        exceptions_dict["sourceIdentifier"].insert(*env_source_identifier);
+    }
+
+    I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<IPSComp>();
+    auto behaviors = i_rulebase->getBehavior(exceptions_dict);
+
+    set<BehaviorValue> override_actions;
+    vector<string> override_ids;
+    for (auto const &behavior : behaviors) {
+        if (behavior.getKey() == BehaviorKey::ACTION) {
+            override_actions.insert(behavior.getValue());
+            const string &override_id = behavior.getId();
+            if (!override_id.empty()) override_ids.push_back(override_id);
+        }
+    }
+
+    if (override_actions.find(BehaviorValue::IGNORE) != override_actions.end()) {
+        dbgDebug(D_IPS) << "Exception matched - action=Detect";
+        return make_tuple(IPSSignatureSubTypes::SignatureAction::DETECT, string("Skip"), override_ids);
+    }
+    if (override_actions.find(BehaviorValue::ACCEPT) != override_actions.end()) {
+        dbgDebug(D_IPS) << "Exception matched - action=Detect";
+        return make_tuple(IPSSignatureSubTypes::SignatureAction::DETECT, string("Accept"), override_ids);
+    }
+    if (override_actions.find(BehaviorValue::REJECT) != override_actions.end()) {
+        dbgDebug(D_IPS) << "Exception matched - action=Prevent";
+        return make_tuple(IPSSignatureSubTypes::SignatureAction::PREVENT, string("Drop"), override_ids);
+    }
+    return make_tuple(action, string("None"), override_ids);
+}
+
+static const auto req_body = LogTriggerConf::WebLogFields::webBody;
+static const auto headers = LogTriggerConf::WebLogFields::webHeaders;
+static const auto url_path = LogTriggerConf::WebLogFields::webUrlPath;
+static const auto url_query = LogTriggerConf::WebLogFields::webUrlQuery;
+static const auto res_body = LogTriggerConf::WebLogFields::responseBody;
+static const auto res_code = LogTriggerConf::WebLogFields::responseCode;
+
+bool
+SignatureAndAction::matchSilent(const Buffer &sample) const
+{
+    dbgTrace(D_IPS) << "Matched silent signature";
+    MatchEvent(signature, IPSSignatureSubTypes::SignatureAction::IGNORE).notify();
+
+    ScopedContext ctx;
+    ctx.registerValue("Audience Team", AudienceTeam::SIGNATURE_DEVELOPERS);
+
+    LogGen log(
+        "Silent Protection",
+        Audience::INTERNAL,
+        Severity::INFO,
+        Priority::MEDIUM,
+        LogField("practiceType", "Threat Prevention"),
+        Tags::IPS,
+        StreamType::JSON_FOG
+    );
+    log << LogField("signatureVersion", signature->getUpdateVersion())
+        << LogField("protectionId", signature->getName())
+        << LogField("indicatorsSource", signature->getSource())
+        << LogField("indicatorsVersion", signature->getFeedVersion())
+        << LogField("incidentType", signature->getIncidentType())
+        << LogField("matchedSample", static_cast<string>(sample), LogFieldOption::XORANDB64);
+
+    auto env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    auto table = Singleton::Consume<I_Table>::by<IPSComp>();
+    auto &ips_state = table->getState<IPSEntry>();
+
+    auto method = env->get<string>(HttpTransactionData::method_ctx);
+    if (method.ok()) log << LogField("httpMethod", method.unpack());
+
+    auto path  = env->get<Buffer>("HTTP_PATH_DECODED");
+    if (path.ok()) log << LogField("httpUriPath", getSubString(path, 1536), LogFieldOption::XORANDB64);
+
+    auto req_header = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log);
+    if (req_header.ok()) log << LogField("httpRequestHeaders", getSubString(req_header), LogFieldOption::XORANDB64);
+
+    auto res_code = env->get<Buffer>("HTTP_RESPONSE_CODE");
+    if (res_code.ok()) log << LogField("httpResponseCode", static_cast<string>(res_code.unpack()));
+
+    auto req_body = env->get<Buffer>("HTTP_REQUEST_BODY");
+    auto res_body = env->get<Buffer>("HTTP_RESPONSE_BODY");
+    uint req_size = req_body.ok() ? req_body.unpack().size() : 0;
+    uint res_size = res_body.ok() ? res_body.unpack().size() : 0;
+    if (req_size + res_size > 1536) {
+        if (req_size + 500 > 1536) {
+            res_size = std::min(500u, res_size);
+            req_size = 1536 - res_size;
+        } else {
+            res_size = 1536 - req_size;
+        }
+    }
+    if (req_size) log << LogField("httpRequestBody", getSubString(req_body, req_size), LogFieldOption::XORANDB64);
+    if (res_size) log << LogField("httpResponseBody", getSubString(res_body, res_size), LogFieldOption::XORANDB64);
+
+    return false;
+}
+
+bool
+SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMPattern> &pattern) const
+{
+    if (signature->getMatch(pattern) != MatchType::MATCH) {
+        dbgTrace(D_IPS) << "Signature doesn't match";
+        return false;
+    }
+
+    if (signature->isSilent()) return matchSilent(context_buffer);
+
+    auto table = Singleton::Consume<I_Table>::by<IPSComp>();
+    auto &ips_state = table->getState<IPSEntry>();
+
+    auto override_action = getAction(ips_state);
+
+    MatchEvent(signature, get<0>(override_action)).notify();
+
+    if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::IGNORE) {
+        dbgDebug(D_IPS) << "Ignored signature";
+        return false;
+    }
+
+    dbgDebug(D_IPS) << "Signature matched - sending log";
+
+    auto &trigger = getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log");
+    bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT;
+
+    auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL;
+    if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::DETECT) severity = Severity::INFO;
+
+    LogGen log = trigger(
+        "Web Request",
+        LogTriggerConf::SecurityType::ThreatPrevention,
+        severity,
+        Priority::HIGH,
+        is_prevent,
+        LogField("practiceType", "Threat Prevention"),
+        Tags::IPS
+    );
+    log
+        << LogField("matchedSignatureConfidence", signature->getConfidenceString())
+        << LogField("matchedSignaturePerformance", signature->getPerformanceString())
+        << LogField("matchedSignatureSeverity", signature->getSeverityString())
+        << LogField("matchedSignatureCVE", makeSeparatedStr(signature->getCveList(), ", "))
+        << LogField("signatureVersion", signature->getUpdateVersion())
+        << LogField("protectionId", signature->getName())
+        << LogField("indicatorsSource", signature->getSource())
+        << LogField("indicatorsVersion", signature->getFeedVersion())
+        << LogField("waapIncidentType", signature->getIncidentType());
+
+    if (context_buffer.size() < 1024) {
+        log << LogField("matchedSample", static_cast<string>(context_buffer), LogFieldOption::XORANDB64);
+    } else {
+        auto sample = context_buffer;
+        sample.keepHead(1024);
+        log << LogField("matchedSample", static_cast<string>(sample), LogFieldOption::XORANDB64);
+    }
+
+    auto year = signature->getYear();
+    if (year.ok()) log << LogField("matchedSignatureYear", to_string(*year));
+
+    auto env = Singleton::Consume<I_Environment>::by<IPSComp>();
+    auto host = env->get<string>(HttpTransactionData::host_name_ctx);
+    if (host.ok()) log << LogField("httpHostName", host.unpack());
+    auto client_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+
+    if (client_ip.ok()) {
+        stringstream client_ip_str;
+        client_ip_str << client_ip.unpack();
+        log << LogField("sourceIP", client_ip_str.str());
+    }
+
+    auto proxy_ip = env->get<string>(HttpTransactionData::proxy_ip_ctx);
+    if (proxy_ip.ok()) {
+        log << LogField("proxyIP", static_cast<string>(proxy_ip.unpack()));
+    }
+
+    auto source_identifier = env->get<string>(HttpTransactionData::source_identifier);
+    if (source_identifier.ok()) {
+        log << LogField("httpSourceId", static_cast<string>(source_identifier.unpack()));
+    }
+
+    auto req_header = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log);
+    if (req_header.ok() && trigger.isWebLogFieldActive(headers)) {
+        log << LogField("httpRequestHeaders", static_cast<string>(req_header.unpack()), LogFieldOption::XORANDB64);
+    }
+
+    auto client_port = env->get<uint16_t>(HttpTransactionData::client_port_ctx);
+    if (client_port.ok()) log << LogField("sourcePort", client_port.unpack());
+    auto method = env->get<string>(HttpTransactionData::method_ctx);
+    if (method.ok()) log << LogField("httpMethod", method.unpack());
+    uint max_size = getConfigurationWithDefault<uint>(1536, "IPS", "Max Field Size");
+    auto path  = env->get<Buffer>("HTTP_PATH_DECODED");
+    if (path.ok() && trigger.isWebLogFieldActive(url_path)) {
+        log << LogField("httpUriPath", getSubString(path, max_size), LogFieldOption::XORANDB64);
+    }
+    auto query = env->get<Buffer>("HTTP_QUERY_DECODED");
+    if (query.ok() && trigger.isWebLogFieldActive(url_query)) {
+        log << LogField("httpUriQuery", getSubString(query, max_size), LogFieldOption::XORANDB64);
+    }
+
+    auto res_code = env->get<Buffer>("HTTP_RESPONSE_CODE");
+    if (res_code.ok() && trigger.isWebLogFieldActive(::res_code)) {
+        log << LogField("httpResponseCode", static_cast<string>(res_code.unpack()));
+    }
+
+    auto req_body = env->get<Buffer>("HTTP_REQUEST_BODY");
+    auto res_body = env->get<Buffer>("HTTP_RESPONSE_BODY");
+    uint req_size = req_body.ok() && trigger.isWebLogFieldActive(::req_body) ? req_body.unpack().size() : 0;
+    uint res_size = res_body.ok() && trigger.isWebLogFieldActive(::res_body) ? res_body.unpack().size() : 0;
+    if (req_size + res_size > max_size) {
+        if (req_size + 500 > max_size) {
+            res_size = std::min(500u, res_size);
+            req_size = max_size - res_size;
+        } else {
+            res_size = max_size - req_size;
+        }
+    }
+    if (req_size) log << LogField("httpRequestBody", getSubString(req_body, req_size), LogFieldOption::XORANDB64);
+    if (res_size) log << LogField("httpResponseBody", getSubString(res_body, res_size), LogFieldOption::XORANDB64);
+
+    log << LogField("waapOverride", get<1>(override_action));
+
+    if (!get<2>(override_action).empty()) log.addToOrigin(LogField("exceptionIdList", get<2>(override_action)));
+    
+    log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
+
+    return is_prevent;
+}
+
+void
+IPSSignaturesResource::load(cereal::JSONInputArchive &ar)
+{
+    vector<CompleteSignature> sigs;
+    cereal::load(ar, sigs);
+
+    all_signatures.reserve(sigs.size());
+    for (auto &sig : sigs) {
+        all_signatures.emplace_back(make_shared<CompleteSignature>(move(sig)));
+    }
+}
+
+class CompleteSignatureWrapper
+{
+public:
+    void
+    load(cereal::JSONInputArchive &ar)
+    {
+        try {
+            sig.load(ar);
+            is_loaded = true;
+        } catch (const cereal::Exception &e) {
+            ar.finishNode();
+            reportError(e.what());
+        } catch (const Config::ConfigException &e) {
+            ar.finishNode();
+            reportError(e.getError());
+        }
+    }
+
+    bool isOk() const { return is_loaded; }
+    void setIndicators(const string &source, const string &version) { sig.setIndicators(source, version); }
+    shared_ptr<CompleteSignature> getPtr() { return make_shared<CompleteSignature>(move(sig)); }
+
+private:
+    void
+    reportError(const string &err)
+    {
+        dbgError(D_IPS) << "Failed to load signature due to: " << err;
+
+        if (sig.getName() != "") {
+            string remediation =
+                "Verify the validity of the '" +
+                sig.getName() +
+                "' signature.";
+
+            LogGen(
+                "Could not load a Snort signature from configured file",
+                ReportIS::Level::ACTION,
+                ReportIS::Audience::SECURITY,
+                ReportIS::Severity::CRITICAL,
+                ReportIS::Priority::URGENT,
+                LogField("EventTopic", "Snort Signatures"),
+                ReportIS::Tags::POLICY_INSTALLATION
+            ) << LogField("EventRemediation", remediation);
+        }
+    }
+
+    CompleteSignature sig;
+    bool is_loaded = false;
+};
+
+void
+SnortSignaturesResourceFile::load(cereal::JSONInputArchive &ar)
+{
+    string time;
+    vector<CompleteSignatureWrapper> sigs;
+    ar(
+        cereal::make_nvp("modificationTime", time),
+        cereal::make_nvp("name", name),
+        cereal::make_nvp("protections", sigs)
+    );
+
+    all_signatures.reserve(sigs.size());
+    for (auto &sig : sigs) {
+        if (sig.isOk()) {
+            sig.setIndicators(name, time);
+            all_signatures.emplace_back(sig.getPtr());
+        }
+    }
+}
+
+void
+SnortSignaturesResource::load(cereal::JSONInputArchive &ar)
+{
+    cereal::load(ar, files);
+}
+
+void
+IPSSignaturesPerContext::addSignature(const IPSSignatureSubTypes::SignatureAndAction &sig)
+{
+    auto patterns = sig.patternsInSignature();
+
+    if (patterns.empty()) {
+        signatures_without_lss.push_back(sig);
+        return;
+    }
+
+    for (auto &pat : patterns) {
+        signatures_per_lss[pat].push_back(sig);
+    }
+}
+
+void
+IPSSignaturesPerContext::calcFirstTier(const string &ctx_name)
+{
+    std::set<PMPattern> patterns;
+    for (const auto &lss_to_sig : signatures_per_lss) {
+        patterns.emplace(lss_to_sig.first);
+    }
+
+    first_tier = Singleton::Consume<I_FirstTierAgg>::by<IPSSignaturesPerContext>()->getHook(ctx_name, patterns);
+}
+
+set<PMPattern>
+IPSSignaturesPerContext::getFirstTierMatches(const Buffer &buffer) const
+{
+    return first_tier->ok() ? first_tier->scanBuf(buffer) : set<PMPattern>();
+}
+
+bool
+IPSSignaturesPerContext::isMatchedPrevent(const Buffer &context_buffer) const
+{
+    auto first_tier_res = getFirstTierMatches(context_buffer);
+
+    for (auto &pat : first_tier_res) {
+        auto find = signatures_per_lss.find(pat);
+        if (find == signatures_per_lss.end()) continue;
+        for (auto &sig : find->second) {
+            if (sig.isMatchedPrevent(context_buffer, first_tier_res)) return true;
+        }
+    }
+
+    for (auto &sig : signatures_without_lss) {
+        if (sig.isMatchedPrevent(context_buffer, first_tier_res)) return true;
+    }
+
+    return false;
+}
+
+void
+IPSSignatures::load(cereal::JSONInputArchive &ar)
+{
+    ar(
+        cereal::make_nvp("assetName", asset_name),
+        cereal::make_nvp("practiceName", practice_name)
+    );
+
+    try {
+        ar(cereal::make_nvp("assetId", asset_id));
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        asset_id = "";
+    }
+
+    try {
+        ar(cereal::make_nvp("practiceId", practice_id));
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        practice_id = "";
+    }
+
+    try {
+        ar(cereal::make_nvp("sourceIdentifier", source_id));
+        for (auto &ch : source_id) {
+            ch = tolower(ch);
+        }
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        source_id = "";
+    }
+
+    RuleSelector ruleSelector;
+    ruleSelector.load(ar);
+    std::vector<IPSSignatureSubTypes::SignatureAndAction> signatures = ruleSelector.selectSignatures();
+
+    if (signatures.empty()) {
+        dbgDebug(D_IPS) << "[IPS] Could not find any match between rules and signatures.";
+        return;
+    }
+
+    for (const auto &sig : signatures) {
+        auto &sig_contexts = sig.getContext();
+        for (auto &sig_context : sig_contexts) {
+            signatures_per_context[sig_context].addSignature(sig);
+        }
+    }
+
+    for (auto &sig_per_ctx : signatures_per_context) {
+        sig_per_ctx.second.calcFirstTier(sig_per_ctx.first);
+    }
+}
+
+bool
+IPSSignatures::isMatchedPrevent(const string &context_name, const Buffer &context_buffer) const
+{
+    auto curr_sig = signatures_per_context.find(context_name);
+
+    if (curr_sig == signatures_per_context.end()) {
+        dbgDebug(D_IPS) << "[IPS] No signatures for " << context_name;
+        return false;
+    }
+
+    auto &config = getConfiguration<IPSSignatures>("IPS", "IpsProtections");
+    ScopedContext ctx;
+    auto SOURCE = EnvKeyAttr::LogSection::SOURCE;
+    if (config.ok()) {
+        ctx.registerValue<string>("practiceName", (*config).getPractice(), SOURCE);
+        ctx.registerValue<string>("practiceId", (*config).getPracticeId(), SOURCE);
+    }
+    ctx.registerValue<string>("practiceSubType", "Web IPS", SOURCE);
+    auto is_matched = curr_sig->second.isMatchedPrevent(context_buffer);
+
+    return is_matched;
+}
+
+bool
+IPSSignatures::isEmpty(const std::string &context) const
+{
+    return signatures_per_context.find(context) == signatures_per_context.end();
+}
+
+void
+SnortSignatures::load(cereal::JSONInputArchive &ar)
+{
+    ar(
+        cereal::make_nvp("assetName", asset_name),
+        cereal::make_nvp("practiceName", practice_name)
+    );
+
+    try {
+        ar(cereal::make_nvp("assetId", asset_id));
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        asset_id = "";
+    }
+
+    try {
+        ar(cereal::make_nvp("practiceId", practice_id));
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        practice_id = "";
+    }
+
+    try {
+        ar(cereal::make_nvp("sourceIdentifier", source_id));
+        for (auto &ch : source_id) {
+            ch = tolower(ch);
+        }
+    } catch (const cereal::Exception &e) {
+        ar.setNextName(nullptr);
+        source_id = "";
+    }
+
+    SnortRuleSelector ruleSelector;
+    ruleSelector.load(ar);
+    std::vector<IPSSignatureSubTypes::SignatureAndAction> signatures = ruleSelector.selectSignatures();
+
+    if (signatures.empty()) {
+        dbgDebug(D_IPS) << "[Snort] Could not find any match between rules and signatures.";
+        return;
+    }
+
+    for (const auto &sig : signatures) {
+        auto &sig_contexts = sig.getContext();
+        for (auto &sig_context : sig_contexts) {
+            signatures_per_context[sig_context].addSignature(sig);
+        }
+    }
+
+    for (auto &sig_per_ctx: signatures_per_context) {
+        sig_per_ctx.second.calcFirstTier(sig_per_ctx.first);
+    }
+}
+
+bool
+SnortSignatures::isMatchedPrevent(const string &context_name, const Buffer &context_buffer) const
+{
+    auto curr_sig = signatures_per_context.find(context_name);
+
+    if (curr_sig == signatures_per_context.end()) {
+        dbgDebug(D_IPS) << "[Snort] No signatures for " << context_name;
+        return false;
+    }
+
+    auto &config = getConfiguration<SnortSignatures>("IPSSnortSigs", "SnortProtections");
+    ScopedContext ctx;
+    auto SOURCE = EnvKeyAttr::LogSection::SOURCE;
+    if (config.ok()) {
+        ctx.registerValue<string>("assetName", (*config).getAsset(), SOURCE);
+        ctx.registerValue<string>("assetId", (*config).getAssetId(), SOURCE);
+        ctx.registerValue<string>("practiceName", (*config).getPractice(), SOURCE);
+        ctx.registerValue<string>("practiceId", (*config).getPracticeId(), SOURCE);
+    }
+    ctx.registerValue<string>("practiceSubType", "Web Snort", SOURCE);
+    auto is_matched = curr_sig->second.isMatchedPrevent(context_buffer);
+
+    return is_matched;
+}
+
+bool
+SnortSignatures::isEmpty(const std::string &context) const
+{
+    return signatures_per_context.find(context) == signatures_per_context.end();
+}

components/security_apps/ips/ips_ut/CMakeLists.txt

@@ -0,0 +1,7 @@
+link_directories(${CMAKE_BINARY_DIR}/core/shmem_ipc)
+
+add_unit_test(
+    ips_ut
+    "signatures_ut.cc;entry_ut.cc;component_ut.cc;configuration.cc;rule_selector_ut.cc;compound_ut.cc;resource_ut.cc"
+    "ips;keywords;pcre2-8;intelligence_is_v2;logging;compression_utils;agent_details;time_proxy;event_is;table;http_transaction_data;nginx_attachment;connkey;pm;metric;encryptor;generic_rulebase;generic_rulebase_evaluators;compression_utils;ip_utilities;-lboost_regex;-lcrypto;-lz"
+)

components/security_apps/ips/ips_ut/component_ut.cc

@@ -0,0 +1,913 @@
+#include "ips_comp.h"
+
+#include <sstream>
+
+#include "cptest.h"
+#include "ips_entry.h"
+#include "new_table_entry.h"
+#include "keyword_comp.h"
+#include "environment.h"
+#include "mock/mock_table.h"
+#include "config.h"
+#include "http_manager.h"
+#include "config_component.h"
+#include "agent_details.h"
+#include "mock/mock_logging.h"
+#include "mock/mock_time_get.h"
+#include "mock/mock_mainloop.h"
+#include "encryptor.h"
+#include "generic_rulebase/generic_rulebase.h"
+#include "generic_rulebase/triggers_config.h"
+
+using namespace testing;
+using namespace std;
+
+class ComponentTest : public Test
+{
+public:
+    ComponentTest()
+    {
+        comp.preload();
+        comp.init();
+    }
+
+    ~ComponentTest()
+    {
+        comp.fini();
+    }
+
+    void
+    loadPolicy(const string &config)
+    {
+        stringstream ss;
+        ss << config;
+        Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(ss);
+    }
+
+    void
+    setTrigger()
+    {
+        string log_trigger(
+            "{"
+            "    \"context\": \"triggerId(5eaeefde6765c30010bae8b6)\","
+            "    \"triggerName\": \"Logging Trigger\","
+            "    \"triggerType\": \"log\","
+            "    \"urlForSyslog\": \"\","
+            "    \"urlForCef\": \"128.1.1.1:333\","
+            "    \"acAllow\": false,"
+            "    \"acDrop\": true,"
+            "    \"complianceViolations\": true,"
+            "    \"complianceWarnings\": true,"
+            "    \"logToAgent\": true,"
+            "    \"logToCloud\": true,"
+            "    \"logToSyslog\": false,"
+            "    \"logToCef\": true,"
+            "    \"tpDetect\": true,"
+            "    \"tpPrevent\": true,"
+            "    \"verbosity\": \"Standard\","
+            "    \"webBody\": true,"
+            "    \"webHeaders\": true,"
+            "    \"webRequests\": true,"
+            "    \"webUrlPath\": true,"
+            "    \"webUrlQuery\": true"
+            "}"
+        );
+
+        stringstream ss(log_trigger);
+        cereal::JSONInputArchive ar(ss);
+        LogTriggerConf trigger;
+        trigger.load(ar);
+
+        setConfiguration(trigger, "rulebase", "log");
+    }
+
+    IPSComp comp;
+    StrictMock<MockTable> table;
+    IPSEntry entry;
+
+    GenericRulebase generic_rulebase;
+    ConfigComponent conf;
+    Encryptor encryptor;
+    KeywordComp keywords;
+    ::Environment env;
+    AgentDetails details;
+    NiceMock<MockLogging> logs;
+    NiceMock<MockTimeGet> time;
+    NiceMock<MockMainLoop> mainloop;
+    static const EventVerdict inspect;
+    static const EventVerdict accept;
+    static const EventVerdict drop;
+
+    HttpHeader end_headers{Buffer(""), Buffer(""), 0, true};
+};
+
+static bool
+operator ==(const EventVerdict &first, const EventVerdict &second)
+{
+    return first.getVerdict() == second.getVerdict();
+}
+
+const EventVerdict ComponentTest::inspect(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
+
+const EventVerdict ComponentTest::accept(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT);
+
+const EventVerdict ComponentTest::drop(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
+
+TEST_F(ComponentTest, check_init_fini_do_not_crush)
+{
+}
+
+TEST_F(ComponentTest, new_table_entry_with_empty_configuration)
+{
+    NewTableEntry().notify();
+}
+
+TEST_F(ComponentTest, new_table_entry_with_configuration)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"configurations\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"contextsConfiguration\": ["
+                            "{"
+                                "\"type\": \"keep\","
+                                "\"name\": \"HTTP_REQUEST_BODY\""
+                            "}"
+                        "]"
+                    "}"
+                "],"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"ddd\\\";\","
+                            "\"context\": [\"HTTP_REQUEST_BODY\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Detect\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Prevent\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+
+    loadPolicy(config);
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillOnce(Return(&entry));
+    NewTableEntry().notify();
+}
+
+TEST_F(ComponentTest, events)
+{
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+    IPSEntry entry;
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+
+    HttpTransactionData transaction;
+    EXPECT_THAT(NewHttpTransactionEvent(transaction).query(), ElementsAre(accept));
+    HttpHeader header_req(Buffer("key"), Buffer("val"), 1);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req).query(), ElementsAre(inspect));
+    HttpBody body_req(Buffer("data"), 0, true);
+    EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(accept));
+
+    EXPECT_THAT(ResponseCodeEvent(200).query(), ElementsAre(inspect));
+    HttpHeader header_res(Buffer("key"), Buffer("val"), 2);
+    EXPECT_THAT(HttpResponseHeaderEvent(header_res).query(), ElementsAre(inspect));
+    HttpBody body_res(Buffer("data"), true, 0);
+    EXPECT_THAT(HttpResponseBodyEvent(body_res, Buffer()).query(), ElementsAre(accept));
+    EXPECT_THAT(EndTransactionEvent().performNamedQuery(), ElementsAre(Pair("ips application", accept)));
+}
+
+TEST_F(ComponentTest, check_url_decoding)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"d d\\\";\","
+                            "\"context\": [\"HTTP_COMPLETE_URL_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Detect\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Prevent\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(drop));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop));
+}
+
+TEST_F(ComponentTest, check_query)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"g=#\\\";\","
+                            "\"context\": [\"HTTP_QUERY_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Detect\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Prevent\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(drop));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop));
+}
+
+TEST_F(ComponentTest, check_query_detect_mode)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"d d\\\";\","
+                            "\"context\": [\"HTTP_QUERY_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Detect\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Detect\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillOnce(Return(&entry));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndTransactionEvent().query(), ElementsAre(accept));
+}
+
+TEST_F(ComponentTest, check_query_inactive_mode)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"g=#\\\";\","
+                            "\"context\": [\"HTTP_QUERY_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Prevent\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Inactive\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(accept));
+}
+
+TEST_F(ComponentTest, check_query_silent_mode)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"silent\": true,"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"g=#\\\";\","
+                            "\"context\": [\"HTTP_QUERY_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Prevent\","
+                        "\"rules\": []"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(accept));
+}
+
+TEST_F(ComponentTest, check_filtering_by_year)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [ \"ggg\", \"Threat_Year_2014\", \"hhh\" ],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"g=#\\\";\","
+                            "\"context\": [\"HTTP_QUERY_DECODED\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Prevent\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Inactive\","
+                                "\"protectionsFromYear\": 2013"
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "GET",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(accept));
+}
+
+TEST_F(ComponentTest, log_fields)
+{
+    string config =
+        "{"
+            "\"IPS\": {"
+                "\"Max Field Size\": ["
+                    "{"
+                        "\"value\": 25"
+                    "}"
+                "],"
+                "\"protections\": ["
+                    "{"
+                        "\"protectionMetadata\": {"
+                            "\"protectionName\": \"Test\","
+                            "\"maintrainId\": \"101\","
+                            "\"severity\": \"Low\","
+                            "\"confidenceLevel\": \"Low\","
+                            "\"performanceImpact\": \"Medium High\","
+                            "\"lastUpdate\": \"20210420\","
+                            "\"tags\": [],"
+                            "\"cveList\": []"
+                        "},"
+                        "\"detectionRules\": {"
+                            "\"type\": \"simple\","
+                            "\"SSM\": \"\","
+                            "\"keywords\": \"data: \\\"ddd\\\";\","
+                            "\"context\": [\"HTTP_REQUEST_BODY\"]"
+                        "}"
+                    "}"
+                "],"
+                "\"IpsProtections\": ["
+                    "{"
+                        "\"context\": \"\","
+                        "\"ruleName\": \"rule1\","
+                        "\"assetName\": \"asset1\","
+                        "\"assetId\": \"1-1-1\","
+                        "\"practiceId\": \"2-2-2\","
+                        "\"practiceName\": \"practice1\","
+                        "\"defaultAction\": \"Detect\","
+                        "\"rules\": ["
+                            "{"
+                                "\"action\": \"Prevent\","
+                                "\"severityLevel\": \"Low or above\","
+                                "\"performanceImpact\": \"High or lower\","
+                                "\"confidenceLevel\": \"Low\""
+                            "}"
+                        "]"
+                    "}"
+                "]"
+            "}"
+        "}";
+    loadPolicy(config);
+    setTrigger();
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    Report report;
+    EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "POST",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 1);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect));
+    HttpHeader header_req2(Buffer("key2"), Buffer("val2"), 2);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req2).query(), ElementsAre(inspect));
+    HttpHeader header_req3(Buffer("key3"), Buffer("val3"), 3);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req3).query(), ElementsAre(inspect));
+    string body_str("data: ddd");
+    HttpBody body_req(Buffer(body_str), 0, true);
+    EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop));
+
+    EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestHeaders=\"key1: val1, key2: val2\""));
+    EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestBody=\"" + body_str + "\""));
+    EXPECT_THAT(report.getSyslog(), HasSubstr("signatureVersion=\"20210420\""));
+}
+
+TEST_F(ComponentTest, log_field_httpRequestHeader)
+{
+    string config =
+        "{"
+        "\"IPS\": {"
+        "\"Max Field Size\": ["
+        "{"
+        "\"value\": 25"
+        "}"
+        "],"
+        "\"protections\": ["
+        "{"
+        "\"protectionMetadata\": {"
+        "\"protectionName\": \"Test\","
+        "\"maintrainId\": \"101\","
+        "\"severity\": \"Low\","
+        "\"confidenceLevel\": \"Low\","
+        "\"performanceImpact\": \"Medium High\","
+        "\"lastUpdate\": \"20210420\","
+        "\"tags\": [],"
+        "\"cveList\": []"
+        "},"
+        "\"detectionRules\": {"
+        "\"type\": \"simple\","
+        "\"SSM\": \"\","
+        "\"keywords\": \"data: \\\"ddd\\\";\","
+        "\"context\": [\"HTTP_REQUEST_BODY\"]"
+        "}"
+        "}"
+        "],"
+        "\"IpsProtections\": ["
+        "{"
+        "\"context\": \"\","
+        "\"ruleName\": \"rule1\","
+        "\"assetName\": \"asset1\","
+        "\"assetId\": \"1-1-1\","
+        "\"practiceId\": \"2-2-2\","
+        "\"practiceName\": \"practice1\","
+        "\"defaultAction\": \"Detect\","
+        "\"rules\": ["
+        "{"
+        "\"action\": \"Prevent\","
+        "\"severityLevel\": \"Low or above\","
+        "\"performanceImpact\": \"High or lower\","
+        "\"confidenceLevel\": \"Low\""
+        "}"
+        "]"
+        "}"
+        "]"
+        "}"
+        "}";
+    loadPolicy(config);
+    setTrigger();
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    IPSEntry entry;
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    Report report;
+    EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "POST",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 1);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect));
+    HttpBody body_req(Buffer("data: ddd"), 0, true);
+    EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop));
+
+    EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestHeaders=\"key1: val1\""));
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    IPSEntry entry1;
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry1));
+    Report report1;
+    EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report1));
+
+    HttpTransactionData new_transaction2(
+        "1.1",
+        "POST",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "d%20d?g=%23",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction2).query(), ElementsAre(inspect));
+    HttpHeader header_req2(Buffer("key2"), Buffer("val2"), 1);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req2).query(), ElementsAre(inspect));
+
+    HttpBody body_req2(Buffer("data: ddd"), 0, true);
+    EXPECT_THAT(HttpRequestBodyEvent(body_req2, Buffer()).query(), ElementsAre(inspect));
+    EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop));
+
+    EXPECT_THAT(report1.getSyslog(), HasSubstr("httpRequestHeaders=\"key2: val2\""));
+}
+
+TEST_F(ComponentTest, prxeem_exception_bug)
+{
+    generic_rulebase.preload();
+    generic_rulebase.init();
+    string config =
+        "{"
+        "    \"IPS\": {"
+        "        \"protections\": ["
+        "            {"
+        "                \"protectionMetadata\": {"
+        "                    \"protectionName\": \"Null HTTP Encodings\","
+        "                    \"maintrainId\": \"101\","
+        "                    \"severity\": \"Low\","
+        "                    \"confidenceLevel\": \"Low\","
+        "                    \"performanceImpact\": \"Medium High\","
+        "                    \"lastUpdate\": \"20210420\","
+        "                    \"tags\": [],"
+        "                    \"cveList\": []"
+        "                },"
+        "                \"detectionRules\": {"
+        "                    \"type\": \"simple\","
+        "                    \"SSM\": \"\","
+        "                    \"keywords\": \"data: \\\"|25|00\\\"; data: \\\"?\\\";\","
+        "                    \"context\": [\"HTTP_COMPLETE_URL_ENCODED\"]"
+        "                }"
+        "            }"
+        "        ],"
+        "        \"IpsProtections\": ["
+        "            {"
+        "                \"context\": \"\","
+        "                \"ruleName\": \"rule1\","
+        "                \"assetName\": \"asset1\","
+        "                \"assetId\": \"1-1-1\","
+        "                \"practiceId\": \"2-2-2\","
+        "                \"practiceName\": \"practice1\","
+        "                \"defaultAction\": \"Prevent\","
+        "                \"rules\": []"
+        "            }"
+        "        ]"
+        "    },"
+        "    \"rulebase\": {"
+        "        \"rulesConfig\": ["
+        "            {"
+        "                \"context\": \"All()\","
+        "                \"priority\": 1,"
+        "                \"ruleId\": \"5eaef0726765c30010bae8bb\","
+        "                \"ruleName\": \"Acme web API\","
+        "                \"assetId\": \"5e243effd858007660b758ad\","
+        "                \"assetName\": \"Acme Power API\","
+        "                \"parameters\": ["
+        "                    {"
+        "                       \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\","
+        "                       \"parameterType\": \"exceptions\","
+        "                       \"parameterName\": \"exception\""
+        "                    }"
+        "                ],"
+        "                \"zoneId\": \"\","
+        "                \"zoneName\": \"\""
+        "            }"
+        "        ],"
+        "        \"exception\": ["
+        "            {"
+        "                \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764003)\","
+        "                \"match\": {"
+        "                   \"type\": \"operator\","
+        "                   \"op\": \"and\","
+        "                   \"items\": [{"
+        "                       \"type\": \"condition\","
+        "                       \"op\": \"equals\","
+        "                       \"key\": \"url\","
+        "                       \"value\": [\"(/en|/de)?/admin/helpdesk/dashboard/operator/advanced_search.*\"]"
+        "                   }, {"
+        "                       \"type\": \"operator\","
+        "                       \"op\": \"or\","
+        "                       \"items\": [{"
+        "                           \"type\": \"condition\","
+        "                           \"op\": \"equals\","
+        "                           \"key\": \"protectionName\","
+        "                           \"value\": [\"Null HTTP Encodings\"]"
+        "                       }, {"
+        "                           \"type\": \"condition\","
+        "                           \"op\": \"equals\","
+        "                           \"key\": \"parameterName\","
+        "                           \"value\": [\"op\\\\.submit\\\\.reset\"]"
+        "                       }]"
+        "                    }]"
+        "                },"
+        "                \"behavior\": {"
+        "                    \"key\": \"action\","
+        "                    \"value\": \"accept\""
+        "                }"
+        "            }"
+        "        ]"
+        "    }"
+        "}";
+    loadPolicy(config);
+
+
+    EXPECT_CALL(table, createStateRValueRemoved(_, _));
+    IPSEntry entry;
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true));
+
+    HttpTransactionData new_transaction(
+        "1.1",
+        "POST",
+        "ffff",
+        IPAddr::createIPAddr("0.0.0.0").unpack(),
+        80,
+        "/admin/helpdesk/dashboard/operator/advanced_search?order=created&stuff=%00",
+        IPAddr::createIPAddr("1.1.1.1").unpack(),
+        5428
+    );
+
+    EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect));
+    HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 0, true);
+    EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect));
+}

components/security_apps/ips/ips_ut/compound_ut.cc

@@ -0,0 +1,155 @@
+#include "compound_protection.h"
+
+#include "cptest.h"
+#include "ips_entry.h"
+#include "mock/mock_table.h"
+#include "environment.h"
+#include "i_keywords_rule.h"
+
+using namespace testing;
+using namespace std;
+
+class CompoundTest : public Test
+{
+public:
+    CompoundTest()
+    {
+        ON_CALL(table, hasState(_)).WillByDefault(Return(true));
+        ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state));
+    }
+
+    template <typename ... PatternContextPair>
+    shared_ptr<IPSSignatureSubTypes::BaseSignature>
+    loadSig(const string &name, const string &operation, PatternContextPair ... pat_ctx)
+    {
+        stringstream ss;
+        ss  << "{"
+                << "\"type\": \"compound\","
+                << "\"operation\": \"" << operation << "\","
+                << "\"operands\": [";
+        getSginatureStream(ss, pat_ctx ...);
+        ss  << "]" << "}";
+
+        cereal::JSONInputArchive ar(ss);
+
+        return CompoundProtection::get(name, ar);
+    }
+
+    template <typename ... Strings>
+    set<PMPattern>
+    turnToPatternSet(const Strings & ... strings)
+    {
+        pat_set.clear();
+        populatePatternSet(strings ...);
+        return pat_set;
+    }
+
+    void setActiveContext(const string &name) { ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), name); }
+
+private:
+    void populatePatternSet() {}
+
+    template <typename ... Strings>
+    void
+    populatePatternSet(const string &pat, const Strings & ... strings)
+    {
+        pat_set.emplace(pat, false, false);
+        populatePatternSet(strings ...);
+    }
+
+    ostream &
+    getSginatureStream(ostream &ss, const string &pattern, const string &context)
+    {
+        ss  << "{"
+                << "\"type\": \"simple\","
+                << "\"SSM\": \"" << pattern << "\","
+                << "\"keywords\": \"\","
+                << "\"context\": ["
+                    << "\"" << context << "\""
+                << "]"
+            << "}";
+        return ss;
+    }
+
+    template <typename ... PatternContextPair>
+    ostream &
+    getSginatureStream(ostream &ss, const string &pattern, const string &context, PatternContextPair ... pat_ctx)
+    {
+        return getSginatureStream(getSginatureStream(ss, pattern, context) << ",", pat_ctx ...);
+    }
+
+    NiceMock<MockTable> table;
+    IPSEntry ips_state;
+    set<PMPattern> pat_set;
+    ::Environment env;
+    ScopedContext ctx;
+};
+
+TEST_F(CompoundTest, BasicLoading)
+{
+    auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+    EXPECT_NE(sig, nullptr);
+    EXPECT_EQ(sig->getSigId(), "Test");
+    EXPECT_THAT(sig->getContext(), ElementsAre("HTTP_REQUEST_DATA", "HTTP_RESPONSE_DATA"));
+    EXPECT_EQ(sig->patternsInSignature(),  turnToPatternSet("aaa", "bbb"));
+}
+
+TEST_F(CompoundTest, BasicOrTest)
+{
+    auto sig = loadSig("Test", "or", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("NO_CONTEXT");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH);
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("ddd")), IPSSignatureSubTypes::BaseSignature::MatchType::CACHE_MATCH);
+}
+
+TEST_F(CompoundTest, BasicOrOrderTest)
+{
+    auto sig = loadSig("Test", "or", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("HTTP_RESPONSE_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH);
+}
+
+TEST_F(CompoundTest, BasicAndTest)
+{
+    auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+    setActiveContext("HTTP_RESPONSE_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH);
+}
+
+TEST_F(CompoundTest, BasicAndOrderTest)
+{
+    auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("HTTP_RESPONSE_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH);
+}
+
+TEST_F(CompoundTest, BasicOrderedAndTest)
+{
+    auto sig = loadSig("Test", "ordered_and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+    setActiveContext("HTTP_RESPONSE_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH);
+}
+
+TEST_F(CompoundTest, BasicOrderedAndOrderTest)
+{
+    auto sig = loadSig("Test", "ordered_and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA");
+
+    setActiveContext("HTTP_RESPONSE_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+    setActiveContext("HTTP_REQUEST_DATA");
+    EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH);
+}

components/security_apps/ips/ips_ut/configuration.cc

@@ -0,0 +1,55 @@
+#include "ips_configuration.h"
+#include "cptest.h"
+
+TEST(configuration, basic_context)
+{
+    cptestPrepareToDie();
+
+    IPSConfiguration::Context ctx1(IPSConfiguration::ContextType::HISTORY, 254);
+    EXPECT_EQ(ctx1.getType(), IPSConfiguration::ContextType::HISTORY);
+    EXPECT_EQ(ctx1.getHistorySize(), 254);
+
+    IPSConfiguration::Context ctx2(IPSConfiguration::ContextType::NORMAL, 0);
+    EXPECT_EQ(ctx2.getType(), IPSConfiguration::ContextType::NORMAL);
+    EXPECT_DEATH(ctx2.getHistorySize(), "Try to access history size for non-history context");
+}
+
+
+TEST(configuration, read_configuration)
+{
+    cptestPrepareToDie();
+
+    std::stringstream conf_str;
+    conf_str <<
+        "{"
+            "\"contextsConfiguration\": ["
+                "{"
+                    "\"name\": \"HTTP_REQUEST_BODY\","
+                    "\"type\": \"history\","
+                    "\"historySize\": 100"
+                "},"
+                "{"
+                    "\"name\": \"HTTP_REQUEST_HEADER\","
+                    "\"type\": \"keep\""
+                "}"
+            "]"
+        "}";
+
+    cereal::JSONInputArchive ar(conf_str);
+
+    IPSConfiguration conf;
+    conf.load(ar);
+
+    auto body = conf.getContext("HTTP_REQUEST_BODY");
+    EXPECT_EQ(body.getType(), IPSConfiguration::ContextType::HISTORY);
+    EXPECT_EQ(conf.getHistorySize("HTTP_REQUEST_BODY"), 100);
+
+    auto header = conf.getContext("HTTP_REQUEST_HEADER");
+    EXPECT_EQ(header.getType(), IPSConfiguration::ContextType::KEEP);
+    EXPECT_DEATH(conf.getHistorySize("HTTP_REQUEST_HEADER"), "Try to access history size for non-history context");
+
+    auto line = conf.getContext("HTTP_REQUEST_LINE");
+    EXPECT_EQ(line.getType(), IPSConfiguration::ContextType::NORMAL);
+
+    EXPECT_DEATH(conf.getHistorySize("NO_CONTEXT"), "Try to access history size for non-exiting context");
+}

components/security_apps/ips/ips_ut/entry_ut.cc

@@ -0,0 +1,268 @@
+#include "ips_entry.h"
+
+#include <sstream>
+
+#include "ips_signatures.h"
+#include "cptest.h"
+#include "keyword_comp.h"
+#include "config.h"
+#include "config_component.h"
+#include "environment.h"
+#include "agent_details.h"
+#include "mock/mock_logging.h"
+#include "mock/mock_time_get.h"
+#include "mock/mock_mainloop.h"
+#include "mock/mock_table.h"
+#include "generic_rulebase/generic_rulebase.h"
+
+using namespace std;
+using namespace testing;
+
+ostream &
+operator<<(ostream &os, const ParsedContextReply &action)
+{
+    return os << (action==ParsedContextReply::ACCEPT ? "ACCEPT" : "DROP");
+}
+
+class MockAgg : Singleton::Provide<I_FirstTierAgg>::SelfInterface
+{
+    shared_ptr<PMHook>
+    getHook(const string &, const set<PMPattern> &pats) override
+    {
+        auto hook = make_shared<PMHook>();
+        hook->prepare(pats);
+        return hook;
+    }
+};
+
+class EntryTest : public Test
+{
+public:
+    EntryTest()
+    {
+        ON_CALL(table, getState(_)).WillByDefault(Return(ptr));
+    }
+
+    void
+    loadSignatures(const string &sigs)
+    {
+        {
+            stringstream ss;
+            ss << "[" << sigs << "]";
+            cereal::JSONInputArchive ar(ss);
+            IPSSignaturesResource resource;
+            resource.load(ar);
+            setResource(resource, "IPS", "protections");
+        }
+        {
+            stringstream ss;
+            ss << "{";
+            ss << "\"context\": \"\",";
+            ss << "\"ruleName\": \"rule1\",";
+            ss << "\"assetName\": \"asset1\",";
+            ss << "\"assetId\": \"1-1-1\",";
+            ss << "\"practiceId\": \"2-2-2\",";
+            ss << "\"practiceName\": \"practice1\",";
+            ss << "\"defaultAction\": \"Detect\",";
+            ss << "\"rules\": [";
+            ss << "{";
+            ss << "\"action\": \"Prevent\",";
+            ss << "\"performanceImpact\": \"High or lower\",";
+            ss << "\"severityLevel\": \"Low or above\",";
+            ss << "\"confidenceLevel\": \"Low\"";
+            ss << "}";
+            ss << "]";
+            ss << "}";
+            cereal::JSONInputArchive ar(ss);
+            IPSSignatures signatures;
+            signatures.load(ar);
+            setConfiguration(signatures, "IPS", "IpsProtections");
+        }
+    }
+
+    void
+    loadSnortSignatures(const string &sigs)
+    {
+        {
+            stringstream ss;
+            ss << "[{ \"modificationTime\": \"22/02/08\", \"name\": \"rules1\", \"protections\": [" << sigs << "] }]";
+            cereal::JSONInputArchive ar(ss);
+            SnortSignaturesResource resource;
+            resource.load(ar);
+            setResource(resource, "IPSSnortSigs", "protections");
+        }
+        {
+            stringstream ss;
+            ss << "{";
+            ss << "\"context\": \"\",";
+            ss << "\"assetName\": \"asset1\",";
+            ss << "\"assetId\": \"1-1-1\",";
+            ss << "\"practiceId\": \"2-2-2\",";
+            ss << "\"practiceName\": \"practice1\",";
+            ss << "\"files\": [ \"rules1\" ],";
+            ss << "\"mode\": \"Prevent\"";
+            ss << "}";
+            cereal::JSONInputArchive ar(ss);
+            SnortSignatures signatures;
+            signatures.load(ar);
+            setConfiguration(signatures, "IPSSnortSigs", "SnortProtections");
+        }
+    }
+
+    ParsedContextReply
+    repondToContext(const string &buf_str, const string &name)
+    {
+        Buffer buf(buf_str);
+        ScopedContext ctx;
+        ctx.registerValue(name, buf);
+        return entry.respond(ParsedContext(buf, name, 0));
+    }
+
+    IPSEntry entry;
+    TableOpaqueBase *ptr = &entry;
+
+private:
+    NiceMock<MockMainLoop> mock_mainloop;
+    NiceMock<MockTimeGet> time;
+    ::Environment env;
+    GenericRulebase generic_rulebase;
+    ConfigComponent conf;
+    KeywordComp keywords;
+    AgentDetails details;
+    NiceMock<MockLogging> logs;
+    NiceMock<MockTable> table;
+    MockAgg mock_agg;
+};
+
+TEST_F(EntryTest, basic_inherited_functions)
+{
+    EXPECT_EQ(IPSEntry::name(), "IPS");
+    EXPECT_EQ(IPSEntry::currVer(), 0);
+    EXPECT_EQ(IPSEntry::minVer(), 0);
+    EXPECT_NE(IPSEntry::prototype(), nullptr);
+    EXPECT_EQ(entry.getListenerName(), IPSEntry::name());
+
+    stringstream ss;
+    {
+        cereal::JSONOutputArchive ar(ss);
+        entry.serialize(ar, 0);
+    }
+    EXPECT_EQ(ss.str(), "");
+
+    // Just make sure it doesn't crush
+    entry.upon(ParsedContext(Buffer(), "Nothing", 0));
+}
+
+TEST_F(EntryTest, check_listenning)
+{
+    EXPECT_TRUE(Listener<ParsedContext>::empty());
+    ptr->uponEnteringContext();
+    EXPECT_FALSE(Listener<ParsedContext>::empty());
+    ptr->uponLeavingContext();
+    EXPECT_TRUE(Listener<ParsedContext>::empty());
+}
+
+TEST_F(EntryTest, check_signature_invoking)
+{
+    EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::ACCEPT);
+    EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT);
+
+    string signature =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"ddd\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+    loadSignatures(signature);
+
+    EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::DROP);
+    EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT);
+}
+
+TEST_F(EntryTest, check_snort_signature_invoking)
+{
+    EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::ACCEPT);
+    EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT);
+
+    string signature =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"ddd\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "},"
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Bad sig\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: jjjj;\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+    loadSnortSignatures(signature);
+
+    EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::DROP);
+    EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT);
+}
+
+TEST_F(EntryTest, flags_test)
+{
+    EXPECT_FALSE(entry.isFlagSet("CONTEXT_A"));
+    EXPECT_FALSE(entry.isFlagSet("CONTEXT_B"));
+    entry.setFlag("CONTEXT_A");
+    EXPECT_TRUE(entry.isFlagSet("CONTEXT_A"));
+    EXPECT_FALSE(entry.isFlagSet("CONTEXT_B"));
+    entry.unsetFlag("CONTEXT_A");
+    EXPECT_FALSE(entry.isFlagSet("CONTEXT_A"));
+    EXPECT_FALSE(entry.isFlagSet("CONTEXT_B"));
+}
+
+TEST_F(EntryTest, get_buffer_test)
+{
+    repondToContext("ddd", "HTTP_REQUEST_BODY");
+    EXPECT_EQ(entry.getBuffer("HTTP_REQUEST_BODY"), Buffer("ddd"));
+    EXPECT_EQ(entry.getBuffer("HTTP_REQUEST_HEADER"), Buffer());
+}
+
+TEST_F(EntryTest, get_and_set_transaction_data)
+{
+    EXPECT_FALSE(entry.getTransactionData(Buffer("transaction_key")).ok());
+    entry.setTransactionData(Buffer("transaction_key"), Buffer("transaction_value"));
+    ASSERT_TRUE(entry.getTransactionData(Buffer("transaction_key")).ok());
+    EXPECT_EQ(entry.getTransactionData(Buffer("transaction_key")).unpack(), Buffer("transaction_value"));
+}

components/security_apps/ips/ips_ut/resource_ut.cc

@@ -0,0 +1,77 @@
+#include "ips_signatures.h"
+#include "cptest.h"
+#include "environment.h"
+#include "config_component.h"
+
+using namespace std;
+using namespace testing;
+
+static const string basic_resource =
+    "{"
+        "\"IPS\": {"
+            "\"VersionId\": \"1234567\","
+            "\"protections\": ["
+                "{"
+                    "\"protectionMetadata\": {"
+                        "\"protectionName\": \"Null HTTP Encodings\","
+                        "\"severity\": \"Medium\","
+                        "\"confidenceLevel\": \"High\","
+                        "\"performanceImpact\": \"Medium\","
+                        "\"lastUpdate\": \"20130101\","
+                        "\"maintrainId\": \"8576967832\","
+                        "\"tags\": [],"
+                        "\"cveList\": [],"
+                        "\"silent\": false"
+                    "},"
+                    "\"detectionRules\": {"
+                        "\"type\": \"simple\","
+                        "\"SSM\": \"aaaa\","
+                        "\"keywords\": \"\","
+                        "\"context\": ["
+                                "\"HTTP_COMPLETE_URL_ENCODED\""
+                        "]"
+                    "}"
+                "},"
+                "{"
+                    "\"protectionMetadata\": {"
+                        "\"protectionName\": \"Null HTTP Encodings\","
+                        "\"severity\": \"Medium\","
+                        "\"confidenceLevel\": \"High\","
+                        "\"performanceImpact\": \"Medium\","
+                        "\"lastUpdate\": \"20130101\","
+                        "\"maintrainId\": \"8576967832\","
+                        "\"tags\": [],"
+                        "\"cveList\": [],"
+                        "\"silent\": false"
+                    "},"
+                    "\"detectionRules\": {"
+                        "\"type\": \"simple\","
+                        "\"SSM\": \"bbbbb\","
+                        "\"keywords\": \"\","
+                        "\"context\": ["
+                                "\"HTTP_COMPLETE_URL_ENCODED\""
+                        "]"
+                    "}"
+                "}"
+            "]"
+        "}"
+    "}";
+
+TEST(resources, basic_resource)
+{
+    ConfigComponent conf;
+    ::Environment env;
+
+    conf.preload();
+
+    registerExpectedSetting<IPSSignaturesResource>("IPS", "protections");
+    registerExpectedSetting<string>("IPS", "VersionId");
+    stringstream resource;
+    resource << basic_resource;
+    Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(resource);
+
+    auto loaded_resources = getSettingWithDefault(IPSSignaturesResource(), "IPS", "protections");
+    EXPECT_EQ(loaded_resources.getSignatures().size(), 2);
+    auto version = getSettingWithDefault<string>("", "IPS", "VersionId");
+    EXPECT_EQ(version, "1234567");
+}

components/security_apps/ips/ips_ut/rule_selector_ut.cc

@@ -0,0 +1,163 @@
+#include <sstream>
+#include "ips_enums.h"
+#include "ips_basic_policy.h"
+#include "cptest.h"
+#include "config.h"
+
+using namespace testing;
+using namespace std;
+
+ostream &
+operator<<(ostream &os, const RuleSelector &selector)
+{
+    selector.print(os);
+    return os;
+}
+
+class RuleSelectorTest : public Test
+{
+public:
+    void
+    load(const string &config)
+    {
+        stringstream ss;
+        ss << config;
+        cereal::JSONInputArchive ar(ss);
+
+        ruleSelector.load(ar);
+    }
+
+    string protection =
+        "{"
+            "\"defaultAction\": \"Prevent\","
+            "\"rules\": ["
+            "{"
+                "\"action\": \"Detect\","
+                "\"performanceImpact\": \"Medium or lower\","
+                "\"severityLevel\": \"Low or above\","
+                "\"confidenceLevel\": \"Medium\","
+                "\"serverProtections\": false,"
+                "\"clientProtections\": true,"
+                "\"protectionsFromYear\": 2020,"
+                "\"protectionTags\": ["
+                    "\"tag1\","
+                    "\"tag2\""
+                "],"
+                "\"protectionIds\": ["
+                    "\"id1\","
+                    "\"id2\""
+                "]"
+            "},"
+            "{"
+                "\"action\": \"Prevent\","
+                "\"performanceImpact\": \"Very low\","
+                "\"severityLevel\": \"Medium or above\","
+                "\"confidenceLevel\": \"Low\","
+                "\"serverProtections\": true,"
+                "\"clientProtections\": false,"
+                "\"protectionsFromYear\": 1999,"
+                "\"protectionTags\": ["
+                    "\"tag11\","
+                    "\"tag22\""
+                "],"
+                "\"protectionIds\": ["
+                    "\"id11\","
+                    "\"id22\""
+                "]"
+            "}"
+            "]"
+        "}";
+
+    string protection2 =
+        "{"
+            "\"defaultAction\": \"Inactive\","
+            "\"rules\": ["
+            "{"
+                "\"action\": \"Detect\","
+                "\"performanceImpact\": \"Medium or lower\","
+                "\"severityLevel\": \"Low or above\","
+                "\"confidenceLevel\": \"Medium\""
+            "},"
+            "{"
+                "\"action\": \"Prevent\""
+            "}"
+            "]"
+        "}";
+
+    string protection3 =
+        "{"
+            "\"defaultAction\": \"Prevent\","
+            "\"rules\": []"
+        "}";
+
+    string protection4 =
+        "{"
+            "\"rules\": ["
+            "{"
+                "\"action\": \"Detect\","
+                "\"performanceImpact\": \"Medium or lower\","
+                "\"severityLevel\": \"Low or above\","
+                "\"confidenceLevel\": \"Medium\""
+            "},"
+            "{"
+                "\"action\": \"Prevent\""
+            "}"
+            "]"
+        "}";
+
+    RuleSelector ruleSelector;
+};
+
+TEST_F(RuleSelectorTest, read_rules)
+{
+    load(protection);
+    ostringstream stream;
+    stream << ruleSelector;
+    string str =  stream.str();
+    string result =
+        "[Rule] action: 1 performanceImpact: 3 severityLevel: 1 confidenceLevel: 3 serverProtections: false"
+        " clientProtections: true protectionsFromYear: 2020 protectionIds: id1, id2 protectionTags: tag1, tag2;"
+        "[Rule] action: 0 performanceImpact: 0 severityLevel: 3 confidenceLevel: 1 serverProtections: true"
+        " clientProtections: false protectionsFromYear: 1999 protectionIds: id11, id22 protectionTags: tag11, tag22;"
+        "[Rule] action: 0";
+
+    EXPECT_EQ(result, str);
+}
+
+TEST_F(RuleSelectorTest, read_semi_rules)
+{
+    load(protection2);
+    ostringstream stream;
+    stream << ruleSelector;
+    string str =  stream.str();
+    string result =
+        "[Rule] action: 1 performanceImpact: 3 severityLevel: 1 confidenceLevel: 3;"
+        "[Rule] action: 0;"
+        "[Rule] action: 2";
+
+    EXPECT_EQ(result, str);
+}
+
+TEST_F(RuleSelectorTest, read_empty_rules)
+{
+    try
+    {
+        load(protection3);
+    }
+    catch(const Config::ConfigException &e)
+    {
+        EXPECT_EQ("rules array is empty", e.getError());
+    }
+}
+
+TEST_F(RuleSelectorTest, read_no_default_action)
+{
+    try
+    {
+        load(protection4);
+    }
+    catch(const cereal::Exception &e)
+    {
+        EXPECT_EQ("JSON Parsing failed - provided NVP (defaultAction) not found", string(e.what()));
+    }
+}

components/security_apps/ips/ips_ut/signatures_ut.cc

@@ -0,0 +1,661 @@
+#include "ips_signatures.h"
+#include "ips_common_types.h"
+
+#include <sstream>
+#include <memory>
+#include "cptest.h"
+
+#include "keyword_comp.h"
+#include "environment.h"
+#include "agent_details.h"
+#include "mock/mock_logging.h"
+#include "time_proxy.h"
+#include "config.h"
+#include "config_component.h"
+#include "i_keywords_rule.h"
+#include "mock/mock_mainloop.h"
+#include "generic_rulebase/generic_rulebase.h"
+#include "generic_rulebase/parameters_config.h"
+#include "generic_rulebase/generic_rulebase_context.h"
+#include "encryptor.h"
+#include "mock/mock_table.h"
+
+using namespace testing;
+using namespace std;
+
+MATCHER_P(IsLog, IteratableFields, "")
+{
+    stringstream ss;
+    {
+        cereal::JSONOutputArchive ar(ss);
+        ar(arg);
+    }
+    for (const auto &field : IteratableFields) {
+        if (ss.str().find(field) == string::npos) return false;
+    }
+
+    return true;
+}
+
+class MockAgg : Singleton::Provide<I_FirstTierAgg>::SelfInterface
+{
+    shared_ptr<PMHook>
+    getHook(const string &, const set<PMPattern> &pats) override
+    {
+        auto hook = make_shared<PMHook>();
+        hook->prepare(pats);
+        return hook;
+    }
+};
+
+class SignatureTest : public Test
+{
+public:
+    SignatureTest()
+    {
+        generic_rulebase.preload();
+        EXPECT_CALL(logs, getCurrentLogId()).Times(AnyNumber());
+        ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state));
+        {
+            stringstream ss;
+            ss << "[" << signature1 << "]";
+            cereal::JSONInputArchive ar(ss);
+            single_signature.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "[" << signature3 << "]";
+            cereal::JSONInputArchive ar(ss);
+            single_signature2.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "[" << signature1 << ", " << signature2 << ", " << signature3 << "]";
+            cereal::JSONInputArchive ar(ss);
+            multiple_signatures.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "[" << signature_performance_very_low << ", " << signature_performance_low << "]";
+            cereal::JSONInputArchive ar(ss);
+            performance_signatures1.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "["  << signature_performance_medium_low << ", " << signature_performance_medium << "]";
+            cereal::JSONInputArchive ar(ss);
+            performance_signatures2.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "[" << signature_performance_medium_high << ", " << signature_performance_high << "]";
+            cereal::JSONInputArchive ar(ss);
+            performance_signatures3.load(ar);
+        }
+        {
+            stringstream ss;
+            ss << "[" << signature_high_confidance << ", " << signature_medium_confidance << "]";
+            cereal::JSONInputArchive ar(ss);
+            high_medium_confidance_signatures.load(ar);
+        }
+    }
+
+    ~SignatureTest()
+    {
+        if (gen_ctx != nullptr) {
+            gen_ctx->deactivate();
+            gen_ctx.reset();
+        }
+    }
+
+    void
+    loadExceptions()
+    {
+        env.preload();
+        env.init();
+
+        BasicRuleConfig::preload();
+        registerExpectedConfiguration<ParameterException>("rulebase", "exception");
+
+        string test_config(
+            "{"
+            "    \"rulebase\": {"
+            "        \"rulesConfig\": ["
+            "            {"
+            "                \"context\": \"All()\","
+            "                \"priority\": 1,"
+            "                \"ruleId\": \"5eaef0726765c30010bae8bb\","
+            "                \"ruleName\": \"Acme web API\","
+            "                \"assetId\": \"5e243effd858007660b758ad\","
+            "                \"assetName\": \"Acme Power API\","
+            "                \"parameters\": ["
+            "                    {"
+            "                       \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\","
+            "                       \"parameterType\": \"exceptions\","
+            "                       \"parameterName\": \"exception\""
+            "                    }"
+            "                ],"
+            "                \"zoneId\": \"\","
+            "                \"zoneName\": \"\""
+            "            }"
+            "        ],"
+            "        \"exception\": ["
+            "            {"
+            "                \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764003)\","
+            "                \"match\": {"
+            "                   \"type\": \"operator\","
+            "                   \"op\": \"or\","
+            "                   \"items\": [{"
+            "                       \"type\": \"condition\","
+            "                       \"op\": \"equals\","
+            "                       \"key\": \"protectionName\","
+            "                       \"value\": [\"Test1\"]"
+            "                    }, {"
+            "                       \"type\": \"condition\","
+            "                       \"op\": \"equals\","
+            "                       \"key\": \"protectionName\","
+            "                       \"value\": [\"Test2\"]"
+            "                    }, {"
+            "                       \"type\": \"condition\","
+            "                       \"op\": \"equals\","
+            "                       \"key\": \"sourceIdentifier\","
+            "                       \"value\": [\"1.1.1.1\"]"
+            "                    }]"
+            "                 },"
+            "                \"behavior\": {"
+            "                    \"key\": \"action\","
+            "                    \"value\": \"accept\""
+            "                }"
+            "            }"
+            "        ]"
+            "    }"
+            "}"
+        );
+
+        istringstream ss(test_config);
+        auto i_config = Singleton::Consume<Config::I_Config>::from(config);
+        i_config->loadConfiguration(ss);
+
+        gen_ctx = make_unique<GenericRulebaseContext>();
+        gen_ctx->activate();
+    }
+
+    void
+    load(const IPSSignaturesResource &policy, const string &severity, const string &confidence)
+    {
+        setResource(policy, "IPS", "protections");
+        stringstream ss;
+        ss << "{";
+        ss << "\"ruleName\": \"rule1\", \"assetName\": \"asset1\", \"practiceName\": \"practice1\",";
+        ss << "\"assetId\": \"1-1-1\", \"practiceId\": \"2-2-2\",";
+        ss << "\"defaultAction\" : " << "\"Detect\",";
+        ss << "\"rules\": [";
+        ss << "{";
+        ss << "\"action\": \"Prevent\",";
+        ss << "\"performanceImpact\": \"High or lower\",";
+        ss << "\"severityLevel\": \"" << severity << "\",";
+        ss << "\"confidenceLevel\": \"" << confidence << "\"";
+        ss << "}";
+        ss << "]";
+        ss << "}";
+        cereal::JSONInputArchive ar(ss);
+        sigs.load(ar);
+    }
+
+    bool
+    checkData(const string &data, const string &ctx_name = "HTTP_REQUEST_BODY")
+    {
+        ParsedContext body(data, ctx_name, 0);
+        ScopedContext ctx;
+        ctx.registerValue<string>(I_KeywordsRule::getKeywordsRuleTag(), ctx_name);
+        ctx.registerValue(body.getName(), body.getBuffer());
+        return sigs.isMatchedPrevent(body.getName(), body.getBuffer());
+    }
+
+    template <typename ...Strings>
+    void
+    expectLog(const string &field, Strings ...more_fields)
+    {
+        vector<string> all_fields;
+        all_fields.push_back(field);
+        expectLog(all_fields, more_fields...);
+    }
+
+    template <typename ...Strings>
+    void
+    expectLog(vector<string> all_fields, const string &field, Strings ...more_fields)
+    {
+        all_fields.push_back(field);
+        expectLog(all_fields, more_fields...);
+    }
+
+
+    void
+    expectLog(vector<string> all_fields)
+    {
+        EXPECT_CALL(logs, sendLog(IsLog(all_fields)));
+    }
+
+    IPSSignatures sigs;
+    IPSSignaturesResource single_signature;
+    IPSSignaturesResource single_signature2;
+    IPSSignaturesResource multiple_signatures;
+    IPSSignaturesResource high_medium_confidance_signatures;
+    IPSSignaturesResource performance_signatures1;
+    IPSSignaturesResource performance_signatures2;
+    IPSSignaturesResource performance_signatures3;
+    NiceMock<MockTable> table;
+    MockAgg mock_agg;
+
+private:
+    GenericRulebase generic_rulebase;
+    unique_ptr<GenericRulebaseContext> gen_ctx;
+    NiceMock<MockMainLoop> mock_mainloop;
+    KeywordComp keywords;
+    TimeProxyComponent time;
+    ::Environment env;
+    ConfigComponent config;
+    Encryptor encryptor;
+    AgentDetails details;
+    StrictMock<MockLogging> logs;
+    IPSEntry ips_state;
+
+    string signature1 =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [\"Protection_Type_Scanning_Tool\"],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"fff\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+    string signature2 =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test2\","
+                "\"maintrainId\": \"102\","
+                "\"severity\": \"Low\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [\"Vul_Type_SQL_Injection\"],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"ddd\","
+                "\"keywords\": \"data: \\\"ddd\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+    string signature3 =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test3\","
+                "\"maintrainId\": \"102\","
+                "\"severity\": \"High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [\"Protection_Type_Scanning_Tool\", \"Vul_Type_SQL_Injection\"],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"ggg\","
+                "\"keywords\": \"\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+
+    string signature_high_confidance =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test3\","
+                "\"maintrainId\": \"103\","
+                "\"severity\": \"Low\","
+                "\"confidenceLevel\": \"High\","
+                "\"performanceImpact\": \"Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"hhh\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+
+    string signature_medium_confidance =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test4\","
+                "\"maintrainId\": \"104\","
+                "\"severity\": \"Low\","
+                "\"confidenceLevel\": \"Medium\","
+                "\"performanceImpact\": \"Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+        "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"mmm\","
+                "\"keywords\": \"data: \\\"mmm\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_very_low =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Very Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"aaa\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_low =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"bbb\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_medium_low =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium Low\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"ccc\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_medium =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"ddd\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_medium_high =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"Medium High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"eee\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+
+    string signature_performance_high =
+        "{"
+            "\"protectionMetadata\": {"
+                "\"protectionName\": \"Test1\","
+                "\"maintrainId\": \"101\","
+                "\"severity\": \"Medium High\","
+                "\"confidenceLevel\": \"Low\","
+                "\"performanceImpact\": \"High\","
+                "\"lastUpdate\": \"20210420\","
+                "\"tags\": [],"
+                "\"cveList\": []"
+            "},"
+            "\"detectionRules\": {"
+                "\"type\": \"simple\","
+                "\"SSM\": \"\","
+                "\"keywords\": \"data: \\\"fff\\\";\","
+                "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]"
+            "}"
+        "}";
+};
+
+TEST_F(SignatureTest, basic_load_of_signatures)
+{
+    EXPECT_TRUE(sigs.isEmpty());
+    load(single_signature, "Low or above", "Low");
+    EXPECT_FALSE(sigs.isEmpty());
+    EXPECT_TRUE(sigs.isEmpty("NO_CONTEXT"));
+    EXPECT_FALSE(sigs.isEmpty("HTTP_REQUEST_BODY"));
+}
+
+TEST_F(SignatureTest, single_signature_matching_override)
+{
+    load(single_signature, "Low or above", "Low");
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\"");
+
+    EXPECT_TRUE(checkData("fffddd"));
+
+    loadExceptions();
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"Info\"");
+
+    EXPECT_FALSE(checkData("fffddd"));
+}
+
+TEST_F(SignatureTest, source_idetifier_exception)
+{
+    load(single_signature2, "Low or above", "Low");
+
+    loadExceptions();
+
+    expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\"");
+
+    EXPECT_TRUE(checkData("gggddd"));
+
+    ScopedContext ctx;
+    ctx.registerValue<string>("sourceIdentifiers", "1.1.1.1");
+
+    expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Info\"");
+
+    EXPECT_FALSE(checkData("gggddd"));
+}
+
+TEST_F(SignatureTest, single_signature_matching)
+{
+    load(single_signature, "Low or above", "Low");
+
+    EXPECT_FALSE(checkData("ggg"));
+
+    EXPECT_FALSE(checkData("ddd"));
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\"");
+
+    EXPECT_TRUE(checkData("fffddd"));
+}
+
+TEST_F(SignatureTest, context_signature_matching)
+{
+    load(single_signature, "Low or above", "Low");
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("fff", "HTTP_REQUEST_BODY"));
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("fff", "HTTP_RESPONSE_BODY"));
+
+    EXPECT_FALSE(checkData("fff", "HTTP_COMPLETE_URL_DECODED"));
+}
+
+TEST_F(SignatureTest, id_to_log_test)
+{
+    load(single_signature, "Low or above", "Low");
+
+    expectLog("\"protectionId\": \"Test1\"");
+
+    EXPECT_TRUE(checkData("fffddd"));
+}
+
+TEST_F(SignatureTest, multiple_signatures_matching)
+{
+    load(multiple_signatures, "Low or above", "Low");
+    EXPECT_FALSE(checkData("hhh"));
+
+    expectLog("\"protectionId\": \"Test2\"", "\"eventSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("ddd"));
+
+    expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("fff"));
+
+    expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\"");
+    EXPECT_TRUE(checkData("ggg"));
+
+    // Only one signature is caught
+    expectLog("\"protectionId\": \"Test2\"", "\"eventSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("fffdddggg"));
+}
+
+TEST_F(SignatureTest, severity_to_log_test)
+{
+    load(multiple_signatures, "Low or above", "Low");
+
+    expectLog("\"matchedSignatureSeverity\": \"Medium High\"");
+    EXPECT_TRUE(checkData("fff"));
+
+    expectLog("\"matchedSignatureSeverity\": \"Low\"");
+    EXPECT_TRUE(checkData("ddd"));
+
+    expectLog("\"matchedSignatureSeverity\": \"High\"");
+    EXPECT_TRUE(checkData("ggg"));
+}
+
+TEST_F(SignatureTest, incident_type)
+{
+    load(multiple_signatures, "Low or above", "Low");
+
+    expectLog("\"waapIncidentType\": \"Scanning Tool\"");
+    EXPECT_TRUE(checkData("fff"));
+
+    expectLog("\"waapIncidentType\": \"SQL Injection\"");
+    EXPECT_TRUE(checkData("ddd"));
+
+    expectLog("\"waapIncidentType\": \"SQL Injection\"");
+    EXPECT_TRUE(checkData("ggg"));
+}
+
+TEST_F(SignatureTest, performance_to_log_very_low)
+{
+    load(performance_signatures1, "Low or above", "Low");
+
+    EXPECT_FALSE(checkData("ggg"));
+
+    expectLog("\"matchedSignaturePerformance\": \"Very Low\"");
+
+    EXPECT_TRUE(checkData("aaa"));
+
+    expectLog("\"matchedSignaturePerformance\": \"Low\"");
+
+    EXPECT_TRUE(checkData("bbb"));
+}
+
+TEST_F(SignatureTest, performance_to_log_medium_low)
+{
+    load(performance_signatures2, "Low or above", "Low");
+
+    EXPECT_FALSE(checkData("ggg"));
+
+    expectLog("\"matchedSignaturePerformance\": \"Medium Low\"");
+
+    EXPECT_TRUE(checkData("ccc"));
+
+    expectLog("\"matchedSignaturePerformance\": \"Medium\"");
+
+    EXPECT_TRUE(checkData("ddd"));
+}
+
+TEST_F(SignatureTest, performance_to_log_medium_high)
+{
+    load(performance_signatures3, "Low or above", "Low");
+
+    EXPECT_FALSE(checkData("ggg"));
+
+    expectLog("\"matchedSignaturePerformance\": \"Medium High\"");
+
+    EXPECT_TRUE(checkData("eee"));
+
+    expectLog("\"matchedSignaturePerformance\": \"High\"");
+
+    EXPECT_TRUE(checkData("fff"));
+}
+
+TEST_F(SignatureTest, high_confidance_signatures_matching)
+{
+    load(high_medium_confidance_signatures, "Low or above", "High");
+    EXPECT_FALSE(checkData("ggg"));
+
+    expectLog("\"protectionId\": \"Test3\"", "\"matchedSignatureConfidence\": \"High\"");
+    EXPECT_TRUE(checkData("hhh"));
+
+    expectLog("\"protectionId\": \"Test4\"", "\"matchedSignatureConfidence\": \"Medium\"");
+    EXPECT_FALSE(checkData("mmm"));
+}

components/security_apps/ips/simple_protection.cc

@@ -0,0 +1,64 @@
+#include "simple_protection.h"
+#include "ips_comp.h"
+#include "debug.h"
+#include "helper.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_IPS);
+
+SimpleProtection::Impl::Impl(
+        const string &_sig_name,
+        const string &ssm,
+        const string &keyword,
+        const vector<string> &_context
+    )
+        :
+    sig_name(_sig_name),
+    context(_context)
+{
+    string deobfuscated_keyword = IPSHelper::deobfuscateKeyword(keyword);
+    if (deobfuscated_keyword != "") {
+        auto compiled = Singleton::Consume<I_KeywordsRule>::by<IPSComp>()->genRule(deobfuscated_keyword);
+        if (!compiled.ok()) {
+            reportConfigurationError(
+                "Failed to complie keywords '" + keyword + "' in signature " + sig_name + ": " + compiled.getErr()
+            );
+        }
+        rule = compiled.unpackMove();
+    }
+
+    auto deobfuscated_ssm = IPSHelper::deobfuscateString(ssm);
+    if (deobfuscated_ssm != "") {
+        auto temp_pattern = PMHook::lineToPattern(deobfuscated_ssm);
+        if (!temp_pattern.ok()) reportConfigurationError("Failed first tier pattern: " + temp_pattern.getErr());
+        pattern = temp_pattern.unpackMove();
+    }
+
+    if (deobfuscated_keyword == "" && deobfuscated_ssm == "") {
+        reportConfigurationError("Both Simple String and Keyword are empty in a simple protection " + sig_name);
+    }
+}
+
+using MatchType = IPSSignatureSubTypes::BaseSignature::MatchType;
+
+MatchType
+SimpleProtection::Impl::getMatch(const set<PMPattern> &matches) const
+{
+    dbgTrace(D_IPS) << "Entering signature";
+    if (!pattern.empty() && matches.find(pattern) == matches.end()) return MatchType::NO_MATCH;
+
+    dbgTrace(D_IPS) << "Checking for rule";
+    if (!rule) return MatchType::MATCH;
+
+    dbgTrace(D_IPS) << "Running keywords";
+    return rule->isMatch() ? MatchType::MATCH : MatchType::NO_MATCH;
+}
+
+set<PMPattern>
+SimpleProtection::Impl::patternsInSignature() const
+{
+    set<PMPattern> res;
+    if (!pattern.empty()) res.insert(pattern);
+    return res;
+}

components/security_apps/ips/snort_basic_policy.cc

@@ -0,0 +1,45 @@
+#include "snort_basic_policy.h"
+#include <cereal/types/string.hpp>
+#include <cereal/types/vector.hpp>
+#include <ostream>
+#include <stdexcept>
+#include <iomanip>
+#include "ips_signatures.h"
+#include "helper.h"
+#include "config.h"
+#include "common.h"
+
+USE_DEBUG_FLAG(D_IPS);
+
+using namespace std;
+
+void
+SnortRuleSelector::load(cereal::JSONInputArchive &ar)
+{
+    string mode;
+    ar(cereal::make_nvp("mode", mode), cereal::make_nvp("files", file_names));
+
+    if (mode == "Inactive") action = IPSSignatureSubTypes::SignatureAction::IGNORE;
+    else if (mode == "Disabled") action = IPSSignatureSubTypes::SignatureAction::IGNORE;
+    else if (mode == "Detect") action = IPSSignatureSubTypes::SignatureAction::DETECT;
+    else if (mode == "Prevent") action = IPSSignatureSubTypes::SignatureAction::PREVENT;
+    else reportConfigurationError("invalid action value " + mode);
+}
+
+vector<IPSSignatureSubTypes::SignatureAndAction>
+SnortRuleSelector::selectSignatures() const
+{
+    vector<IPSSignatureSubTypes::SignatureAndAction> res;
+
+    if (action == IPSSignatureSubTypes::SignatureAction::IGNORE) return res;
+
+    auto signatures = getResource<SnortSignaturesResource>("IPSSnortSigs", "protections");
+    if (!signatures.ok()) return res;
+
+    for (auto &file : file_names) {
+        for (auto &signature : (*signatures).getSignatures(file)) {
+            res.emplace_back(signature, action);
+        }
+    }
+    return res;
+}

components/security_apps/orchestration/downloader/downloader.cc

@@ -21,6 +21,11 @@
 #include "rest.h"
 #include "cereal/external/rapidjson/document.h"
 
+#include "customized_cereal_map.h"
+#include "cereal/archives/json.hpp"
+#include "cereal/types/vector.hpp"
+#include "cereal/types/string.hpp"
+
 #include <fstream>
 
 using namespace std;
@@ -28,6 +33,31 @@ using namespace rapidjson;
 
 USE_DEBUG_FLAG(D_ORCHESTRATOR);
 
+// LCOV_EXCL_START Reason: WA for NSaaS upgrade
+class TenantProfileMap
+{
+public:
+    void
+    load(const string &raw_value)
+    {
+        vector<string> tenants_and_profiles;
+        {
+            stringstream string_stream(raw_value);
+            cereal::JSONInputArchive archive(string_stream);
+            cereal::load(archive, tenants_and_profiles);
+        }
+        for (const auto &tenant_profile_pair : tenants_and_profiles) {
+            value.push_back(tenant_profile_pair);
+        }
+    }
+
+    const vector<string> & getValue() const { return value; }
+private:
+    vector<string> value;
+};
+
+// LCOV_EXCL_STOP
+
 class Downloader::Impl : Singleton::Provide<I_Downloader>::From<Downloader>
 {
 public:
@@ -51,6 +81,9 @@ public:
         const string &service_name
     ) const override;
 
+    void createTenantProfileMap();
+    string getProfileFromMap(const string &tenant_id) const override;
+
 private:
     Maybe<string> downloadFileFromFogByHTTP(
         const GetResourceFile &resourse_file,
@@ -74,6 +107,7 @@ private:
     tuple<string, string> splitQuery(const string &query) const;
     string vectorToPath(const vector<string> &vec) const;
     string dir_path;
+    map<string, string> tenant_profile_map;
 };
 
 void
@@ -111,6 +145,42 @@ Downloader::Impl::downloadFileFromFog(
     return file_path;
 }
 
+void
+Downloader::Impl::createTenantProfileMap()
+{
+    dbgFlow(D_ORCHESTRATOR) << "Creating a tenant-profile map from the agent settings";
+    tenant_profile_map.clear();
+    auto maybe_tenant_profile_map = getProfileAgentSetting<TenantProfileMap>("TenantProfileMap");
+    if (maybe_tenant_profile_map.ok()) {
+        dbgTrace(D_ORCHESTRATOR) << "Managed to read the TenantProfileMap agent settings";
+        TenantProfileMap tpm = maybe_tenant_profile_map.unpack();
+        for (const string &str : tpm.getValue()) {
+            string delimiter = ":";
+            string tenant = str.substr(0, str.find(delimiter));
+            string profile = str.substr(str.find(delimiter) + 1);
+            dbgTrace(D_ORCHESTRATOR)
+                << "Loading into the map. Tenant: "
+                << tenant
+                << " Profile: "
+                << profile;
+            tenant_profile_map[tenant] = profile;
+        }
+    } else {
+        dbgTrace(D_ORCHESTRATOR) << "Couldn't load the TenantProfileMap agent settings";
+    }
+}
+
+// LCOV_EXCL_START Reason: NSaaS old profiles support
+string
+Downloader::Impl::getProfileFromMap(const string &tenant_id) const
+{
+    if (tenant_profile_map.find(tenant_id) == tenant_profile_map.end()) {
+        return "";
+    }
+    return tenant_profile_map.at(tenant_id);
+}
+// LCOV_EXCL_STOP
+
 Maybe<map<pair<string, string>, string>>
 Downloader::Impl::downloadVirtualFileFromFog(
     const GetResourceFile &resourse_file,
@@ -130,8 +200,10 @@ Downloader::Impl::downloadVirtualFileFromFog(
 
     Document document;
     document.Parse(downloaded_data.unpack().c_str());
-    if (document.HasParseError()) return genError("JSON file is not valid.");
-
+    if (document.HasParseError()) {
+        dbgWarning(D_ORCHESTRATOR) << "JSON file is not valid";
+        return genError("JSON file is not valid.");
+    }
     const Value &tenants_data = document[tenants_key.c_str()];
     for (Value::ConstValueIterator itr = tenants_data.Begin(); itr != tenants_data.End(); ++itr) {
 
@@ -145,9 +217,21 @@ Downloader::Impl::downloadVirtualFileFromFog(
 
         if (artifact_data != itr->MemberEnd()) {
             auto profile_id_obj = itr->FindMember(profile_id_key.c_str());
-            if (profile_id_obj == itr->MemberEnd()) continue;
+            string profile_id;
+            if (profile_id_obj == itr->MemberEnd()) {
+                if (tenant_profile_map.count(tenant_id)) {
+                    dbgWarning(D_ORCHESTRATOR)
+                        << "Forcing profile ID to be "
+                        << getProfileFromMap(tenant_id);
+                    profile_id = getProfileFromMap(tenant_id);
+                } else {
+                    dbgWarning(D_ORCHESTRATOR) << "Couldn't force profile ID";
+                    continue;
+                }
+            }
 
-            string profile_id =  profile_id_obj->value.GetString();
+            if (profile_id.empty()) profile_id = profile_id_obj->value.GetString();
+            dbgTrace(D_ORCHESTRATOR) << "Found a profile ID " << profile_id;
 
             string file_path =
                 dir_path + "/" + resourse_file.getFileName() + "_" +
@@ -161,6 +245,9 @@ Downloader::Impl::downloadVirtualFileFromFog(
             if (orchestration_tools->writeFile(buffer.GetString(), file_path)) {
                 res.insert({{tenant_id, profile_id}, file_path});
             }
+
+            orchestration_tools->fillKeyInJson(file_path, "profileID", profile_id);
+            orchestration_tools->fillKeyInJson(file_path, "tenantID", tenant_id);
             continue;
         }
 
@@ -387,4 +474,5 @@ Downloader::preload()
     registerExpectedConfiguration<string>("orchestration", "Default file download path");
     registerExpectedConfiguration<string>("orchestration", "Self signed certificates acceptable");
     registerExpectedConfiguration<bool>("orchestration", "Add tenant suffix");
+    registerConfigLoadCb([this]() { pimpl->createTenantProfileMap(); });
 }

components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc

@@ -345,6 +345,8 @@ TEST_F(DownloaderTest, download_virtual_policy)
     EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_0000_file, "/tmp/virtualPolicy_0000_profile_1234.download"))
         .WillOnce(Return(true));
 
+    EXPECT_CALL(mock_orchestration_tools, fillKeyInJson(_, _, _)).WillRepeatedly(Return());
+
     EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_1111_file, "/tmp/virtualPolicy_1111_profile_1235.download"))
         .WillOnce(Return(true));
 
@@ -429,6 +431,8 @@ TEST_F(DownloaderTest, download_virtual_settings)
         )
     ).WillOnce(Return(true));
 
+    EXPECT_CALL(mock_orchestration_tools, fillKeyInJson(_, _, _)).WillRepeatedly(Return());
+
     stringstream file_path;
     file_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789"
                 "_profile_4c721b40-85df-4364-be3d-303a10ee9780.download";

components/security_apps/orchestration/include/mock/mock_downloader.h

@@ -40,6 +40,12 @@ public:
         downloadFileFromURL,
         Maybe<std::string>(const std::string &, const std::string &, Package::ChecksumTypes, const std::string &)
     );
+
+    MOCK_CONST_METHOD1(
+        getProfileFromMap,
+        std::string(const std::string &)
+    );
+
 };
 
 #endif // __MOCK_DOWNLOADER_H__

components/security_apps/orchestration/include/mock/mock_orchestration_tools.h

@@ -31,13 +31,6 @@ operator<<(std::ostream &os, const std::map<T, S> &)
     return os;
 }
 
-template <typename T, typename S>
-std::ostream &
-operator<<(std::ostream &os, const Maybe<std::map<T, S>> &)
-{
-    return os;
-}
-
 class MockOrchestrationTools
         :
     public Singleton::Provide<I_OrchestrationTools>::From<MockProvider<I_OrchestrationTools>>
@@ -56,6 +49,7 @@ public:
         Maybe<std::map<std::string, std::string>>(const std::string &, const std::string &, const std::string &)
     );
     MOCK_CONST_METHOD1(doesFileExist,        bool(const std::string &));
+    MOCK_CONST_METHOD3(fillKeyInJson,        void(const std::string &, const std::string &, const std::string &));
     MOCK_CONST_METHOD1(createDirectory,      bool(const std::string &));
     MOCK_CONST_METHOD1(doesDirectoryExist,   bool(const std::string &));
     MOCK_CONST_METHOD1(executeCmd,           bool(const std::string &));

components/security_apps/orchestration/orchestration_comp.cc

@@ -1113,17 +1113,26 @@ private:
         // Download virtual policy
         bool is_empty = true;
         GetResourceFile resource_v_policy_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY);
+        I_Downloader *downloader = Singleton::Consume<I_Downloader>::by<OrchestrationComp>();
         for (const auto &tenant: *updated_policy_tenants) {
             if (!tenant.getVersion().empty()) {
                 is_empty = false;
+
+                string profile_to_use = tenant.getProfileID().empty() ?
+                    downloader->getProfileFromMap(tenant.getTenantID()) :
+                    tenant.getProfileID();
+
                 dbgTrace(D_ORCHESTRATOR)
                     << "Adding a tenant to the multi-tenant list. Tenant: "
-                    << tenant.getTenantID();
+                    << tenant.getTenantID()
+                    << " Profile: "
+                    << profile_to_use;
                 auto tenant_manager = Singleton::Consume<I_TenantManager>::by<OrchestrationComp>();
-                tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), tenant.getProfileID());
+
+                tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), profile_to_use);
                 resource_v_policy_file.addTenant(
                     tenant.getTenantID(),
-                    tenant.getProfileID(),
+                    profile_to_use,
                     tenant.getVersion(),
                     tenant.getChecksum()
                 );
@@ -1132,7 +1141,7 @@ private:
 
         if (!is_empty) {
             auto new_virtual_policy_files =
-                Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadVirtualFileFromFog(
+                downloader->downloadVirtualFileFromFog(
                     resource_v_policy_file,
                     I_OrchestrationTools::SELECTED_CHECKSUM_TYPE
                 );
@@ -1151,9 +1160,24 @@ private:
             for (const auto &tenant: *updated_settings_tenants) {
                 if (!tenant.getVersion().empty()) {
                     is_empty = false;
+
+                    string profile_to_use = tenant.getProfileID().empty() ?
+                        downloader->getProfileFromMap(tenant.getTenantID()) :
+                        tenant.getProfileID();
+
+                    dbgTrace(D_ORCHESTRATOR)
+                        << "Handling virtual settings: Tenant ID: "
+                        << tenant.getTenantID()
+                        << ", Profile ID: "
+                        << profile_to_use
+                        << ", version: "
+                        << tenant.getVersion()
+                        << ", checksum: "
+                        << tenant.getChecksum();
+
                     resource_v_settings_file.addTenant(
                         tenant.getTenantID(),
-                        tenant.getProfileID(),
+                        profile_to_use,
                         tenant.getVersion(),
                         tenant.getChecksum()
                     );
@@ -1169,6 +1193,11 @@ private:
                 if (new_virtual_settings_files.ok()) {
                     for (const auto &tenant_file: *new_virtual_settings_files) {
                         auto tenant_profile = TenantProfilePair(tenant_file.first.first, tenant_file.first.second);
+                        dbgTrace(D_ORCHESTRATOR)
+                            << "Downloaded a file from the FOG: Tenant ID: "
+                            << tenant_profile.getTenantId()
+                            << ", Profile ID: "
+                            << tenant_profile.getProfileId();
                         sorted_files[tenant_profile].push_back(tenant_file.second);
                     }
                 }

components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc

@@ -45,6 +45,7 @@ public:
     bool removeFile(const string &path) const override;
     bool copyFile(const string &src_path, const string &dst_path) const override;
     bool doesFileExist(const string &file_path) const override;
+    void fillKeyInJson(const string &filename, const string &_key, const string &_val) const override;
     bool createDirectory(const string &directory_path) const override;
     bool doesDirectoryExist(const string &dir_path) const override;
     bool executeCmd(const string &cmd) const override;
@@ -78,6 +79,41 @@ checkExistence(const string &path, bool is_dir)
     }
 }
 
+// LCOV_EXCL_START Reason: NSaaS upgrade WA
+void
+OrchestrationTools::Impl::fillKeyInJson(const string &filename, const string &_key, const string &_val) const
+{
+    // Load the JSON file into a string
+    std::ifstream ifs(filename);
+    std::string jsonStr((std::istreambuf_iterator<char>(ifs)), (std::istreambuf_iterator<char>()));
+
+    dbgTrace(D_ORCHESTRATOR) << "Trying to parse " << filename;
+    // Parse the JSON string
+    Document doc;
+    doc.Parse(jsonStr.c_str());
+
+    // Check if the key exists
+    if (doc.HasMember(_key.c_str())) {
+        dbgTrace(D_ORCHESTRATOR) << _key << " already exists.";
+        return;
+    }
+
+    // Add the  key with value
+    Value key(_key.c_str(), doc.GetAllocator());
+    Value val(_val.c_str(), doc.GetAllocator());
+    doc.AddMember(key, val, doc.GetAllocator());
+
+    // Write the modified JSON to a new file
+    StringBuffer buffer;
+    Writer<StringBuffer> writer(buffer);
+    doc.Accept(writer);
+    std::ofstream ofs(filename);
+    ofs << buffer.GetString() << std::endl;
+
+    dbgTrace(D_ORCHESTRATOR) << _key << " added with val " << _val;
+}
+// LCOV_EXCL_STOP
+
 bool
 OrchestrationTools::Impl::doesFileExist(const string &file_path) const
 {

components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc

@@ -265,14 +265,14 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
     EXPECT_CALL(mock_service_controller, getPolicyVersion())
         .Times(2).WillRepeatedly(ReturnRef(first_policy_version));
 
-    vector<string> active_tenants = { "1236", "1235" };
+    set<string> active_tenants = { "1236", "1235" };
     EXPECT_CALL(tenant_manager, fetchActiveTenants()).WillOnce(Return(active_tenants));
 
     EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1235", "2311"));
     EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1236", "2611"));
 
-    vector<string> first_tenant_profiles = { "2611" };
-    vector<string> second_tenant_profiles = { "2311"};
+    set<string> first_tenant_profiles = { "2611" };
+    set<string> second_tenant_profiles = { "2311"};
     EXPECT_CALL(
         tenant_manager,
         fetchProfileIds("1236")).WillRepeatedly(Return(first_tenant_profiles)

components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc

@@ -1466,8 +1466,8 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
         make_pair("/etc/cp/conf/tenant2_profile_1235_policy.json", "")}
     };
 
-    vector<string> ids = {"family1_id2"};
-    vector<string> empty_ids;
+    set<string> ids = {"family1_id2"};
+    set<string> empty_ids;
 
     EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillOnce(Return(ids));
     EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillOnce(Return(empty_ids));
@@ -1592,7 +1592,7 @@ TEST_F(ServiceControllerTest, cleanup_virtual_files)
         "222222\n"
         "333333\n";
 
-    vector<string> active_tenants = {
+    set<string> active_tenants = {
         "222222"
     };
 

components/security_apps/waap/include/i_serialize.h

@@ -189,10 +189,10 @@ protected:
                     "object sent successfully after " << i << " retry attempts";
                 return true;
             }
-            dbgWarning(D_WAAP) << "Failed to send object. Attempt: " << i;
+            dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i;
             mainloop->yield(wait_next_attempt);
         }
-        dbgError(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
+        dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
             max_send_obj_retries;
         return false;
     }
@@ -243,10 +243,10 @@ protected:
                     "object sent successfully after " << i << " retry attempts";
                 return true;
             }
-            dbgWarning(D_WAAP) << "Failed to send object. Attempt: " << i;
+            dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i;
             mainloop->yield(wait_next_attempt);
         }
-        dbgError(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
+        dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " <<
             max_send_obj_retries;
         return false;
     }

components/security_apps/waap/reputation/reputation_features_agg.cc

@@ -348,6 +348,10 @@ ReputationFeaturesAgg::Impl::reportReputationFeatures()
     I_MainLoop *i_mainLoop = Singleton::Consume<I_MainLoop>::by<ReputationFeaturesAgg>();
 
     string tenantId = agentDetails->getTenantId();
+    if (tenantId.empty())
+    {
+        tenantId = "Elpis";
+    }
     string agentId = agentDetails->getAgentId();
     if (Singleton::exists<I_InstanceAwareness>())
     {

components/security_apps/waap/waap_clib/ConfidenceCalculator.cc

@@ -137,9 +137,13 @@ bool ConfidenceCalculator::postData()
     dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url;
 
     WindowLogPost currentWindow(m_time_window_logger_backup);
-    return sendNoReplyObjectWithRetry(currentWindow,
+    bool ok = sendNoReplyObjectWithRetry(currentWindow,
         I_Messaging::Method::PUT,
         url);
+    if (!ok) {
+        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
+    }
+    return ok;
 }
 
 void ConfidenceCalculator::pullData(const std::vector<std::string>& files)
@@ -149,7 +153,7 @@ void ConfidenceCalculator::pullData(const std::vector<std::string>& files)
         mergeProcessedFromRemote();
     }
     std::string url = getPostDataUrl();
-    std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
+    std::string sentFile = url.erase(0, strlen("/storage/waap/"));
     dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "pulling files, skipping: " << sentFile;
     for (auto file : files)
     {
@@ -159,10 +163,15 @@ void ConfidenceCalculator::pullData(const std::vector<std::string>& files)
         }
         dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Pulling the file: " << file;
         WindowLogGet getWindow;
-        sendObjectWithRetry(getWindow,
+        bool ok = sendObjectWithRetry(getWindow,
             I_Messaging::Method::GET,
             getUri() + "/" + file);
 
+        if (!ok) {
+            dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get file: " << file;
+            continue;
+        }
+
         KeyValSourcesLogger remoteLogger = getWindow.getWindowLogger().unpack();
         for (auto& log : remoteLogger)
         {
@@ -215,6 +224,10 @@ void ConfidenceCalculator::pullProcessedData(const std::vector<std::string>& fil
             m_confidence_level = getConfFile.getConfidenceLevels().unpackMove();
         }
     }
+    // is_first_pull = false -> at least one file was downloaded and merged
+    if (is_first_pull) {
+        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state";
+    }
 }
 
 void ConfidenceCalculator::postProcessedData()

components/security_apps/waap/waap_clib/ScannerDetector.cc

@@ -100,9 +100,13 @@ bool ScannerDetector::postData()
     dbgTrace(D_WAAP) << "Sending the data to: " << url;
 
     SourcesMonitorPost currentWindow(m_sources_monitor_backup);
-    return sendNoReplyObjectWithRetry(currentWindow,
+    bool ok = sendNoReplyObjectWithRetry(currentWindow,
         I_Messaging::Method::PUT,
         url);
+    if (!ok) {
+        dbgError(D_WAAP) << "Failed to post collected data to: " << url;
+    }
+    return ok;
 }
 
 void ScannerDetector::pullData(const std::vector<std::string>& files)
@@ -118,10 +122,15 @@ void ScannerDetector::pullData(const std::vector<std::string>& files)
         }
         dbgTrace(D_WAAP) << "Pulling the file: " << file;
         SourcesMonitorGet getMonitor;
-        sendObjectWithRetry(getMonitor,
+        bool ok = sendObjectWithRetry(getMonitor,
             I_Messaging::Method::GET,
             getUri() + "/" + file);
 
+        if (!ok) {
+            dbgError(D_WAAP) << "Failed to get data from: " << file;
+            continue;
+        }
+
         SourceKeyValsMap remoteMonitor = getMonitor.getSourcesMonitor().unpack();
         for (const auto& srcData : remoteMonitor)
         {

components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc

@@ -96,9 +96,13 @@ bool TrustedSourcesConfidenceCalculator::postData()
     dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url;
 
     TrsutedSourcesLogger logger(m_logger);
-    return sendNoReplyObjectWithRetry(logger,
+    bool ok = sendNoReplyObjectWithRetry(logger,
         I_Messaging::Method::PUT,
         url);
+    if (!ok) {
+        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
+    }
+    return ok;
 }
 
 void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>& files)
@@ -116,7 +120,12 @@ void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>
         bool res = sendObjectWithRetry(getTrustFile,
             I_Messaging::Method::GET,
             getUri() + "/" + file);
-        if (res && getTrustFile.getTrustedLogs().ok())
+        if (!res)
+        {
+            dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get file: " << file;
+            continue;
+        }
+        if (getTrustFile.getTrustedLogs().ok())
         {
             mergeFromRemote(getTrustFile.getTrustedLogs().unpack());
         }
@@ -134,20 +143,22 @@ void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::stri
     pullProcessedData(files);
 }
 
-void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string>& files)
-{
+void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string>& files) {
     dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources";
-    for (auto file : files)
-    {
+    bool pull_ok = false;
+    for (auto file: files) {
         GetTrustedFile getTrustFile;
         bool res = sendObjectWithRetry(getTrustFile,
             I_Messaging::Method::GET,
             getUri() + "/" + file);
-        if (res && getTrustFile.getTrustedLogs().ok())
-        {
+        if (res && getTrustFile.getTrustedLogs().ok()) {
             mergeFromRemote(getTrustFile.getTrustedLogs().unpack());
+            pull_ok = true;
         }
     }
+    if (!pull_ok && !files.empty()) {
+        dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to pull state data";
+    }
 }
 
 void TrustedSourcesConfidenceCalculator::postProcessedData()

components/security_apps/waap/waap_clib/WaapSampleValue.cc

@@ -37,5 +37,6 @@ SampleValue::getSampleString() const
 void
 SampleValue::findMatches(const Regex &pattern, std::vector<RegexMatch> &matches) const
 {
-    pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr);
+    static const size_t maxMatchesPerSignature = 5;
+    pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr, maxMatchesPerSignature);
 }

components/security_apps/waap/waap_clib/Waf2Regex.cc

@@ -140,7 +140,7 @@ bool SingleRegex::hasMatch(const std::string& s) const {
     return true;
 }
 
-size_t SingleRegex::findAllMatches(const std::string& s, std::vector<RegexMatch>& matches) const {
+size_t SingleRegex::findAllMatches(const std::string& s, std::vector<RegexMatch>& matches, size_t maxMatches) const {
     size_t matchesCount = 0;
 
     // Optimized regex that always immediately reports a "simulated" match without spending time to do a scan
@@ -234,7 +234,7 @@ size_t SingleRegex::findAllMatches(const std::string& s, std::vector<RegexMatch>
         // continue searching for next match starting from end of this match
         // (first two entries in ov[] are start and end offsets of current full match)
         startOffset = ov[1];
-    } while (true);
+    } while (matchesCount < maxMatches);
 
     return matchesCount;
 }
@@ -418,7 +418,7 @@ bool Regex::hasMatch(const std::string& s) const {
 }
 
 size_t Regex::findAllMatches(const std::string& s, std::vector<RegexMatch>& matches,
-    const Waap::RegexPreconditions::PmWordSet *pmWordSet) const {
+    const Waap::RegexPreconditions::PmWordSet *pmWordSet, size_t maxMatches) const {
     matches.clear();
 
     if (m_regexPreconditions && pmWordSet) {
@@ -442,7 +442,7 @@ size_t Regex::findAllMatches(const std::string& s, std::vector<RegexMatch>& matc
                 }
 
                 // Scan only regexes that are enabled by aho-corasick scan
-                m_sre[regexIndex]->findAllMatches(s, matches);
+                m_sre[regexIndex]->findAllMatches(s, matches, maxMatches);
                 dbgTrace(D_WAAP_REGEX) << "Regex['" << m_sre[regexIndex]->getName() <<
                     "',index=" << regexIndex << "]::findAllMatches(): " << matches.size() << " matches found (so far)";
 
@@ -453,7 +453,7 @@ size_t Regex::findAllMatches(const std::string& s, std::vector<RegexMatch>& matc
     else {
         // When optimization is disabled - scan all regexes
         for (SingleRegex* pSingleRegex : m_sre) {
-            pSingleRegex->findAllMatches(s, matches);
+            pSingleRegex->findAllMatches(s, matches, maxMatches);
             dbgTrace(D_WAAP_REGEX) << "Regex['" << m_regexName << "']['" << pSingleRegex->getName() <<
                 "']::findAllMatches(): " << matches.size() << " matches found (so far)";
         }

components/security_apps/waap/waap_clib/Waf2Regex.h

@@ -55,7 +55,8 @@ public:
         const std::string &regexMatchName="", const std::string &regexMatchValue="");
     ~SingleRegex();
     bool hasMatch(const std::string &s) const;
-    size_t findAllMatches(const std::string &s, std::vector<RegexMatch> &matches) const;
+    size_t findAllMatches(const std::string &s, std::vector<RegexMatch> &matches,
+            size_t max_matches = std::string::npos) const;
     size_t findMatchRanges(const std::string &s, std::vector<RegexMatchRange> &matchRanges) const;
     const std::string &getName() const;
 private:
@@ -76,8 +77,8 @@ public:
         std::shared_ptr<Waap::RegexPreconditions> regexPreconditions);
     ~Regex();
     bool hasMatch(const std::string &s) const;
-    size_t findAllMatches(const std::string &v, std::vector<RegexMatch> &maches,
-        const Waap::RegexPreconditions::PmWordSet *pmWordSet=nullptr) const;
+    size_t findAllMatches(const std::string &v, std::vector<RegexMatch> &matches,
+        const Waap::RegexPreconditions::PmWordSet *pmWordSet=nullptr, size_t max_matches = std::string::npos) const;
     std::string sub(const std::string &s, const std::string &repl="") const;
     // Run regex search, and for each found match - run callback.
     // The callback can cancel replacement of the match (leave source match "as-is"), provide a replacement string,

components/utils/CMakeLists.txt

@@ -1,3 +1,4 @@
 add_subdirectory(http_transaction_data)
 add_subdirectory(ip_utilities)
+add_subdirectory(keywords)
 add_subdirectory(pm)

components/utils/keywords/CMakeLists.txt

@@ -0,0 +1,2 @@
+add_library(keywords keywords_rule.cc single_keyword.cc data_keyword.cc pcre_keyword.cc length_keyword.cc byte_extract_keyword.cc compare_keyword.cc jump_keyword.cc stateop_keyword.cc no_match_keyword.cc)
+add_subdirectory(keywords_ut)

components/utils/keywords/byte_extract_keyword.cc

@@ -0,0 +1,319 @@
+#include "single_keyword.h"
+#include "output.h"
+#include "debug.h"
+
+#include <map>
+#include <strings.h>
+#include "limits.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class ByteExtractKeyword : public SingleKeyword
+{
+public:
+    explicit ByteExtractKeyword(const vector<KeywordAttr> &attr, VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState *prev) const override;
+
+private:
+    enum class BaseId
+    {
+        BIN,
+        HEX = 16,
+        DEC = 10,
+        OCT = 8
+    };
+
+    void
+    setOffset(const KeywordAttr &attr, const VariablesMapping &vars)
+    {
+        offset.setAttr(attr, vars, "byte_extract");
+    }
+
+    void
+    setRelative(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_relative.setAttr(attr, "byte_extract");
+    }
+
+    void
+    setLittleEndian(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_little_end.setAttr(attr, "byte_extract");
+    }
+
+    void
+    setDataType(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        if (data_type != BaseId::BIN) {
+            throw KeywordError("Double definition of the data type in the 'byte_extract' keyword");
+        }
+
+        auto &vec = attr.getParams();
+        if (vec.size() != 2) throw KeywordError("Malformed data type in the 'byte_extract' keyword");
+
+        if (vec[1] == "hex") {
+            data_type = BaseId::HEX;
+        } else if (vec[1] == "dec") {
+            data_type = BaseId::DEC;
+        } else if (vec[1] == "oct") {
+            data_type = BaseId::OCT;
+        } else {
+            throw KeywordError("Unknown data type in the 'byte_extract' keyword: " + vec[1]);
+        }
+    }
+
+    void
+    setContext(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        ctx.setAttr(attr, "byte_extract");
+    }
+
+    void
+    setAlign(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        if (align != 1) throw KeywordError("Double definition of the 'align' in the 'byte_extract' keyword");
+        auto &vec = attr.getParams();
+        if (vec.size() != 2) throw KeywordError("Malformed 'align' in the 'byte_extract' keyword");
+
+        if (vec[1] == "2") {
+            align = 2;
+        } else if (vec[1] == "4") {
+            align = 4;
+        } else {
+            throw KeywordError("Unknown 'align' in the 'byte_extract' keyword: " + vec[1]);
+        }
+    }
+
+    bool
+    isConstant() const
+    {
+        return !is_relative && bytes.isConstant() && offset.isConstant();
+    }
+
+    pair<uint, uint> getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const;
+    uint applyAlignment(uint value) const;
+    Maybe<uint> readValue(uint start, uint length, const Buffer &buf) const;
+    Maybe<uint> readStringValue(uint start, uint length, const Buffer &buf) const;
+
+    NumericAttr bytes;
+    uint        var_id;
+    NumericAttr offset;
+    BoolAttr    is_relative;
+    BoolAttr    is_little_end;
+    BaseId      data_type = BaseId::BIN;
+    int         align = 1;
+    CtxAttr     ctx;
+
+    static const map<string, void(ByteExtractKeyword::*)(const KeywordAttr &, const VariablesMapping &)> setops;
+};
+
+const map<string, void(ByteExtractKeyword::*)(const KeywordAttr &, const VariablesMapping &)>
+ByteExtractKeyword::setops = {
+    { "offset",             &ByteExtractKeyword::setOffset       },
+    { "relative",           &ByteExtractKeyword::setRelative     },
+    { "little_endian",      &ByteExtractKeyword::setLittleEndian },
+    { "string",             &ByteExtractKeyword::setDataType     },
+    { "part",               &ByteExtractKeyword::setContext      },
+    { "align",              &ByteExtractKeyword::setAlign        }
+};
+
+ByteExtractKeyword::ByteExtractKeyword(const vector<KeywordAttr> &attrs, VariablesMapping &vars)
+        :
+    offset()
+{
+    //two requied attributes - number of bytes and var name
+    if (attrs.size() < 2) throw KeywordError("Invalid number of attributes in the 'byte_extract' keyword");
+
+    //parisng first attribute (Required) - number of bytes
+    auto &bytes_param = attrs[0].getParams();
+    if (bytes_param.size() != 1) {
+        throw KeywordError("More than one element in the 'bytes' in the 'byte_extract' keyword");
+    }
+    bytes.setAttr("bytes", bytes_param[0], vars, "byte_extract", static_cast<uint>(BaseId::DEC), true);
+    if (bytes.isConstant() && bytes.evalAttr(nullptr) == 0) {
+        throw KeywordError("Number of bytes is zero in the 'byte_extract' keyword");
+    }
+
+    //parisng second attribute (Required) - variable name
+    auto &var_name_param = attrs[1].getParams();
+    if (var_name_param.size() != 1) {
+        throw KeywordError("More than one element in the variable name in the 'byte_extract' keyword");
+    }
+    const string &var_name = var_name_param[0];
+    auto curr = setops.find(var_name);
+    if (curr != setops.end()) {
+        throw KeywordError("'" + var_name + "' cannot be the variable name in the 'byte_extract' keyword");
+    }
+    if (isdigit(var_name[0]) || var_name[0] == '-') {
+        throw KeywordError("Malformed variable name in the 'byte_extract' keyword");
+    }
+
+    var_id = vars.addNewVariable(var_name);
+
+    //parsing the other optional attributes
+    for (uint i = 2; i < attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'byte_extract' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i], vars);
+    }
+
+    if (data_type == BaseId::BIN) {
+        if (!bytes.isConstant()) {
+            throw KeywordError("Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword");
+        }
+        int num_bytes = bytes.evalAttr(nullptr);
+        if (num_bytes != 1 && num_bytes != 2 && num_bytes != 4) {
+            throw KeywordError("Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword");
+        }
+        if (is_little_end && num_bytes == 1) {
+            throw KeywordError(
+                "Little endian is set, "
+                "but the number of bytes is invalid in the 'byte_extract' keyword"
+            );
+        }
+        if (align != 1) {
+            throw KeywordError("The 'align' is set and data type is binary in the 'byte_extract' keyword");
+        }
+    } else {
+        if (is_little_end) {
+            throw KeywordError("Little endian is set, but the data type is not binary in the 'byte_extract' keyword");
+        }
+    }
+}
+
+static uint
+addOffset(uint offset, int add)
+{
+    if (add < 0 && offset < static_cast<uint>(-add)) {
+        dbgWarning(D_KEYWORD)
+            << "The offset was set to 0 "
+            << "due to an attempt to jump before the beginning of the buffer in the 'jump' keyword";
+        return 0;
+    }
+    return offset + add;
+}
+
+pair<uint, uint>
+ByteExtractKeyword::getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const
+{
+    uint relative_offset = is_relative ? prev->getOffset(ctx) : 0;
+    int offset_attr = offset.evalAttr(prev);
+    uint start_offset = addOffset(relative_offset, offset_attr);
+
+    if (start_offset >= buf_size) return make_pair(0, 0);
+
+    uint length = buf_size - start_offset;
+    return make_pair(start_offset, length);
+}
+
+Maybe<uint>
+ByteExtractKeyword::readValue(uint start, uint length, const Buffer &buf) const
+{
+    if (data_type != BaseId::BIN) return readStringValue(start, length, buf);
+
+    uint res = 0;
+    for (uint i = 0; i < length; i++) {
+        uint ch = buf[start + i];
+        if (is_little_end) {
+            ch <<= 8*i;
+            res += ch;
+        } else {
+            res <<= 8;
+            res += ch;
+        }
+    }
+    return res;
+}
+
+Maybe<uint>
+ByteExtractKeyword::readStringValue(uint start, uint length, const Buffer &buf) const
+{
+    const u_char *data = buf.getPtr(start, length).unpack(); // start and length were checked outside of the function
+    string val_str(reinterpret_cast<const char *>(data), length);
+
+    uint base = static_cast<uint>(data_type);
+    try {
+        size_t idx;
+        auto res = stoul(val_str, &idx, base);
+        if (idx != val_str.length()) throw invalid_argument("");
+        if (res > INT_MAX) {
+            throw out_of_range("");
+        }
+        return res;
+    }
+    catch (invalid_argument &) {
+        return genError("Unable to convert the \"" + val_str + "\" to a number due to an invalid argument");
+    }
+    catch (out_of_range &) {
+        return genError(
+            "Unable to convert the \""
+            + val_str
+            + "\" to a number. The maximum is: "
+            + to_string(INT_MAX)
+        );
+    }
+}
+
+uint
+ByteExtractKeyword::applyAlignment(uint value) const
+{
+    int reminder = value % align;
+    if (reminder != 0) {
+        value += (align - reminder);
+    }
+    return value;
+}
+
+MatchStatus
+ByteExtractKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    auto part = Singleton::Consume<I_Environment>::by<KeywordComp>()->get<Buffer>(static_cast<string>(ctx));
+
+    if (!part.ok()) return MatchStatus::NoMatchFinal;
+
+    uint bytes_to_extr = bytes.evalAttr(prev);
+    if (bytes_to_extr == 0) {
+        dbgDebug(D_KEYWORD) << "Number of bytes is zero in the 'byte_extract' keyword";
+        return MatchStatus::NoMatch;  //the case of constant number of bytes was checked during compilation
+    }
+    
+    uint start_offset, length_to_end;
+    tie(start_offset, length_to_end) = getStartOffsetAndLength((*part).size(), prev);
+
+    uint offset_after_extracted_bytes = applyAlignment(start_offset + bytes_to_extr);
+
+    if (length_to_end == 0 || offset_after_extracted_bytes > (*part).size()) {
+        dbgDebug(D_KEYWORD)
+            << "Offset after the number of bytes to extract exceeds the buffer size in the 'byte_extract' keyword";
+        return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+    }
+
+    auto res = readValue(start_offset, bytes_to_extr, *part);
+    if (!res.ok()) {
+        dbgDebug(D_KEYWORD) << "Trying to store an invalid value in the 'byte_extract' keyword: " + res.getErr();
+        return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+    }
+
+    uint extracted_val = res.unpack();
+
+    if (extracted_val > INT_MAX) {
+        dbgDebug(D_KEYWORD) << "Value exceeds the maximum in the 'byte_extract' keyword";
+        return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+    }
+
+    //add variable and move offset after number of extracted bytes
+    VariableRuntimeState new_var(prev, var_id, extracted_val);
+    OffsetRuntimeState new_offset(&new_var, ctx, offset_after_extracted_bytes);
+    return runNext(&new_offset);
+}
+
+unique_ptr<SingleKeyword>
+genByteExtractKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<ByteExtractKeyword>(attr, known_vars);
+}

components/utils/keywords/compare_keyword.cc

@@ -0,0 +1,70 @@
+#include "single_keyword.h"
+#include "output.h"
+#include "debug.h"
+
+#include <map>
+#include <strings.h>
+#include "limits.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class CompareKeyword : public SingleKeyword
+{
+public:
+    explicit CompareKeyword(const vector<KeywordAttr> &attr, const VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override;
+
+private:
+    bool
+    isConstant() const
+    {
+        return first_val.isConstant() && second_val.isConstant();
+    }
+
+    NumericAttr    first_val;
+    NumericAttr    second_val;
+    ComparisonAttr comparison;
+};
+
+CompareKeyword::CompareKeyword(const vector<KeywordAttr> &attrs, const VariablesMapping &vars)
+{
+    if (attrs.size() != 3) throw KeywordError("Invalid number of attributes in the 'compare' keyword");
+    
+    auto &first_val_param = attrs[0].getParams();
+    if (first_val_param.size() != 1) {
+        throw KeywordError("More than one element in the first value in the 'compare' keyword");
+    }
+    first_val.setAttr("first_val", first_val_param[0], vars, "compare");
+
+    auto &comparison_param = attrs[1].getParams();
+    if (comparison_param.size() != 1) {
+        throw KeywordError("More than one element in the comparison operator in the 'compare' keyword");
+    }
+    comparison.setAttr(comparison_param[0], "compare");
+
+    auto &second_val_param = attrs[2].getParams();
+    if (second_val_param.size() != 1) {
+        throw KeywordError("More than one element in the second value in the 'compare' keyword");
+    }
+    second_val.setAttr("second_val", second_val_param[0], vars, "compare");
+}
+
+MatchStatus
+CompareKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    int keyword_first_val = first_val.evalAttr(prev);
+    int keyword_second_val = second_val.evalAttr(prev);
+
+    if (comparison(keyword_first_val, keyword_second_val)) return runNext(prev);
+
+    // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match
+    return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+}
+
+unique_ptr<SingleKeyword>
+genCompareKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<CompareKeyword>(attr, known_vars);
+}

components/utils/keywords/data_keyword.cc

@@ -0,0 +1,409 @@
+#include "single_keyword.h"
+#include "output.h"
+#include "debug.h"
+
+#include <map>
+#include <strings.h>
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class DataKeyword : public SingleKeyword
+{
+public:
+    explicit DataKeyword(const vector<KeywordAttr> &attr, const VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override;
+
+private:
+    void
+    setOffset(const KeywordAttr &attr, const VariablesMapping &vars)
+    {
+        offset.setAttr(attr, vars, "data");
+    }
+
+    void
+    setDepth(const KeywordAttr &attr, const VariablesMapping &vars)
+    {
+        depth.setAttr(attr, vars, "data");
+    }
+
+    void
+    setCaret(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_caret.setAttr(attr, "data");
+    }
+
+    void
+    setRelative(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_relative.setAttr(attr, "data");
+    }
+
+    void
+    setCaseInsensitive(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_case_insensitive.setAttr(attr, "data");
+    }
+
+    void
+    setContext(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        ctx.setAttr(attr, "data");
+    }
+
+    void parseString(const string &str);
+
+    void
+    addChar(char ch)
+    {
+        pattern.push_back(static_cast<unsigned char>(ch));
+    }
+
+    void calcTables();
+
+    pair<uint, uint> getStartAndEndOffsets(uint buf_size, const I_KeywordRuntimeState *prev) const;
+    uint bytesMatched(const Buffer&, uint) const;
+
+    uint
+    moveOnMatch() const
+    {
+        return pattern.size();
+    }
+
+    uint
+    moveOnNoMatch(uint offset_from_end, unsigned char first_unmatched_byte) const
+    {
+        dbgAssert(shift.size() > offset_from_end) << "Shift table of the 'data' keyword is shorter than the offset";
+
+        uint skip_size;
+        if (skip[first_unmatched_byte]>offset_from_end) {
+            skip_size = skip[first_unmatched_byte]-offset_from_end;
+        } else {
+            skip_size = 1;
+        }
+
+        return max(shift[offset_from_end], skip_size);
+    }
+
+    bool
+    isConstant() const
+    {
+        return !is_relative && offset.isConstant() && depth.isConstant();
+    }
+
+    vector<unsigned char> pattern;
+    uint                  skip[256];
+    vector<uint>          shift;
+
+    NumericAttr           offset;
+    NumericAttr           depth;
+    BoolAttr              is_negative;
+    BoolAttr              is_caret;
+    BoolAttr              is_relative;
+    BoolAttr              is_case_insensitive;
+    CtxAttr               ctx;
+
+    static const map<string, void(DataKeyword::*)(const KeywordAttr &, const VariablesMapping &)> setops;
+};
+
+const map<string, void(DataKeyword::*)(const KeywordAttr &, const VariablesMapping &)> DataKeyword::setops = {
+    { "relative", &DataKeyword::setRelative        },
+    { "offset",   &DataKeyword::setOffset          },
+    { "depth",    &DataKeyword::setDepth           },
+    { "caret",    &DataKeyword::setCaret           },
+    { "nocase",   &DataKeyword::setCaseInsensitive },
+    { "part",     &DataKeyword::setContext         }
+};
+
+DataKeyword::DataKeyword(const vector<KeywordAttr> &attrs, const VariablesMapping &vars)
+        :
+    offset(),
+    depth()
+{
+    auto &pattern_param = attrs[0].getParams();
+
+    if (pattern_param.size() != 1) throw KeywordError("More than one element in the 'data' keyword pattern");
+    const string &string_pattern = pattern_param[0];
+
+    if (string_pattern.length() == 0) throw KeywordError("No input for the 'data' keyword");
+
+    uint start = 0;
+    if (string_pattern[0] == '!') {
+        is_negative.setAttr("data", "negative");
+        start++;
+    }
+    if (string_pattern[start] != '"') throw KeywordError("The data pattern does not begin with '\"'");
+
+    uint end = string_pattern.length()-1;
+    if (string_pattern[end] != '"') throw KeywordError("The data pattern does not end with '\"'");
+
+    if (start+1 >= end) throw KeywordError("No input for the 'data' keyword");
+
+    parseString(string_pattern.substr(start+1, end-(start+1)));
+
+    for (uint i = 1; i<attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'data' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i], vars);
+    }
+
+    calcTables();
+}
+
+void
+DataKeyword::calcTables()
+{
+    if (is_case_insensitive) {
+        for (auto &ch : pattern) {
+            if (isupper(ch)) {
+                ch = tolower(ch);
+            }
+        }
+    }
+
+    // Initialize skip table - when we meet a charecter that isn't in the pattern, we skip the whole pattern
+    for (auto &ch_skip : skip) {
+        ch_skip = pattern.size();
+    }
+
+    // Go over the charecters in the pattern.
+    // We can skip from a charecter to the end of the pattern.
+    // If a charecter appear more than once, the latest occurence take precedent.
+    for (uint index = 0; index<pattern.size(); index++) {
+        unsigned char ch = pattern[index];
+
+        uint dist_to_end = pattern.size()-(index+1);
+        if (is_case_insensitive && islower(ch)) {
+            skip[toupper(ch)] = dist_to_end;
+        }
+        skip[ch] = dist_to_end;
+    }
+
+    // Initialize the shift table.
+    shift.resize(pattern.size(), 0);
+
+    uint end_offset = pattern.size()-1;
+    // Go over all the suffixes (from the empty to the pattern-1)
+    for (size_t suffix_len = 0; suffix_len<pattern.size(); suffix_len++) {
+        // Find the smallest shift, so when shifting the suffix left:
+        // 1. All chars overlapping between pattern and shifted suffix match.
+        // 2. If the character before the shifted suffix overlaps the pattern, it doesn't match.
+        // pattern = "hellololo", suff=2 (must match "[^o]lo"), shift=4 ("hel(lo)lolo")
+        // pattern = "olo" suff=2 (must match "[^o]lo"), shift=2 ("(.o)lo")
+        //           characters before the patterns are considered wild.
+        for (uint shift_offset = 1; shift_offset<=pattern.size(); shift_offset++) {
+            // Verify that in the current offset matches the suffix
+            size_t num_of_overlapping_char;
+            unsigned char *suffix_start_ptr;
+            unsigned char *shifted_suffix_start_ptr;
+            if (shift_offset+suffix_len <= pattern.size()) {
+                // Shifted suffix doesn't exceed the pattern. Compare the whole suffix.
+                num_of_overlapping_char  = suffix_len;
+                suffix_start_ptr         = pattern.data() + pattern.size() - suffix_len;
+                shifted_suffix_start_ptr = suffix_start_ptr - shift_offset;
+            } else {
+                // Shifted suffix exceeds the pattern. Compare only the overlaping charecters.
+                num_of_overlapping_char  = pattern.size() - shift_offset;
+                suffix_start_ptr         = pattern.data() + shift_offset;
+                shifted_suffix_start_ptr = pattern.data();
+            }
+
+            if (bcmp(suffix_start_ptr, shifted_suffix_start_ptr, num_of_overlapping_char) != 0) continue;
+
+            // Verify that what comes after the suffix doesn't match
+            if (shift_offset+suffix_len < pattern.size()) {
+                if (pattern[end_offset-suffix_len] == pattern[end_offset-(shift_offset+suffix_len)]) continue;
+            }
+
+            // Set the currect shift offset
+            shift[suffix_len] = shift_offset;
+            break;
+        }
+    }
+}
+
+void
+DataKeyword::parseString(const string &str)
+{
+    string hex;
+    bool hex_mode = false;
+    bool after_bslash = false;
+
+    for (auto ch : str) {
+        if (after_bslash) {
+            if (!isprint(ch)) {
+                throw KeywordError(
+                    "Illegal backslash character '" +
+                    dumpHexChar(ch) +
+                    "' in the pattern in the 'data' keyword"
+                );
+            }
+            addChar(ch);
+            after_bslash = false;
+            continue;
+        }
+
+        switch (ch) {
+        case '|': {
+            if (!hex_mode) {
+                hex = "";
+                hex_mode = true;
+            } else {
+                if (hex.size()>0) throw KeywordError("Stoping in the middle of hex string in the 'data' keyword");
+                hex_mode = false;
+            }
+            break;
+        }
+        case '\\': {
+            if (hex_mode) throw KeywordError("Backslash in hex string in the 'data' keyword");
+            after_bslash = true;
+            break;
+        }
+        case '"': {
+            throw KeywordError("Unescaped double quotation mark in the 'data' keyword");
+            break;
+        }
+        default:
+            if (hex_mode) {
+                if (!isxdigit(ch)) {
+                    if (ch != ' ') {
+                        throw KeywordError(
+                            "Illegal character '" +
+                            dumpHexChar(ch) +
+                            "' in the hex string in the 'data' keyword"
+                        );
+                    }
+                    if (hex.size()>0) {
+                        throw KeywordError("Space separating nibbles in the hex string in the 'data' keyword");
+                    }
+                    break;
+                }
+
+                hex += ch;
+
+                if (hex.size()>=2) {
+                    addChar(stol(hex, nullptr, 16));
+                    hex = "";
+                }
+            } else {
+                if (!isprint(ch)) {
+                    throw KeywordError(
+                        "Illegal character '" +
+                        dumpHexChar(ch) +
+                        "' in the pattern in the 'data' keyword"
+                    );
+                }
+                addChar(ch);
+            }
+        }
+    }
+
+    if ( hex_mode || after_bslash ) {
+        throw KeywordError("The 'data' keyword's pattern has ended in the middle of the parsing");
+    }
+}
+
+static uint
+addOffset(uint offset, int add)
+{
+    if (add<0 && offset<static_cast<uint>(-add)) return 0;
+    return offset + add;
+}
+
+pair<uint, uint>
+DataKeyword::getStartAndEndOffsets(uint buf_size, const I_KeywordRuntimeState *prev) const
+{
+    uint relative_offset = is_relative?prev->getOffset(ctx):0;
+    int offset_attr = offset.evalAttr(prev);
+    uint start_offset = addOffset(relative_offset, offset_attr);
+
+    if (depth.isSet()) {
+        uint depth_size = addOffset(start_offset, depth.evalAttr(prev));
+        buf_size = std::min(buf_size, depth_size);
+    }
+    if (is_caret) {
+        buf_size = std::min(buf_size, start_offset+static_cast<uint>(pattern.size()));
+    }
+
+    return make_pair(start_offset, buf_size);
+}
+
+uint
+DataKeyword::bytesMatched(const Buffer &buf, uint offset) const
+{
+    if (is_case_insensitive) {
+        for (uint i = 0;  i<pattern.size(); i++) {
+            if (pattern[pattern.size()-(i+1)] != tolower(buf[offset-(i+1)])) return i;
+        }
+    } else {
+        for (uint i = 0 ; i < pattern.size() ; i++ ) {
+            if (pattern[pattern.size()-(i+1)] != buf[offset-(i+1)] ) return i;
+        }
+    }
+    return pattern.size();
+}
+
+MatchStatus
+DataKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    dbgAssert(pattern.size()>0) << "Trying to run on an uninitialized keyword data";
+
+    dbgDebug(D_KEYWORD) << "Searching for " << dumpHex(pattern);
+
+    auto part = Singleton::Consume<I_Environment>::by<KeywordComp>()->get<Buffer>(static_cast<string>(ctx));
+    if (!part.ok()) {
+        if (is_negative) return runNext(prev);
+        return MatchStatus::NoMatchFinal;
+    }
+
+    const auto &buf = part.unpack();
+
+    dbgTrace(D_KEYWORD) << "Full buffer: " << dumpHex(buf);
+
+    uint offset, max_offset;
+
+    tie(offset, max_offset) = getStartAndEndOffsets(buf.size(), prev);
+    offset += pattern.size();
+
+    bool match_found = false;
+    while (offset<=max_offset) {
+        // Short circuit for the common, simple case where the last byte doesn't match
+        if (skip[buf[offset-1]]) {
+            offset += skip[buf[offset - 1]];
+            continue;
+        }
+
+        // Full search Boyer-Moore
+        uint match_size = bytesMatched(buf, offset);
+        if (match_size == pattern.size()) {
+            if (is_negative) {
+                return isConstant()?MatchStatus::NoMatchFinal:MatchStatus::NoMatch;
+            }
+            match_found = true;
+            OffsetRuntimeState new_offset(prev, ctx, offset);
+            auto next_keyword_result = runNext(&new_offset);
+            if (next_keyword_result!=MatchStatus::NoMatch) return next_keyword_result;
+            offset += moveOnMatch();
+        } else {
+            offset += moveOnNoMatch(match_size, buf[offset-(match_size+1)]);
+        }
+    }
+
+    // No matchs is a success for negative keywords
+    if (is_negative && !match_found) return runNext(prev);
+
+    // If there were no matchs and the keyword is an effected by other keywords, then we know that the rule won't match
+    if (isConstant() && !match_found) return MatchStatus::NoMatchFinal;
+
+    return MatchStatus::NoMatch;
+}
+
+unique_ptr<SingleKeyword>
+genDataKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<DataKeyword>(attr, known_vars);
+}

components/utils/keywords/jump_keyword.cc

@@ -0,0 +1,174 @@
+#include "single_keyword.h"
+#include "output.h"
+#include "debug.h"
+
+#include <map>
+#include <strings.h>
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class jumpKeyword : public SingleKeyword
+{
+public:
+    explicit jumpKeyword(const vector<KeywordAttr> &attr, const VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override;
+
+private:
+    enum class JumpFromId
+    {
+        RELATIVE,
+        FROM_BEGINNING,
+        FROM_END
+    };
+
+    void
+    setContext(const KeywordAttr &attr)
+    {
+        ctx.setAttr(attr, "byte_extract");
+    }
+
+    void
+    setAlign(const KeywordAttr &attr)
+    {
+        if (align != 1) throw KeywordError("Double definition of the 'align' in the 'jump' keyword");
+        auto &vec = attr.getParams();
+        if (vec.size() != 2) throw KeywordError("Malformed 'align' in the 'jump' keyword");
+
+        if (vec[1] == "2") {
+            align = 2;
+        } else if (vec[1] == "4") {
+            align = 4;
+        } else {
+            throw KeywordError("Unknown 'align' in the 'jump' keyword: " + vec[1]);
+        }
+    }
+    
+    bool
+    isConstant() const
+    {
+        return jumping_from != JumpFromId::RELATIVE && jumping_val.isConstant();
+    }
+
+    JumpFromId  jumping_from;
+    NumericAttr jumping_val;
+    int         align = 1;
+    CtxAttr     ctx;
+
+    static const map<string, void(jumpKeyword::*)(const KeywordAttr &)> setops;
+    uint getStartOffset(uint buf_size, const I_KeywordRuntimeState *prev) const;
+    uint applyAlignment(uint value) const;
+    uint addOffset(uint offset, int add) const;
+};
+
+const map<string, void(jumpKeyword::*)(const KeywordAttr &)> jumpKeyword::setops = {
+    { "part",            &jumpKeyword::setContext   },
+    { "align",           &jumpKeyword::setAlign     }
+};
+
+jumpKeyword::jumpKeyword(const vector<KeywordAttr> &attrs, const VariablesMapping &vars)
+{
+
+    //two requied attributes - jumping value and jumping from
+    if (attrs.size() < 2) throw KeywordError("Invalid number of attributes in the 'jump' keyword");
+
+    //parisng first attribute (Required) - jumping value
+    auto &jumping_val_param = attrs[0].getParams();
+    if (jumping_val_param.size() != 1) {
+        throw KeywordError("More than one element in the jumping value in the 'jump' keyword");
+    }
+    jumping_val.setAttr("jumping value", jumping_val_param[0], vars, "jump");
+
+    //parisng second attribute (Required) - jumping from
+    auto &jumping_from_param = attrs[1].getParams();
+    if (jumping_from_param.size() != 1) {
+        throw KeywordError("More than one element in the jumping 'from' parameter in the 'jump' keyword");
+    }
+    
+    if (jumping_from_param[0] == "from_beginning") {
+        jumping_from = JumpFromId::FROM_BEGINNING;
+    } else if (jumping_from_param[0] == "from_end") {
+        jumping_from = JumpFromId::FROM_END;
+    } else if (jumping_from_param[0] == "relative") {
+        jumping_from = JumpFromId::RELATIVE;
+    } else {
+        throw KeywordError("Unknown jumping 'from' parameter in the 'jump' keyword: " + jumping_from_param[0]);
+    }
+
+    //parisng optional attributes
+    for (uint i = 2; i < attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute " + attrs[i].getAttrName() + " in the 'jump' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i]);
+    }
+}
+
+uint
+jumpKeyword::applyAlignment(uint value) const
+{
+    int reminder = value % align;
+    if (reminder != 0) {
+        value += (align - reminder);
+    }
+    return value;
+}
+
+uint
+jumpKeyword::addOffset(uint offset, int add) const
+{
+    if (add < 0 && offset < static_cast<uint>(-add)) {
+        dbgWarning(D_KEYWORD)
+            << "The offset was set to 0 "
+            << "due to an attempt to jump before the beginning of the buffer in the 'jump' keyword";
+        return 0;
+    }
+    return applyAlignment(offset + add);
+}
+
+uint
+jumpKeyword::getStartOffset(uint buf_size, const I_KeywordRuntimeState *prev) const
+{
+    switch (jumping_from) {
+        case JumpFromId::FROM_BEGINNING: {
+            return 0;
+        }
+        case JumpFromId::FROM_END: {
+            return buf_size;
+        }
+        case JumpFromId::RELATIVE: {
+            return prev->getOffset(ctx);
+        }
+    }
+    dbgAssert(false) << "Invalid jumping 'from' parameter";
+    return 0;
+}
+
+MatchStatus
+jumpKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    auto part = Singleton::Consume<I_Environment>::by<KeywordComp>()->get<Buffer>(static_cast<string>(ctx));
+
+    if (!part.ok()) return MatchStatus::NoMatchFinal;
+
+    uint start_offset = getStartOffset((*part).size(), prev);
+
+    uint offset_to_jump = addOffset(start_offset, jumping_val.evalAttr(prev));
+
+    if (offset_to_jump > (*part).size()) {
+        dbgDebug(D_KEYWORD) << "New offset exceeds the buffer size in the 'jump' keyword";
+        return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+    }
+
+    OffsetRuntimeState new_offset(prev, ctx, offset_to_jump);
+    return runNext(&new_offset);
+}
+
+unique_ptr<SingleKeyword>
+genJumpKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<jumpKeyword>(attr, known_vars);
+}

components/utils/keywords/keywords_rule.cc

@@ -0,0 +1,171 @@
+#include "keyword_comp.h"
+
+#include <vector>
+#include "sentinel_runtime_state.h"
+
+using namespace std;
+
+static const string whitespaces = " \t";
+
+static string
+getSubStrNoPadding(const string &str, uint start, uint end)
+{
+    auto r_start = str.find_first_not_of(whitespaces, start);
+    auto r_end = str.find_last_not_of(whitespaces, end-1);
+
+    if (r_end==string::npos || r_start==string::npos || r_start>r_end) {
+        throw KeywordError("Found an empty section in the '"+ str + "'");
+    }
+
+    return str.substr(r_start, r_end-r_start+1);
+}
+
+static vector<string>
+split(const string &str, const string &delim, uint start = 0)
+{
+    vector<string> res;
+    uint part_start = start;
+    bool escape = false;
+    bool in_string = false;
+
+    for (uint index = start; index<str.size(); index++) {
+        if (escape) {
+            escape = false;
+            continue;
+        }
+
+        switch (str[index]) {
+            case '\\': {
+                escape = true;
+                break;
+            }
+            case '"': {
+                in_string = !in_string;
+                break;
+            }
+            default:
+                if (!in_string && delim.find(str[index])!=string::npos) {
+                    res.push_back(getSubStrNoPadding(str, part_start, index));
+                    part_start = index+1;
+                }
+        }
+    }
+
+    if (escape||in_string) throw KeywordError("Split has ended in the middle of the parsing");
+
+    if (str.find_first_not_of(whitespaces, part_start)!=string::npos) {
+        res.push_back(getSubStrNoPadding(str, part_start, str.size()));
+    }
+
+    return res;
+}
+
+KeywordAttr::KeywordAttr(const string &str) : params(split(str, whitespaces))
+{
+}
+
+KeywordParsed::KeywordParsed(string const &keyword) {
+    auto index = keyword.find_first_of(':');
+    if (index!=string::npos) {
+        for (auto &str : split(keyword, ",", index+1)) {
+            attr.push_back(KeywordAttr(str));
+        }
+    } else {
+        index = keyword.size();
+    }
+
+    name = getSubStrNoPadding(keyword, 0, index);
+    if (name.find_first_of(whitespaces)!=string::npos) {
+        throw KeywordError("'" + name + "' - cannot be a keyword name");
+    }
+}
+
+uint
+SentinelRuntimeState::getOffset(const std::string &) const
+{
+    return 0;
+}
+
+// LCOV_EXCL_START Reason: this function is tested in one_element_list_negative_test but marked as not covered.
+uint
+SentinelRuntimeState::getVariable(uint var_id) const
+{
+    dbgAssert(false) << "Could not find the variable ID: " << var_id;
+    return 0;
+}
+// LCOV_EXCL_STOP
+
+class SentinelKeyword : public SingleKeyword
+{
+public:
+    MatchStatus
+    isMatch() const
+    {
+        SentinelRuntimeState curr_state;
+        return runNext(&curr_state);
+    }
+
+private:
+    // LCOV_EXCL_START Reason: Unreachable function.
+    MatchStatus
+    isMatch(const I_KeywordRuntimeState *state) const override
+    {
+        return runNext(state);
+    }
+    // LCOV_EXCL_STOP
+};
+
+class KeywordComp::Impl : Singleton::Provide<I_KeywordsRule>::From<KeywordComp>
+{
+public:
+    Maybe<shared_ptr<VirtualRule>>
+    genRule(const string &rule)
+    {
+        shared_ptr<VirtualRule> res;
+        try {
+            res = KeywordsRuleImpl::genRule(rule);
+        } catch (const KeywordError &e) {
+            return genError(e.getErr());;
+        }
+        return move(res);
+    }
+
+private:
+    class KeywordsRuleImpl : public VirtualRule
+    {
+    public:
+        bool isMatch() const override { return start.isMatch() == MatchStatus::Match; }
+
+        static unique_ptr<KeywordsRuleImpl>
+        genRule(const string &rule)
+        {
+            auto res = make_unique<KeywordsRuleImpl>();
+
+            auto pos = rule.find_last_not_of(whitespaces);
+            if (pos==string::npos) {
+                // Empty rule
+                return res;
+            }
+
+            if (rule[pos]!=';') throw KeywordError(rule + " - end of text pass rule");
+
+            VariablesMapping known_vars;
+
+            auto key_vec = split(rule, ";");
+            for (auto &keyword : key_vec) {
+                res->start.appendKeyword(getKeywordByName(keyword, known_vars));
+            }
+
+            return res;
+        }
+
+    private:
+        SentinelKeyword start;
+    };
+};
+
+KeywordComp::KeywordComp() : Component("KeywordComp"), pimpl(make_unique<KeywordComp::Impl>()) {}
+
+KeywordComp::~KeywordComp() {}
+
+string I_KeywordsRule::keywords_tag = "keywords_rule_tag";

components/utils/keywords/keywords_ut/CMakeLists.txt

@@ -0,0 +1,5 @@
+add_unit_test(
+    keywords_ut
+    "keywords_ut.cc;single_keyword_ut.cc"
+    "keywords;pcre2-8;buffers;singleton;table;event_is;metric;-lboost_regex"
+)

components/utils/keywords/keywords_ut/keywords_ut.cc

@@ -0,0 +1,996 @@
+#include "keyword_comp.h"
+#include "environment.h"
+#include "mock/mock_table.h"
+#include "cptest.h"
+#include "mock/mock_time_get.h"
+#include "mock/mock_mainloop.h"
+#include "config.h"
+#include "config_component.h"
+
+using namespace std;
+
+class KeywordsRuleTest : public ::testing::Test
+{
+public:
+    void
+    appendBuffer(const string &id, const string &str)
+    {
+        buffers[id] += Buffer(str);
+    }
+
+    string
+    ruleCompileFail(const string &_rule)
+    {
+        auto rule = Singleton::Consume<I_KeywordsRule>::from(comp)->genRule(_rule);
+        EXPECT_FALSE(rule.ok()) << "Compile supposed to fail";
+        return rule.getErr();
+    }
+
+    bool
+    ruleRun(const string &_rule, const string &default_ctx = "default")
+    {
+        auto rule = Singleton::Consume<I_KeywordsRule>::from(comp)->genRule(_rule);
+        EXPECT_TRUE(rule.ok()) << "Compile not supposed to fail: " << rule.getErr();
+        ScopedContext ctx;
+        ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), default_ctx);
+        for (auto &value : buffers) {
+            ctx.registerValue(value.first, value.second);
+        }
+        return (*rule)->isMatch();
+    }
+
+private:
+    KeywordComp comp;
+    ::testing::NiceMock<MockMainLoop> mock_mainloop;
+    ::testing::NiceMock<MockTimeGet> mock_timer;
+    Environment env;
+    map<string, Buffer> buffers;
+};
+
+TEST_F(KeywordsRuleTest, data_basic_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_TRUE(ruleRun("data: \"234\" , part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("data: \"234\";", "HTTP_RESPONSE_BODY"));
+    EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, data_relative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("data: \"567\", part HTTP_RESPONSE_BODY; data: \"234\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(
+        ruleRun("data: \"567\", part HTTP_RESPONSE_BODY; data: \"234\", part HTTP_RESPONSE_BODY, relative;")
+    );
+    EXPECT_TRUE(ruleRun("data: \"234\", part HTTP_RESPONSE_BODY; data: \"567\", part HTTP_RESPONSE_BODY, relative;"));
+}
+
+TEST_F(KeywordsRuleTest, data_depth_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("data: \"345\", depth 5, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"345\", depth 4, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, data_nocase_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "abcdefg");
+
+    EXPECT_TRUE(ruleRun("data: \"cde\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"CDE\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("data: \"CDE\", nocase, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, data_offset_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("data: \"345\", offset 2, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"345\", offset 3, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, data_caret_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY, caret;"));
+    EXPECT_TRUE(ruleRun("data: \"345\", caret, part HTTP_RESPONSE_BODY, offset 2;"));
+}
+
+TEST_F(KeywordsRuleTest, data_negative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_FALSE(ruleRun("data: !\"345\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("data: !\"365\", part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, data_part_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+    appendBuffer("HTTP_REQUEST_BODY", "abcdefg");
+
+    EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY; data: \"cde\", part HTTP_REQUEST_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY; data: \"cde\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_REQUEST_BODY; data: \"cde\", part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, pcre_basic_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/5..7/\", part HTTP_RESPONSE_BODY;"));
+}
+
+
+TEST_F(KeywordsRuleTest, pcre_relative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/R\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(
+        ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/\", relative, part HTTP_RESPONSE_BODY;")
+    );
+    EXPECT_TRUE(ruleRun("pcre: \"/2.4/\", part HTTP_RESPONSE_BODY; pcre: \"/5.7/R\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(
+        ruleRun("pcre: \"/2.4/\", part HTTP_RESPONSE_BODY; pcre: \"/5.7/\", relative, part HTTP_RESPONSE_BODY;")
+    );
+}
+
+TEST_F(KeywordsRuleTest, pcre_depth_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", depth 5, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", depth 4, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, pcre_nocase_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "abcdefg");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/c.e/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/C.E/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("pcre: \"/C.E/i\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("pcre: \"/C.E/\", nocase, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, pcre_offset_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", offset 2, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", offset 300, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, pcre_part_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+    appendBuffer("HTTP_REQUEST_BODY", "abcdefg");
+
+    EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY; pcre: \"/c.e/\", part HTTP_REQUEST_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY; pcre: \"/c.e/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", part HTTP_REQUEST_BODY; pcre: \"/c.e/\", part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, pcre_negative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_FALSE(ruleRun("pcre: !\"/3.5/\", part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("pcre: !\"/3..5/\", part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, compare_comparison_test) {
+    EXPECT_TRUE(ruleRun("compare: 0, =, 0;"));
+    EXPECT_TRUE(ruleRun("compare: -1, =, -1;"));
+    EXPECT_FALSE(ruleRun("compare: 0, =, 1;"));
+    EXPECT_FALSE(ruleRun("compare: -1, =, -2;"));
+    EXPECT_FALSE(ruleRun("compare: 1, =, -1;"));
+    EXPECT_FALSE(ruleRun("compare: -1, =, 1;"));
+    EXPECT_TRUE(ruleRun("compare: 2, !=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: 2, <=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: 2, <, 3;"));
+    EXPECT_FALSE(ruleRun("compare: 2, >, 3;"));
+    EXPECT_FALSE(ruleRun("compare: 2, >=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, !=, -3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, >=, -3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, >, -3;"));
+    EXPECT_FALSE(ruleRun("compare: -2, <, -3;"));
+    EXPECT_FALSE(ruleRun("compare: -2, <=, -3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, !=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, <=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: -2, <, 3;"));
+    EXPECT_FALSE(ruleRun("compare: -2, >, 3;"));
+    EXPECT_FALSE(ruleRun("compare: -2, >=, 3;"));
+    EXPECT_TRUE(ruleRun("compare: 2, !=, -3;"));
+    EXPECT_TRUE(ruleRun("compare: 2, >=, -3;"));
+    EXPECT_TRUE(ruleRun("compare: 2, >, -3;"));
+    EXPECT_FALSE(ruleRun("compare: 2, <, -3;"));
+    EXPECT_FALSE(ruleRun("compare: 2, <=, -3;"));
+}
+
+TEST_F(KeywordsRuleTest, compare_compile_fail_test) {
+    EXPECT_EQ(ruleCompileFail("compare: 0;"), "Invalid number of attributes in the 'compare' keyword");
+    EXPECT_EQ(ruleCompileFail("compare: 0, =;"), "Invalid number of attributes in the 'compare' keyword");
+    EXPECT_EQ(ruleCompileFail("compare: 0, =, 0, 0;"), "Invalid number of attributes in the 'compare' keyword");
+    EXPECT_EQ(
+        ruleCompileFail("compare: 0 1, =, 0;"),
+        "More than one element in the first value in the 'compare' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("compare: 0, = =, 0;"),
+        "More than one element in the comparison operator in the 'compare' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("compare: 0, =, 0 1;"),
+        "More than one element in the second value in the 'compare' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("compare: 0, ==, 0;"),
+        "Unknown comparison operator in the 'compare' keyword: Could not find the operator: =="
+    );
+}
+
+TEST_F(KeywordsRuleTest, length_basic_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+    appendBuffer("HTTP_REQUEST_BODY", "");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "length: length_var, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 9;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "length: length_var, part HTTP_REQUEST_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "length: length_var, part HTTP_REQUEST_BODY;"
+            "compare: length_var, =, 1;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, length_part_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("length: length_var, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("length: length_var, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, length_relative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"234\", part HTTP_RESPONSE_BODY;"
+            "length: relative_length_var, part HTTP_RESPONSE_BODY, relative;"
+            "compare: relative_length_var, =, 5;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"234\", part HTTP_RESPONSE_BODY;"
+            "length: relative_length_var, part HTTP_RESPONSE_BODY;"
+            "compare: relative_length_var, =, 5;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"89\", part HTTP_RESPONSE_BODY;"
+            "length: zero_length_var, part HTTP_RESPONSE_BODY, relative;"
+            "compare: zero_length_var, =, 0;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"89\", part HTTP_RESPONSE_BODY;"
+            "length: zero_length_var, part HTTP_RESPONSE_BODY;"
+            "compare: zero_length_var, =, 0;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, length_compare_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123");
+    EXPECT_FALSE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;"));
+
+    appendBuffer("HTTP_RESPONSE_BODY", "456");
+    EXPECT_TRUE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;"));
+
+    appendBuffer("HTTP_RESPONSE_BODY", "789");
+    EXPECT_TRUE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, length_compile_fail_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_EQ(
+        ruleCompileFail("length: two_elem 2, part HTTP_RESPONSE_BODY;"),
+        "More than one element in the variable name in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length: relative, part HTTP_RESPONSE_BODY;"),
+        "The 'relative' cannot be the variable name in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length: part, part HTTP_RESPONSE_BODY;"),
+        "The 'part' cannot be the variable name in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length: -minus, part HTTP_RESPONSE_BODY;"),
+        "Malformed variable name in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length: 1digit, part HTTP_RESPONSE_BODY;"),
+        "Malformed variable name in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length: bad_attr, partt HTTP_RESPONSE_BODY;"),
+        "Unknown attribute 'partt' in the 'length' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("length:;"),
+        "Invalid number of attributes in the 'length' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_dec_string_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, dec_var, string dec, part HTTP_RESPONSE_BODY;"
+            "data: \"234\", offset dec_var, part HTTP_RESPONSE_BODY;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 1, dec_var, string dec, part HTTP_RESPONSE_BODY;"
+            "data: \"123\", offset dec_var, part HTTP_RESPONSE_BODY;"
+        )
+    );
+
+    appendBuffer("HTTP_REQUEST_BODY", "A");
+
+    EXPECT_FALSE(ruleRun("byte_extract: 1, bad_dec_var, string dec, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_hex_string_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "A123");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, hex_var, string hex, part HTTP_RESPONSE_BODY;"
+            "compare: hex_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 2, hex_var, string hex, part HTTP_RESPONSE_BODY;"
+            "compare: hex_var, =, 161;"
+        )
+    );
+
+    appendBuffer("HTTP_REQUEST_BODY", "10G");
+
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 2, hex_var, string hex, part HTTP_REQUEST_BODY;"
+            "compare: hex_var, =, 10;"
+        )
+    );
+    EXPECT_FALSE(ruleRun("byte_extract: 3, bad_hex_var, string oct, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_oct_string_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "13ABC");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 2, oct_var, string oct, part HTTP_RESPONSE_BODY;"
+            "compare: oct_var, =, 11;"
+        )
+    );
+
+    appendBuffer("HTTP_REQUEST_BODY", "118");
+
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 2, oct_var, string oct, part HTTP_REQUEST_BODY;"
+            "compare: oct_var, =, 13;"
+        )
+    );
+    EXPECT_FALSE(ruleRun("byte_extract: 3, bad_oct_var, string oct, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_binary_data_test) {
+    string one_byte_binary_data = {10};
+    appendBuffer("HTTP_RESPONSE_BODY", one_byte_binary_data);
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, binary_data_var, part HTTP_RESPONSE_BODY;"
+            "compare: binary_data_var, =, 10;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 1, dec_data_var, offset 2, string dec, part HTTP_RESPONSE_BODY;"
+            "compare: dec_data_var, =, 10;"
+        )
+    );
+
+    string two_bytes_binary_data = {1, 0, 0};
+    appendBuffer("HTTP_REQUEST_BODY", two_bytes_binary_data);
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 2, binary_data_var, part HTTP_REQUEST_BODY;"
+            "compare: binary_data_var , =, 256;"
+        )
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 3, not1/2/4, part HTTP_REQUEST_BODY;"),
+        "Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail(
+            "byte_extract: 1, no_constant, part HTTP_REQUEST_BODY;"
+            "byte_extract: no_constant, var, part HTTP_REQUEST_BODY;"
+        ),
+        "Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_bad_num_of_bytes_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "0");
+
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 0, zero_bytes_var, string dec, part HTTP_RESPONSE_BODY;"),
+        "Number of bytes is zero in the 'byte_extract' keyword"
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 1, one_byte_var, string dec, part HTTP_RESPONSE_BODY;"
+            "byte_extract: one_byte_var, zero_bytes_var, string dec, part HTTP_RESPONSE_BODY;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_part_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123");
+    EXPECT_TRUE(ruleRun("byte_extract: 1, part_var, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("byte_extract: 1, part_var, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_offset_test) {
+    appendBuffer("HTTP_REQUEST_BODY", "1A23456789hello");
+
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 1, hex_var, offset 1, string hex, part HTTP_REQUEST_BODY; "
+            "data: \"9hell\", offset hex_var, part HTTP_REQUEST_BODY;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, hex_var, offset 1, string hex, part HTTP_REQUEST_BODY;"
+            "data: \"hell\", offset hex_var, part HTTP_REQUEST_BODY;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 1, dec_var, offset -1, string dec, part HTTP_REQUEST_BODY;"
+            "data: \"1A2\", offset dec_var, part HTTP_REQUEST_BODY;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, dec_var, offset -1, string dec, part HTTP_REQUEST_BODY;"
+            "data: \"A2\", offset dec_var, part HTTP_REQUEST_BODY;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_relative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "123456789");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"12\", part HTTP_RESPONSE_BODY;"
+            "byte_extract: 1, relative_var, relative, string dec, part HTTP_RESPONSE_BODY;"
+            "compare: relative_var, =, 3;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"12\", part HTTP_RESPONSE_BODY;"
+            "byte_extract: 1, non_relative_var, string dec, part HTTP_RESPONSE_BODY;"
+            "compare: non_relative_var, =, 3;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"89\", part HTTP_RESPONSE_BODY;"
+            "byte_extract: 1, relative_var, string dec, relative, part HTTP_RESPONSE_BODY;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_endianness_test) {
+    string little_end_test_str = {8, 0, 0};
+    appendBuffer("HTTP_RESPONSE_BODY", little_end_test_str);
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 2, lit_end_var, little_endian, part HTTP_RESPONSE_BODY;"
+            "compare: lit_end_var, =, 8;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 2, big_end_var, part HTTP_RESPONSE_BODY;"
+            "compare: big_end_var, =, 8;"
+        )
+    );
+
+    little_end_test_str[1] = 0;
+    little_end_test_str[2] = 1;
+    appendBuffer("HTTP_REQUEST_BODY", little_end_test_str);
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 2, lit_end_with_offset_var,"
+            "offset 1, little_endian, part HTTP_REQUEST_BODY;"
+            "compare: lit_end_with_offset_var, =, 256;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: 2, big_end_with_offset_var, offset 1, part HTTP_REQUEST_BODY;"
+            "compare: big_end_with_offset_var, =, 256;"
+        )
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, var, little_endian, part HTTP_REQUEST_BODY;"),
+        "Little endian is set, but the number of bytes is invalid in the 'byte_extract' keyword"
+    );
+
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 2, no_binary, little_endian, string dec, part HTTP_REQUEST_BODY;"),
+        "Little endian is set, but the data type is not binary in the 'byte_extract' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_align_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, align2_var, align 2, string dec, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 2;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, align4_var, align 4, string dec, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, align2_var, offset 3, align 2, string dec, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, align4_var, offset 3, align 4, string dec, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+
+    appendBuffer("HTTP_REQUEST_BODY", "123");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: 1, align2_var, offset 1, align 2, string dec, part HTTP_REQUEST_BODY;"
+            "length: length_var, relative, part HTTP_REQUEST_BODY;"
+            "compare: length_var, =, 1;"
+        )
+    );
+    EXPECT_FALSE(ruleRun("byte_extract: 1, align4_var, align 4, string dec, part HTTP_REQUEST_BODY;"));
+    EXPECT_FALSE(ruleRun("byte_extract: 1, align2_var, offset 2, align 2, string dec, part HTTP_REQUEST_BODY;"));
+
+    string binary_data_str = { 1 };
+    appendBuffer("HTTP_REQUEST_BODY", binary_data_str);
+
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, align_binary_var, align 2, part HTTP_REQUEST_BODY;"),
+        "The 'align' is set and data type is binary in the 'byte_extract' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_overflow_test) {
+    string overflow_dec_data_str = to_string((uint)INT_MAX + 1);
+    appendBuffer("HTTP_RESPONSE_BODY", overflow_dec_data_str);
+
+    EXPECT_FALSE(
+        ruleRun(
+            "byte_extract: " + to_string(overflow_dec_data_str.length()) + ","
+            "overflow_var, string dec, part HTTP_RESPONSE_BODY;"
+        )
+    );
+
+    string max_value_dec_data_str = to_string(INT_MAX);
+    appendBuffer("HTTP_REQUEST_BODY", max_value_dec_data_str);
+
+    EXPECT_TRUE(
+        ruleRun(
+            "byte_extract: " + to_string(max_value_dec_data_str.length()) + ","
+            "max_var, string dec, part HTTP_REQUEST_BODY;"
+            "compare: max_var, =, " + max_value_dec_data_str + ";"
+        )
+    );
+
+    string overflow_binary_data_str = { 0x7f, 0x7f, 0x7f, 0x7f, 0 };
+    appendBuffer("HTTP_REQUEST_HEADERS", overflow_binary_data_str);
+
+    EXPECT_FALSE(ruleRun("byte_extract: 5 ,overflow_num_var, string dec, part HTTP_REQUEST_HEADERS;"));
+}
+
+TEST_F(KeywordsRuleTest, byte_extract_compile_fail_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1 2, dec_var, string dec, part HTTP_RESPONSE_BODY;"),
+        "More than one element in the 'bytes' in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, dec_var 1, string dec, part HTTP_RESPONSE_BODY;"),
+        "More than one element in the variable name in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, align, string dec, part HTTP_RESPONSE_BODY;"),
+        "'align' cannot be the variable name in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, -1, string dec, part HTTP_RESPONSE_BODY;"),
+        "Malformed variable name in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_data_type, string dechex, part HTTP_RESPONSE_BODY;"),
+        "Unknown data type in the 'byte_extract' keyword: dechex"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, 1var, string dec, part HTTP_RESPONSE_BODY;"),
+        "Malformed variable name in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_align, align 3, part HTTP_RESPONSE_BODY;"),
+        "Unknown 'align' in the 'byte_extract' keyword: 3"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_constant, offset 0x;"),
+        "Malformed constant '0x' in the 'offset' in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(ruleCompileFail("byte_extract: 1;"), "Invalid number of attributes in the 'byte_extract' keyword");
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_attr, offset;"),
+        "Malformed offset' in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_attr, string hex dec;"),
+        "Malformed data type in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_attr, ofset 5;"),
+        "Unknown attribute 'ofset' in the 'byte_extract' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("byte_extract: 1, bad_align, align 2 4;"),
+        "Malformed 'align' in the 'byte_extract' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, jump_from_beginning_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 9;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 9;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "length: length_var, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 10, from_beginning, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_FALSE(ruleRun("jump: 11, from_beginning, part HTTP_RESPONSE_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, jump_relative_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"1\", part HTTP_RESPONSE_BODY;"
+            "jump: 1, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 8;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"1\", part HTTP_RESPONSE_BODY;"
+            "jump: 1, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 9;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"1\", part HTTP_RESPONSE_BODY;"
+            "jump: -2, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "data: \"1\", part HTTP_RESPONSE_BODY;"
+            "jump: 9, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "data: \"1\", part HTTP_RESPONSE_BODY;"
+            "jump: 10, relative, part HTTP_RESPONSE_BODY;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, jump_from_end_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_FALSE(ruleRun("jump: 1, from_end, part HTTP_RESPONSE_BODY;"));
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -1, from_end, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 1;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -10, from_end, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -11, from_end, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, combined_jumps_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: -1, from_end, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 1;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: -1, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -1, from_end, part HTTP_RESPONSE_BODY;"
+            "jump: 1, relative, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 0;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: -1, from_end, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 1;"
+        )
+    );
+}
+
+
+TEST_F(KeywordsRuleTest, jump_alignment_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, align 2, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 8;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 1, from_beginning, align 4, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 6;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: 2, relative, align 2, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 4;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: 2, relative, align 2, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 7;"
+        )
+    );
+    EXPECT_FALSE(
+        ruleRun(
+            "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;"
+            "jump: 2, relative, align 4, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 3;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 2, from_beginning, align 2, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 8;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 4, from_beginning, align 4, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 6;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 0, from_beginning, align 2, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+    EXPECT_TRUE(
+        ruleRun(
+            "jump: 0, from_beginning, align 4, part HTTP_RESPONSE_BODY;"
+            "length: length_var, relative, part HTTP_RESPONSE_BODY;"
+            "compare: length_var, =, 10;"
+        )
+    );
+}
+
+TEST_F(KeywordsRuleTest, jump_part_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_TRUE(ruleRun("jump: 1, from_beginning, part HTTP_RESPONSE_BODY;"));
+    EXPECT_FALSE(ruleRun("jump: 1, from_beginning, part HTTP_REQUEST_BODY;"));
+}
+
+TEST_F(KeywordsRuleTest, jump_compile_fail_test) {
+    appendBuffer("HTTP_RESPONSE_BODY", "1234567890");
+
+    EXPECT_EQ(ruleCompileFail("jump: 1;"), "Invalid number of attributes in the 'jump' keyword");
+    EXPECT_EQ(
+        ruleCompileFail("jump: 2 1, from_beginning;"),
+        "More than one element in the jumping value in the 'jump' keyword"
+    );
+    EXPECT_EQ(
+        ruleCompileFail("jump: 2, from_relative;"),
+        "Unknown jumping 'from' parameter in the 'jump' keyword: from_relative"
+    );
+    EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 3;"), "Unknown 'align' in the 'jump' keyword: 3");
+    EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 1;"), "Unknown 'align' in the 'jump' keyword: 1");
+    EXPECT_EQ(ruleCompileFail("jump: 2, relative, align2 2;"), "Unknown attribute align2 in the 'jump' keyword");
+    EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 2 4;"), "Malformed 'align' in the 'jump' keyword");
+    EXPECT_EQ(
+        ruleCompileFail("jump: 2, from_beginning relative;"),
+        "More than one element in the jumping 'from' parameter in the 'jump' keyword"
+    );
+}
+
+TEST_F(KeywordsRuleTest, stateop)
+{
+    using testing::_;
+
+    ConfigComponent conf;
+    testing::StrictMock<MockTable> table;
+
+    std::unique_ptr<TableOpaqueBase> opq;
+    TableOpaqueBase *opq_ptr;
+    bool has_stage = false;
+    EXPECT_CALL(table, createStateRValueRemoved(_, _))
+        .WillOnce(testing::DoAll(
+            testing::Invoke(
+                [&] (const type_index &, std::unique_ptr<TableOpaqueBase> &other)
+                {
+                    opq = std::move(other);
+                    opq_ptr = opq.get();
+                    has_stage = true;
+                }
+            ),
+            testing::Return(true)
+        ));
+    EXPECT_CALL(table, getState(_)).WillRepeatedly(testing::ReturnPointee(&opq_ptr));
+    EXPECT_CALL(table, hasState(_)).WillRepeatedly(testing::ReturnPointee(&has_stage));
+
+    EXPECT_FALSE(ruleRun("stateop: state sss, isset;"));
+
+    EXPECT_TRUE(ruleRun("stateop: state sss, unset;"));
+    EXPECT_FALSE(ruleRun("stateop: state sss, isset;"));
+
+    EXPECT_TRUE(ruleRun("stateop: state sss, set;"));
+    EXPECT_TRUE(ruleRun("stateop: state sss, isset;"));
+    EXPECT_FALSE(ruleRun("stateop: state dd, isset;"));
+
+    EXPECT_TRUE(ruleRun("stateop: state sss, unset;"));
+    EXPECT_FALSE(ruleRun("stateop: state sss, isset;"));
+}
+
+TEST_F(KeywordsRuleTest, no_match)
+{
+    EXPECT_FALSE(ruleRun("no_match;"));
+}

components/utils/keywords/keywords_ut/single_keyword_ut.cc

@@ -0,0 +1,171 @@
+#include "../sentinel_runtime_state.h"
+#include "../single_keyword.h"
+#include "cptest.h"
+#include <list>
+
+using namespace std;
+using namespace testing;
+
+#define FIRST_VARIABLE_ID 1
+#define FIRST_VARIABLE_VAL 2u
+#define SECOND_VARIABLE_ID 3
+#define SECOND_VARIABLE_VAL 4u
+#define THIRD_VARIABLE_ID 5
+#define THIRD_VARIABLE_VAL 6u
+
+#define FIRST_OFFSET 4u
+#define SECOND_OFFSET 5u
+#define THIRD_OFFSET 6u
+
+const static unsigned int zero = 0;
+
+class I_KeywordRuntimeStateTest : public Test
+{
+public:
+    //  Constructs SentinelKeyword as head because it is the only I_KeywordRuntimeState implementation
+    //  that does not hold I_KeywordRuntimeState *prev
+    I_KeywordRuntimeStateTest() : list_head(&sentinel) {}
+
+    uint
+    getOffset(const string &id)
+    {
+        return list_head->getOffset(id);
+    }
+
+    uint
+    getVariable(uint requested_var_id)
+    {
+        return list_head->getVariable(requested_var_id);
+    }
+
+    void
+    addOffsetState(const string &_ctx, uint _offset)
+    {
+        offset_list.push_front(make_unique<OffsetRuntimeState>(list_head, _ctx, _offset));
+        list_head = offset_list.front().get();
+    }
+
+    void
+    addVariableState(uint _var_id, uint _val)
+    {
+        variable_list.push_front(make_unique<VariableRuntimeState>(list_head, _var_id, _val));
+        list_head = variable_list.front().get();
+    }
+
+private:
+    list<unique_ptr<VariableRuntimeState>> variable_list;
+    list<unique_ptr<OffsetRuntimeState>> offset_list;
+    SentinelRuntimeState sentinel;
+    I_KeywordRuntimeState *list_head;
+};
+
+TEST_F(I_KeywordRuntimeStateTest, one_element_list_positive_test) {
+    EXPECT_EQ(getOffset("HTTP_METHOD"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, one_variable_state_list_positive_test) {
+    addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL);
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero);
+    EXPECT_EQ(getOffset("HTTP_METHOD"), zero);
+    EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, one_offset_state_list_positive_test) {
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero);
+    addOffsetState("HTTP_REQUEST_HEADERS", FIRST_OFFSET);
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), FIRST_OFFSET);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, one_element_list_negative_test) {
+    cptestPrepareToDie();
+    EXPECT_DEATH(getVariable(FIRST_OFFSET), "");
+    EXPECT_DEATH(getVariable(SECOND_OFFSET), "");
+    EXPECT_DEATH(getVariable(THIRD_OFFSET), "");
+}
+
+TEST_F(I_KeywordRuntimeStateTest, variable_runtime_state_list_positive_test) {
+    //  Notice that variables ids and values are different
+    addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL);
+    addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL);
+    addVariableState(THIRD_VARIABLE_ID, THIRD_VARIABLE_VAL);
+
+    EXPECT_EQ(getOffset("HTTP_METHOD"), zero);
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQUEST_BODY"), zero);
+
+    EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL);
+    EXPECT_EQ(getVariable(SECOND_VARIABLE_ID), SECOND_VARIABLE_VAL);
+    EXPECT_EQ(getVariable(THIRD_VARIABLE_ID), THIRD_VARIABLE_VAL);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, OffsetRuntimeState_list_negative_test) {
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET);
+    addOffsetState("HTTP_REQ_COOKIE", SECOND_OFFSET);
+    addOffsetState("HTTP_METHOD", THIRD_OFFSET);
+
+    cptestPrepareToDie();
+    EXPECT_DEATH(getVariable(FIRST_OFFSET), "");
+    EXPECT_DEATH(getVariable(SECOND_OFFSET), "");
+    EXPECT_DEATH(getVariable(THIRD_OFFSET), "");
+}
+
+TEST_F(I_KeywordRuntimeStateTest, offset_runtime_state_list_positive_test) {
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero);
+    EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero);
+
+    addOffsetState("HTTP_REQUEST_HEADERS", FIRST_OFFSET);
+    addOffsetState("HTTP_REQ_COOKIE", SECOND_OFFSET);
+
+    EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), FIRST_OFFSET);
+    EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), SECOND_OFFSET);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_positive_test) {
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero);
+    EXPECT_EQ(getOffset("HTTP_METHOD"), zero);
+
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET);
+    addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL);
+    addOffsetState("HTTP_METHOD", THIRD_OFFSET);
+
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), FIRST_OFFSET);
+    EXPECT_EQ(getOffset("HTTP_METHOD"), THIRD_OFFSET);
+
+    EXPECT_EQ(getVariable(SECOND_VARIABLE_ID), SECOND_VARIABLE_VAL);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_negative_test) {
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET);
+    addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL);
+    addOffsetState("HTTP_METHOD", THIRD_OFFSET);
+
+    cptestPrepareToDie();
+    EXPECT_DEATH(getVariable(FIRST_OFFSET), "");
+    EXPECT_DEATH(getVariable(THIRD_OFFSET), "");
+}
+
+TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_offset_shadowing_test) {
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET);
+
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), FIRST_OFFSET);
+
+    addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL);
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", THIRD_OFFSET);
+
+    EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), THIRD_OFFSET);
+}
+
+TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_variable_shadowing_test) {
+    addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL);
+
+    EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL);
+
+    addOffsetState("HTTP_COMPLETE_URL_ENCODED", SECOND_OFFSET);
+    addVariableState(FIRST_VARIABLE_ID, THIRD_VARIABLE_VAL);
+
+    EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), THIRD_VARIABLE_VAL);
+}

components/utils/keywords/length_keyword.cc

@@ -0,0 +1,162 @@
+#include "single_keyword.h"
+#include "output.h"
+#include "debug.h"
+#include "flags.h"
+
+#include <map>
+#include <strings.h>
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class LengthKeyword : public SingleKeyword
+{
+public:
+    explicit LengthKeyword(const vector<KeywordAttr> &attr, VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override;
+
+private:
+    enum class Mode { EXACT, MIN, MAX, COUNT };
+    using ModeFlags = Flags<Mode>;
+
+    void
+    setRelative(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_relative.setAttr(attr, "length");
+    }
+
+    void
+    setExact(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation");
+        mode.setFlag(Mode::EXACT);
+    }
+
+    void
+    setMin(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation");
+        mode.setFlag(Mode::MIN);
+    }
+
+
+    void
+    setMax(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation");
+        mode.setFlag(Mode::MAX);
+    }
+
+
+    void
+    setContext(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        ctx.setAttr(attr, "length");
+    }
+
+    bool
+    isConstant() const
+    {
+        return !is_relative && compare_size.isConstant();
+    }
+
+    BoolAttr    is_relative;
+    ModeFlags   mode;
+    CtxAttr     ctx;
+    uint        var_id;
+    NumericAttr compare_size;
+
+    static const map<string, void(LengthKeyword::*)(const KeywordAttr &, const VariablesMapping &)> setops;
+};
+
+const map<string, void(LengthKeyword::*)(const KeywordAttr &, const VariablesMapping &)> LengthKeyword::setops = {
+    { "relative",   &LengthKeyword::setRelative },
+    { "exact",      &LengthKeyword::setExact    },
+    { "min",        &LengthKeyword::setMin      },
+    { "max",        &LengthKeyword::setMax      },
+    { "part",       &LengthKeyword::setContext  }
+};
+
+LengthKeyword::LengthKeyword(const vector<KeywordAttr> &attrs, VariablesMapping &vars)
+{
+    if (attrs.size() == 0) throw KeywordError("Invalid number of attributes in the 'length' keyword");
+    
+    //parisng first attribute (Required) - variable name
+    auto &var_name_param = attrs[0].getParams();
+
+    if (var_name_param.size() != 1) {
+        throw KeywordError("More than one element in the variable name in the 'length' keyword");
+    }
+    
+    const string &string_var_name = var_name_param[0];
+
+    if (string_var_name == "relative") {
+        throw KeywordError("The 'relative' cannot be the variable name in the 'length' keyword");
+    }
+    if (string_var_name == "part") {
+        throw KeywordError("The 'part' cannot be the variable name in the 'length' keyword");
+    }
+    if (string_var_name == "exact") {
+        throw KeywordError("The 'exact' cannot be the variable name in the 'length' keyword");
+    }
+    if (string_var_name == "min") {
+        throw KeywordError("The 'min' cannot be the variable name in the 'length' keyword");
+    }
+    if (string_var_name == "max") {
+        throw KeywordError("The 'max' cannot be the variable name in the 'length' keyword");
+    }
+
+    //parsing the other optional attributes
+    for (uint i = 1; i < attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'length' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i], vars);
+    }
+
+    if (mode.empty()) {
+        if (isdigit(string_var_name[0]) || string_var_name[0] == '-') {
+            throw KeywordError("Malformed variable name in the 'length' keyword");
+        }
+
+        var_id = vars.addNewVariable(string_var_name);
+    } else {
+        compare_size.setAttr("length value", string_var_name, vars, "length", 10, true);
+    }
+}
+
+MatchStatus
+LengthKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    auto part = Singleton::Consume<I_Environment>::by<KeywordComp>()->get<Buffer>(static_cast<string>(ctx));
+
+    if (!part.ok()) return MatchStatus::NoMatchFinal;
+
+    uint offset = is_relative ? prev->getOffset(ctx) : 0;
+    uint size = (*part).size();
+
+    if (offset <= size) {
+        if (mode.isSet(Mode::EXACT)) {
+            if (size - offset == static_cast<uint>(compare_size.evalAttr(prev))) return runNext(prev);
+        } else if (mode.isSet(Mode::MIN)) {
+            if (size - offset >= static_cast<uint>(compare_size.evalAttr(prev))) return runNext(prev);
+        } else if (mode.isSet(Mode::MAX)) {
+            if (size - offset <= static_cast<uint>(compare_size.evalAttr(prev))) return runNext(prev);
+        } else {
+            VariableRuntimeState new_length_var(prev, var_id, size-offset);
+            return runNext(&new_length_var);
+        }
+    }
+
+    // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match
+    return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch;
+}
+
+unique_ptr<SingleKeyword>
+genLengthKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<LengthKeyword>(attr, known_vars);
+}

components/utils/keywords/no_match_keyword.cc

@@ -0,0 +1,29 @@
+#include "single_keyword.h"
+#include "table_opaque.h"
+#include "debug.h"
+
+#include <map>
+#include <strings.h>
+
+#include "cereal/types/set.hpp"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class NoMatchKeyword : public SingleKeyword
+{
+public:
+    explicit NoMatchKeyword(const vector<KeywordAttr> &attr, VariablesMapping &)
+    {
+        if (!attr.empty()) throw KeywordError("The 'no_match' keyword doesn't take attributes");
+    }
+
+    MatchStatus isMatch(const I_KeywordRuntimeState *) const override { return MatchStatus::NoMatchFinal; }
+};
+
+unique_ptr<SingleKeyword>
+genNoMatchKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<NoMatchKeyword>(attr, known_vars);
+}

components/utils/keywords/pcre_keyword.cc

@@ -0,0 +1,372 @@
+#include "single_keyword.h"
+
+#include <algorithm>
+
+#define PCRE2_CODE_UNIT_WIDTH 8
+#include <pcre2.h>
+
+#include "output.h"
+#include "debug.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class PCREKeyword : public SingleKeyword
+{
+public:
+    explicit PCREKeyword(const vector<KeywordAttr> &attr, const VariablesMapping &known_vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState *prev) const override;
+
+private:
+    void
+    setOffset(const KeywordAttr &attr, const VariablesMapping &vars)
+    {
+        offset.setAttr(attr, vars, "pcre");
+    }
+
+    void
+    setDepth(const KeywordAttr &attr, const VariablesMapping &vars)
+    {
+        depth.setAttr(attr, vars, "pcre");
+    }
+
+    void
+    setRelative(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_relative.setAttr(attr, "pcre");
+    }
+
+    void
+    setCaseInsensitive(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        is_case_insensitive.setAttr(attr, "pcre");
+    }
+
+    void
+    setContext(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        ctx.setAttr(attr, "pcre");
+    }
+
+    string parseString(const string &str);
+    pair<string, string> findExprInStr(const string &str, size_t start, size_t end);
+    void parseOptions(const string &str);
+    void compilePCRE(const string &str);
+
+    pair<uint, uint> getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const;
+
+    bool
+    isConstant() const
+    {
+        return !is_relative && offset.isConstant() && depth.isConstant();
+    }
+
+    class PCREDelete
+    {
+    public:
+        void
+        operator()(pcre2_code *ptr)
+        {
+            pcre2_code_free(ptr);
+        }
+    };
+    unique_ptr<pcre2_code, PCREDelete> pcre_machine;
+
+    class PCREResultDelete
+    {
+    public:
+        void
+        operator()(pcre2_match_data *ptr)
+        {
+            pcre2_match_data_free(ptr);
+        }
+    };
+    unique_ptr<pcre2_match_data, PCREResultDelete> pcre_result;
+
+    NumericAttr offset;
+    NumericAttr depth;
+    BoolAttr is_negative;
+    BoolAttr is_relative;
+
+    BoolAttr is_case_insensitive;
+    BoolAttr is_multiline;
+    BoolAttr is_dotall;
+    BoolAttr is_extended;
+    BoolAttr is_dollar_endonly;
+    BoolAttr is_anchor;
+    BoolAttr is_ungreedy;
+
+    CtxAttr ctx;
+
+    string pcre_expr;
+
+    static const map<string, void(PCREKeyword::*)(const KeywordAttr &, const VariablesMapping &)> setops;
+};
+
+const map<string, void(PCREKeyword::*)(const KeywordAttr&, const VariablesMapping&)> PCREKeyword::setops = {
+    { "relative", &PCREKeyword::setRelative        },
+    { "offset",   &PCREKeyword::setOffset          },
+    { "depth",    &PCREKeyword::setDepth           },
+    { "nocase",   &PCREKeyword::setCaseInsensitive },
+    { "part",     &PCREKeyword::setContext         },
+};
+
+PCREKeyword::PCREKeyword(const vector<KeywordAttr> &attrs, const VariablesMapping &known_vars)
+        :
+    offset(),
+    depth()
+{
+    auto &expr_param = attrs[0].getParams();
+    if (expr_param.size() != 1) throw KeywordError("More than one element in the 'pcre' keyword pattern");
+    auto expr = parseString(expr_param[0]);
+    dbgDebug(D_KEYWORD) << "Creating a new 'pcre' expression: " << expr;
+
+    for (uint i = 1; i<attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'pcre' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i], known_vars);
+    }
+
+    compilePCRE(expr);
+}
+
+string
+PCREKeyword::parseString(const string &str)
+{
+    size_t start_offset = 0, end_offset = str.size();
+
+    if (start_offset<end_offset && str[start_offset]=='!') {
+        is_negative.setAttr("pcre", "negative");
+        start_offset++;
+    }
+
+    if (start_offset+1>=end_offset || str[start_offset]!='"' || str[end_offset-1]!='"') {
+        throw KeywordError("The 'pcre' expression should be enclosed in quotation marks");
+    }
+    start_offset++;
+    end_offset--;
+
+    string expr, options;
+    tie(expr, options) = findExprInStr(str, start_offset, end_offset);
+
+    parseOptions(options);
+
+    return expr;
+}
+
+pair<string, string>
+PCREKeyword::findExprInStr(const string &str, size_t start, size_t end)
+{
+    if (start>=end) throw KeywordError("The 'pcre' string is empty");
+
+    // There are two way to write the regular expression:
+    // Either between '/' charecters: "/regexp/"  (this is the default delimiter)
+    // Or use 'm' to set a delimiter: "mDregexpD" (here 'D' is used as the delimiter)
+    // The switch will set the parameter 'start' so the 'str[start]' is the first delimiter.
+    switch (str[start]) {
+        case '/': {
+            break;
+        }
+        case 'm': {
+            start++;
+            if (start>=end) {
+                throw KeywordError("Failed to detect a delimiter in the 'pcre' keyword regular expression");
+            }
+            break;
+        }
+        default:
+            KeywordError("Bad start for the 'pcre' regular expression");
+    }
+
+    size_t expr_end = str.find_last_of(str[start], end-1);
+    start++;
+    if (expr_end<=start) throw KeywordError("The 'pcre' regular expression is empty");
+
+    auto options_start = expr_end+1;
+    return make_pair(str.substr(start, expr_end-start), str.substr(options_start, end-options_start));
+}
+
+void
+PCREKeyword::parseOptions(const string &options)
+{
+    for (auto ch : options) {
+        switch (ch) {
+            case 'i': {
+                is_case_insensitive.setAttr("pcre", "nocase");
+                break;
+            }
+            case 'R': {
+                is_relative.setAttr("pcre", "relative");
+                break;
+            }
+            case 'm': {
+                is_multiline.setAttr("pcre", "multiline");
+                break;
+            }
+            case 's': {
+                is_dotall.setAttr("pcre", "dotall");
+                break;
+            }
+            case 'x': {
+                is_extended.setAttr("pcre", "extended");
+                break;
+            }
+            case 'E': {
+                is_dollar_endonly.setAttr("pcre", "dollar_endonly");
+                break;
+            }
+            case 'A': {
+                is_anchor.setAttr("pcre", "anchor");
+                break;
+            }
+            case 'G': {
+                is_ungreedy.setAttr("pcre", "ungreedy");
+                break;
+            }
+            default:
+                throw KeywordError("Unknown option " +  dumpHexChar(ch) + " in the 'pcre' keyword");
+        }
+    }
+}
+
+void
+PCREKeyword::compilePCRE(const string &expr)
+{
+    uint32_t options = PCRE2_NO_AUTO_CAPTURE;
+    if (is_case_insensitive) options |= PCRE2_CASELESS;
+    if (is_multiline)        options |= PCRE2_MULTILINE;
+    if (is_dotall)           options |= PCRE2_DOTALL;
+    if (is_extended)         options |= PCRE2_EXTENDED;
+    if (is_dollar_endonly)   options |= PCRE2_DOLLAR_ENDONLY;
+    if (is_anchor)           options |= PCRE2_ANCHORED;
+    if (is_ungreedy)         options |= PCRE2_UNGREEDY;
+
+    int error;
+    PCRE2_SIZE error_offset;
+    auto pattern = reinterpret_cast<PCRE2_SPTR>(expr.c_str());
+    pcre_machine.reset(pcre2_compile(pattern, expr.size(), options, &error, &error_offset, nullptr));
+    if (pcre_machine == nullptr) {
+        vector<u_char> msg;
+        msg.reserve(128);
+        pcre2_get_error_message(error, msg.data(), msg.capacity());
+
+        throw KeywordError(
+            "Failed to compile the 'pcre' at offset "
+            + to_string(error_offset)
+            + " with error: "
+            + reinterpret_cast<char *>(msg.data())
+        );
+    }
+
+    pcre_result.reset(pcre2_match_data_create_from_pattern(pcre_machine.get(), nullptr));
+    if (pcre_result == nullptr) {
+        throw KeywordError("Failed to allocate PCRE results container");
+    }
+
+    pcre_expr = expr;
+}
+
+static uint
+addOffset(uint offset, int add)
+{
+    if (add<0 && offset<static_cast<uint>(-add)) return 0;
+    return offset + add;
+}
+
+pair<uint, uint>
+PCREKeyword::getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const
+{
+    uint keyword_offset = is_relative?prev->getOffset(ctx):0;
+    uint start_offset = addOffset(keyword_offset, offset.evalAttr(prev));
+
+    if (start_offset>=buf_size) return make_pair(0, 0);
+
+    uint length = buf_size-start_offset;
+
+    if (depth.isSet()) {
+        length = min(length, static_cast<uint>(depth.evalAttr(prev)));
+    }
+
+    return make_pair(start_offset, length);
+}
+
+MatchStatus
+PCREKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    dbgAssert(pcre_machine!=nullptr) << "Trying to run on an uninitialized keyword 'pcre'";
+
+    auto part = Singleton::Consume<I_Environment>::by<KeywordComp>()->get<Buffer>(static_cast<string>(ctx));
+
+    if (!part.ok()) {
+        if (is_negative) {
+            return runNext(prev);
+        }
+        return MatchStatus::NoMatchFinal;
+    }
+
+    uint offset, length;
+    tie(offset, length) = getStartOffsetAndLength((*part).size(), prev);
+    auto buf = (*part).getPtr(offset, length);
+
+    if (!buf.ok()) {
+        dbgTrace(D_KEYWORD) << "Could not get the buffer for the 'pcre' keyword";
+        return MatchStatus::NoMatchFinal;
+    }
+    const unsigned char *ptr = *buf;
+
+    bool match_found = false;
+    uint buf_offset_found;
+    for (uint buf_pos = 0; buf_pos<length; buf_pos = buf_offset_found) {
+        dbgDebug(D_KEYWORD) << "Looking for expression: " << pcre_expr;
+        dbgTrace(D_KEYWORD) << "Running pcre_exec for expression: " << ptr;
+        int result = pcre2_match(
+            pcre_machine.get(),
+            ptr,
+            length,
+            buf_pos,
+            0,
+            pcre_result.get(),
+            nullptr
+        );
+
+        if (result<0) {
+            // No match (possiblely due to an error)
+            dbgDebug(D_KEYWORD) << "No match, possiblely due to an error in 'pcre_exec'";
+            break;
+        } else {
+            dbgDebug(D_KEYWORD) << "Match found";
+        }
+
+        if (is_negative) {
+            return isConstant()?MatchStatus::NoMatchFinal:MatchStatus::NoMatch;
+        }
+        match_found = true;
+        buf_offset_found = pcre2_get_ovector_pointer(pcre_result.get())[0];
+        OffsetRuntimeState new_offset(prev, ctx, offset+buf_offset_found);
+        auto next_keyword_result = runNext(&new_offset);
+        if (next_keyword_result!=MatchStatus::NoMatch) return next_keyword_result;
+        if (buf_offset_found<=buf_pos) buf_offset_found = buf_pos+1; // Deal with empty matches
+    }
+
+    // No matchs is a success for negative keywords
+    if (is_negative && !match_found) {
+        return runNext(prev);
+    }
+
+    // If there were no matchs and the keyword is an effected by other keywords, then we know that the rule won't match
+    if (isConstant() && !match_found) {
+        return MatchStatus::NoMatchFinal;
+    }
+
+    return MatchStatus::NoMatch;
+}
+
+unique_ptr<SingleKeyword>
+genPCREKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<PCREKeyword>(attr, known_vars);
+}

components/utils/keywords/sentinel_runtime_state.h

@@ -0,0 +1,13 @@
+#ifndef ___SENTINEL_RUNTIME_STATE_H__
+#define ___SENTINEL_RUNTIME_STATE_H__
+
+#include "single_keyword.h"
+
+class SentinelRuntimeState : public I_KeywordRuntimeState
+{
+public:
+    uint getOffset(const std::string &) const override;
+    uint getVariable(uint) const override;
+};
+
+#endif //  ___SENTINEL_RUNTIME_STATE_H__

components/utils/keywords/single_keyword.cc

@@ -0,0 +1,276 @@
+#include "single_keyword.h"
+
+#include <algorithm>
+
+using namespace std;
+
+void
+SingleKeyword::appendKeyword(unique_ptr<SingleKeyword> &&_next)
+{
+    if (next==nullptr) {
+        next = move(_next);
+    } else {
+        next->appendKeyword(move(_next));
+    }
+}
+
+MatchStatus
+SingleKeyword::runNext(const I_KeywordRuntimeState *curr) const
+{
+    if (next==nullptr) {
+        return MatchStatus::Match;
+    }
+    return next->isMatch(curr);
+}
+
+OffsetRuntimeState::OffsetRuntimeState(
+    const I_KeywordRuntimeState *_p,
+    const string &_ctx,
+    uint _offset)
+        :
+    prev(_p),
+    ctx(_ctx),
+    offset(_offset)
+{
+}
+
+uint
+OffsetRuntimeState::getOffset(const string &requested_ctx) const
+{
+    if (ctx==requested_ctx) return offset;
+    return prev->getOffset(requested_ctx);
+}
+
+uint
+OffsetRuntimeState::getVariable(uint requested_var_id) const
+{
+    return prev->getVariable(requested_var_id);
+}
+
+VariableRuntimeState::VariableRuntimeState(
+    const I_KeywordRuntimeState *_p,
+    uint _var_id,
+    uint _val)
+        :
+    prev(_p),
+    var_id(_var_id),
+    value(_val)
+{
+}
+
+uint
+VariableRuntimeState::getOffset(const string &requested_ctx) const
+{
+    return prev->getOffset(requested_ctx);
+}
+
+uint
+VariableRuntimeState::getVariable(uint requested_var_id) const
+{
+    if (var_id==requested_var_id) return value;
+    return prev->getVariable(requested_var_id);
+}
+
+uint
+VariablesMapping::addNewVariable(const string &param)
+{
+    auto iter = mapping.find(param);
+    if (iter==mapping.end()) {
+        mapping[param] = mapping.size();
+    }
+    return mapping[param];
+}
+
+Maybe<uint>
+VariablesMapping::getVariableId(const string &param) const
+{
+    auto iter = mapping.find(param);
+    if (iter==mapping.end()) {
+        return genError(string("Unknown parameter ")+param);
+    }
+    return iter->second;
+}
+
+void
+NumericAttr::setAttr(
+    const KeywordAttr &attr,
+    const VariablesMapping &known_vars,
+    const string &keyword_name,
+    const uint base,
+    bool is_unsigned_val)
+{
+    auto &vec = attr.getParams();
+    if (vec.size()!= 2) {
+        throw KeywordError("Malformed " + attr.getAttrName() + "' in the '" + keyword_name + "' keyword");
+    }
+
+    setAttr(attr.getAttrName(), vec[1], known_vars, keyword_name, base, is_unsigned_val);
+}
+
+void
+NumericAttr::setAttr(
+    const string &attr_name,
+    const string &param,
+    const VariablesMapping &known_vars,
+    const string &keyword_name,
+    const uint base,
+    bool is_unsigned_val)
+{
+    if (isSet()) {
+        throw KeywordError("Double definition of the '" + attr_name + "' in the '" + keyword_name + "' keyword");
+    }
+
+    if (is_unsigned_val && param[0]=='-') {
+        throw KeywordError(
+            "Negative constant '" +
+            param +
+            "' in the '" +
+            attr_name +
+            "' in the '" +
+            keyword_name +
+            "' keyword"
+        );
+    }
+
+    if (isdigit(param[0]) || param[0] == '-') {
+        status = Status::Const;
+        try {
+            size_t idx;
+            val = stol(param, &idx, base);
+            if (idx != param.length()) throw invalid_argument("");
+        }
+        catch (...) {
+            throw KeywordError(
+                "Malformed constant '" +
+                param +
+                "' in the '" +
+                attr_name +
+                "' in the '" +
+                keyword_name +
+                "' keyword"
+            );
+        }
+    } else {
+        status = Status::Var;
+        val = known_vars.getVariableId(param).unpack<KeywordError>(
+            "In " + keyword_name +
+            " in " + attr_name + ": "
+        );
+    }
+}
+
+int
+NumericAttr::evalAttr(const I_KeywordRuntimeState *prev) const
+{
+    if (status==Status::Var) {
+        return prev->getVariable(val);
+    }
+    return val;
+}
+
+void
+BoolAttr::setAttr(const KeywordAttr &attr, const string &keyword_name)
+{
+    if (attr.getParams().size()!=1) {
+        throw KeywordError("Malformed " + attr.getAttrName() + "' in the '" + keyword_name + "' keyword");
+    }
+
+    val = true;
+}
+
+void
+BoolAttr::setAttr(const string &keyword_name, const string &attr_name)
+{
+    if (val) throw KeywordError("Double definition of the '" + attr_name + "' in the '" + keyword_name + "' keyword");
+
+    val = true;
+}
+
+void
+CtxAttr::setAttr(const KeywordAttr &attr, const string &keyword_name)
+{
+    if (is_set) throw KeywordError("Double definition of the 'part' in the '" + keyword_name + "' keyword");
+    is_set = true;
+    auto vec = attr.getParams();
+    if (vec.size()!=2) throw KeywordError("Malformed 'part' in the '" + keyword_name + "' keyword");
+    ctx = vec[1];
+}
+
+const map<string, ComparisonAttr::CompId> ComparisonAttr::name_to_operator {
+    { "=",          CompId::EQUAL                 },
+    { "!=",         CompId::NOT_EQUAL             },
+    { "<",          CompId::LESS_THAN             },
+    { ">",          CompId::GREATER_THAN          },
+    { "<=",         CompId::LESS_THAN_OR_EQUAL    },
+    { ">=",         CompId::GREATER_THAN_OR_EQUAL }
+};
+
+Maybe<ComparisonAttr::CompId>
+ComparisonAttr::getComparisonByName(const string &name)
+{
+    auto iter = name_to_operator.find(name);
+    if (iter == name_to_operator.end()) {
+        return genError("Could not find the operator: " + name);
+    }
+    return iter->second;
+}
+
+void
+ComparisonAttr::setAttr(const string &param, const string &keyword_name)
+{
+    if (isSet()) {
+        throw KeywordError("Double definition of the comparison opearator in the '" + keyword_name + "' keyword");
+    }
+    is_set = true;
+    comp_val = getComparisonByName(param).unpack<KeywordError>(
+        "Unknown comparison operator in the '" + keyword_name  + "' keyword: "
+    );
+}
+
+bool
+ComparisonAttr::operator()(int first_val, int second_val) const
+{
+    switch (comp_val) {
+        case ComparisonAttr::CompId::EQUAL: {
+            return first_val == second_val;
+        }
+        case ComparisonAttr::CompId::NOT_EQUAL: {
+            return first_val != second_val;
+        }
+        case ComparisonAttr::CompId::LESS_THAN: {
+            return first_val < second_val;
+        }
+        case ComparisonAttr::CompId::GREATER_THAN: {
+            return first_val > second_val;
+        }
+        case ComparisonAttr::CompId::LESS_THAN_OR_EQUAL: {
+            return first_val <= second_val;
+        }
+        case ComparisonAttr::CompId::GREATER_THAN_OR_EQUAL: {
+            return first_val >= second_val;
+        }
+    }
+    dbgAssert(false) << "ComparisonAttr::operator found an invalid comparison operator";
+    return false;
+}
+
+using InitFunc = unique_ptr<SingleKeyword>(*)(const vector<KeywordAttr> &, VariablesMapping &);
+
+const map<string, InitFunc> initializers = {
+    {"data",          genDataKeyword        },
+    {"pcre",          genPCREKeyword        },
+    {"length",        genLengthKeyword      },
+    {"byte_extract",  genByteExtractKeyword },
+    {"compare",       genCompareKeyword     },
+    {"stateop",       genStateopKeyword     },
+    {"no_match",      genNoMatchKeyword     },
+    {"jump",          genJumpKeyword        }
+};
+
+unique_ptr<SingleKeyword>
+getKeywordByName(const KeywordParsed &keyword, VariablesMapping &known_vars)
+{
+    auto iter = initializers.find(keyword.getName());
+    if (iter==initializers.end()) throw KeywordError(keyword.getName() + " - unknown keyword type");
+    return iter->second(keyword.getAttr(), known_vars);
+}

components/utils/keywords/single_keyword.h

@@ -0,0 +1,300 @@
+#ifndef ___SINGLE_KEYWORD_H__
+#define ___SINGLE_KEYWORD_H__
+
+#include <map>
+#include <string>
+#include <vector>
+#include <memory>
+
+#include "buffer.h"
+#include "keyword_comp.h"
+#include "debug.h"
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+enum class MatchStatus { Match, NoMatch, NoMatchFinal };
+
+class KeywordError
+{
+public:
+    KeywordError(const std::string &str) : err(str)
+    {
+    }
+
+    const std::string &
+    getErr() const
+    {
+        return err;
+    }
+
+private:
+    std::string err;
+};
+
+class KeywordAttr
+{
+public:
+    KeywordAttr(const std::string &str);
+
+    const std::string&
+    getAttrName() const
+    {
+        return params[0];
+    }
+
+    const std::vector<std::string> &
+    getParams() const
+    {
+        return params;
+    }
+
+private:
+    std::vector<std::string> params;
+};
+
+class KeywordParsed
+{
+public:
+    KeywordParsed(const std::string &keyword);
+
+    const std::string &
+    getName() const
+    {
+        return name;
+    }
+
+    const std::vector<KeywordAttr> &
+    getAttr() const
+    {
+        return attr;
+    }
+
+private:
+    std::string name;
+    std::vector<KeywordAttr> attr;
+};
+
+class I_KeywordRuntimeState
+{
+public:
+    virtual uint getOffset(const std::string &ctx) const = 0;
+    virtual uint getVariable(uint requested_var_id) const = 0;
+protected:
+    virtual ~I_KeywordRuntimeState()
+    {
+    }
+};
+
+class OffsetRuntimeState : public I_KeywordRuntimeState
+{
+public:
+    OffsetRuntimeState(const I_KeywordRuntimeState *prev, const std::string &ctx, uint offset);
+    virtual ~OffsetRuntimeState()
+    {
+    }
+    uint getOffset(const std::string &requested_ctx_id) const override;
+    uint getVariable(uint requested_var_id) const override;
+
+private:
+    const I_KeywordRuntimeState *prev;
+    std::string ctx;
+    uint offset;
+};
+
+class VariableRuntimeState : public I_KeywordRuntimeState
+{
+public:
+    VariableRuntimeState(const I_KeywordRuntimeState *prev, uint var_id, uint val);
+    virtual ~VariableRuntimeState()
+    {
+    }
+
+    uint getOffset(const std::string &requested_ctx_id) const override;
+    uint getVariable(uint requested_var_id) const override;
+
+private:
+    const I_KeywordRuntimeState *prev;
+    uint var_id;
+    uint value;
+};
+
+class VariablesMapping
+{
+public:
+    uint addNewVariable(const std::string &name);
+    Maybe<uint> getVariableId(const std::string &name) const;
+
+private:
+    std::map<std::string, uint> mapping;
+};
+
+class NumericAttr
+{
+    enum class Status { Unset, Const, Var };
+public:
+    void setAttr(
+        const KeywordAttr &attr,
+        const VariablesMapping &known_vars,
+        const std::string &keyword_name,
+        const uint base = 10,
+        bool is_unsigned_val = false);
+
+    void setAttr(
+        const std::string &attr_name,
+        const std::string &param,
+        const VariablesMapping &known_vars,
+        const std::string &keyword_name,
+        const uint base = 10,
+        bool is_unsigned_val = false);
+
+    int evalAttr(const I_KeywordRuntimeState *prev) const;
+
+    bool
+    isConstant() const
+    {
+        return status!=Status::Var;
+    }
+
+    bool
+    isSet() const
+    {
+        return status!=Status::Unset;
+    }
+
+private:
+    Status status = Status::Unset;
+    int val = 0;
+};
+
+class BoolAttr
+{
+public:
+    void setAttr(const KeywordAttr &attr, const std::string &keyword_name);
+    void setAttr(const std::string &keyword_name, const std::string &attr_name);
+
+    operator bool() const
+    {
+        return val;
+    }
+
+private:
+    bool val = false;
+};
+
+class CtxAttr
+{
+public:
+    void setAttr(const KeywordAttr &attr, const std::string &keyword_name);
+
+    operator std::string() const
+    {
+        if (!is_set) {
+            auto env = Singleton::Consume<I_Environment>::by<KeywordComp>();
+            auto default_ctx = env->get<std::string>(I_KeywordsRule::getKeywordsRuleTag());
+            if (default_ctx.ok()) return *default_ctx;
+            dbgError(D_KEYWORD) << "Running keyword rule without specific context and without default";
+            return "Missing Default Context";
+        }
+        return ctx;
+    }
+
+private:
+    std::string ctx;
+    bool is_set = false;
+};
+
+class ComparisonAttr
+{
+public:
+    enum class CompId
+    {
+        EQUAL,
+        NOT_EQUAL,
+        LESS_THAN,
+        GREATER_THAN,
+        LESS_THAN_OR_EQUAL,
+        GREATER_THAN_OR_EQUAL
+    };
+
+    void setAttr(const std::string &param, const std::string &keyword_name);
+    bool operator()(int first_val, int second_val) const;
+
+    bool
+    isSet() const
+    {
+        return is_set;
+    }
+
+private:
+    Maybe<CompId> getComparisonByName(const std::string &name);
+
+    const static std::map<std::string, CompId> name_to_operator;
+    bool is_set = false;
+    CompId comp_val;
+};
+
+class SingleKeyword
+{
+public:
+    SingleKeyword()
+    {
+    }
+
+    virtual ~SingleKeyword()
+    {
+    }
+
+    MatchStatus runNext(const I_KeywordRuntimeState *curr) const;
+    void appendKeyword(std::unique_ptr<SingleKeyword> &&_next);
+    virtual MatchStatus isMatch(const I_KeywordRuntimeState *prev) const = 0;
+
+private:
+    std::unique_ptr<SingleKeyword> next;
+};
+
+std::unique_ptr<SingleKeyword> genDataKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genPCREKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genLengthKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genByteExtractKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genCompareKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genStateopKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genNoMatchKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> genJumpKeyword(
+    const std::vector<KeywordAttr> &attr,
+    VariablesMapping &vars
+);
+
+std::unique_ptr<SingleKeyword> getKeywordByName(
+    const KeywordParsed &parsed_data,
+    VariablesMapping &vars
+);
+
+#endif // ___SINGLE_KEYWORD_H__

components/utils/keywords/stateop_keyword.cc

@@ -0,0 +1,145 @@
+#include "single_keyword.h"
+#include "table_opaque.h"
+#include "debug.h"
+#include "flags.h"
+
+#include <map>
+#include <strings.h>
+
+#include "cereal/types/set.hpp"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_KEYWORD);
+
+class StateopKeyword : public SingleKeyword
+{
+public:
+    explicit StateopKeyword(const vector<KeywordAttr> &attr, VariablesMapping &vars);
+    MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override;
+
+private:
+    enum class Operation { ISSET, SET, UNSET, COUNT };
+    using OpFlags = Flags<Operation>;
+
+    void
+    setState(const KeywordAttr &attr, const VariablesMapping &)
+    {
+        auto &var_name_param = attr.getParams();
+
+        if (var_name_param.size() != 2) {
+            throw KeywordError("More than one element in the state name in the 'stateop' keyword");
+        }
+
+        var_name = var_name_param[1];
+    }
+
+    void
+    setTesting(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation");
+        mode.setFlag(Operation::ISSET);
+    }
+
+    void
+    setSetting(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation");
+        mode.setFlag(Operation::SET);
+    }
+
+    void
+    setUnsetting(const KeywordAttr &, const VariablesMapping &)
+    {
+        if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation");
+        mode.setFlag(Operation::UNSET);
+    }
+
+    string  var_name;
+    OpFlags mode;
+
+    static const map<string, void(StateopKeyword::*)(const KeywordAttr &, const VariablesMapping &)> setops;
+};
+
+const map<string, void(StateopKeyword::*)(const KeywordAttr &, const VariablesMapping &)> StateopKeyword::setops = {
+    { "isset", &StateopKeyword::setTesting   },
+    { "set",   &StateopKeyword::setSetting   },
+    { "unset", &StateopKeyword::setUnsetting },
+    { "state", &StateopKeyword::setState     }
+};
+
+StateopKeyword::StateopKeyword(const vector<KeywordAttr> &attrs, VariablesMapping &vars)
+{
+    if (attrs.size() != 2) throw KeywordError("Invalid number of attributes in the 'stateop' keyword");
+    
+    for (uint i = 0; i < attrs.size(); i++) {
+        auto curr = setops.find(attrs[i].getAttrName());
+        if (curr == setops.end()) {
+            throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'stateop' keyword");
+        }
+        auto set_func = curr->second;
+        (this->*set_func)(attrs[i], vars);
+    }
+
+    if (var_name == "" || mode.empty()) {
+        throw KeywordError("Bad 'stateop' attribute configuration");
+    }
+}
+
+class KeywordStateop : public TableOpaqueSerialize<KeywordStateop>
+{
+public:
+    KeywordStateop() : TableOpaqueSerialize<KeywordStateop>(this) {}
+
+    bool hasVariable(const string &state) { return states.find(state) != states.end(); }
+    void addVariable(const string &state) { states.insert(state); }
+    void removeVariable(const string &state) { states.erase(state); }
+
+    // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists
+    template <typename T>
+    void
+    serialize(T &ar, uint32_t)
+    {
+        ar(states);
+    }
+
+    static std::string name() { return "KeywordStateop"; }
+    static std::unique_ptr<TableOpaqueBase> prototype() { return std::make_unique<KeywordStateop>(); }
+    static uint currVer() { return 0; }
+    static uint minVer() { return 0; }
+    // LCOV_EXCL_STOP
+
+private:
+    set<string> states;
+};
+
+MatchStatus
+StateopKeyword::isMatch(const I_KeywordRuntimeState *prev) const
+{
+    auto table = Singleton::Consume<I_Table>::by<KeywordComp>();
+
+    if (mode.isSet(Operation::ISSET)) {
+        if (!table->hasState<KeywordStateop>()) return MatchStatus::NoMatchFinal;
+        auto &state = table->getState<KeywordStateop>();
+        if (state.hasVariable(var_name)) return runNext(prev);
+        else return MatchStatus::NoMatchFinal;
+    } else if (mode.isSet(Operation::SET)) {
+        if (!table->hasState<KeywordStateop>()) table->createState<KeywordStateop>();
+        table->getState<KeywordStateop>().addVariable(var_name);
+        return runNext(prev);
+    } else if (mode.isSet(Operation::UNSET)) {
+        if (table->hasState<KeywordStateop>()) table->getState<KeywordStateop>().removeVariable(var_name);
+        return runNext(prev);
+    } else {
+        dbgAssert(false) << "Impossible 'stateop' keyword without operation";
+    }
+
+    // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match
+    return MatchStatus::NoMatchFinal;
+}
+
+unique_ptr<SingleKeyword>
+genStateopKeyword(const vector<KeywordAttr> &attr, VariablesMapping &known_vars)
+{
+    return make_unique<StateopKeyword>(attr, known_vars);
+}

core/agent_core_utilities/agent_core_utilities.cc

@@ -68,6 +68,19 @@ makeDir(const string &path, mode_t permission)
     return true;
 }
 
+/// @brief Get basename of a path
+/// @param path path to a file
+/// @return base file name
+string
+getFileName(const string &path)
+{
+    dbgFlow(D_INFRA_UTILS) << "Trying to extract file name from path: " << path;
+    size_t pos = path.rfind("/");
+    if (pos != string::npos) return path.substr(pos+1, path.length() - pos);
+
+    return path;
+}
+
 bool
 makeDirRecursive(const string &path, mode_t permission)
 {

core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc

@@ -77,6 +77,7 @@ TEST_F(AgentCoreUtilUT, directoryTest)
     EXPECT_TRUE(NGEN::Filesystem::deleteDirectory("/tmp/1", true));
     EXPECT_FALSE(NGEN::Filesystem::exists("/tmp/1"));
 }
+
 TEST_F(AgentCoreUtilUT, printTest)
 {
     EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(0), "0 Bytes");
@@ -96,3 +97,9 @@ TEST_F(AgentCoreUtilUT, printTest)
     EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(1000*gigabyte), "1000.00 GB");
     EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(1024*gigabyte), "1024.00 GB");
 }
+
+
+TEST_F(AgentCoreUtilUT, fileBasenameTest)
+{
+    EXPECT_EQ(NGEN::Filesystem::getFileName("/test/base/file/name"), "name");
+}

core/config/config.cc

@@ -150,6 +150,8 @@ private:
     bool commitFailure(const string &error);
     bool reloadConfigurationImpl(const string &version, bool is_async);
     void reloadConfigurationContinuesWrapper(const string &version, uint id);
+    vector<string> fillMultiTenantConfigFiles(const map<string, set<string>> &tenants);
+    vector<string> fillMultiTenantExpectedConfigFiles(const map<string, set<string>> &tenants);
 
     string
     getActiveTenant() const
@@ -274,7 +276,7 @@ private:
     map<string, set<ConfigFileType>> expected_configuration_files;
     set<string> config_file_paths;
 
-    I_TenantManager *tenant_mananger = nullptr;
+    I_TenantManager *tenant_manager = nullptr;
 
     vector<ConfigCb> configuration_prepare_cbs;
     vector<ConfigCb> configuration_commit_cbs;
@@ -322,7 +324,7 @@ void
 ConfigComponent::Impl::init()
 {
     reloadFileSystemPaths();
-    tenant_mananger = Singleton::Consume<I_TenantManager>::by<ConfigComponent>();
+    tenant_manager = Singleton::Consume<I_TenantManager>::by<ConfigComponent>();
 
     if (!Singleton::exists<I_MainLoop>()) return;
     auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();
@@ -338,7 +340,7 @@ ConfigComponent::Impl::init()
 
     mainloop->addRecurringRoutine(
         I_MainLoop::RoutineType::System,
-        tenant_mananger->getTimeoutVal(),
+        tenant_manager->getTimeoutVal(),
         [this] () { clearOldTenants(); },
         "Config comp old tenant cleanup"
     );
@@ -681,7 +683,7 @@ bool
 ConfigComponent::Impl::areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const
 {
     return (tenant_profile.getTenantId() == default_tenant_id && tenant_profile.getProfileId() == default_profile_id)
-        || tenant_mananger->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getProfileId());
+        || tenant_manager->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getProfileId());
 }
 
 void
@@ -817,6 +819,45 @@ ConfigComponent::Impl::commitFailure(const string &error)
     return false;
 }
 
+vector<string>
+ConfigComponent::Impl::fillMultiTenantConfigFiles(const map<string, set<string>> &active_tenants)
+{
+    vector<string> files;
+    for (const auto &tenant_profiles : active_tenants) {
+        const string &tenant = tenant_profiles.first;
+        const set<string> &profile_ids = tenant_profiles.second;
+        for (const auto &profile_id : profile_ids) {
+            string settings_path =
+                config_directory_path + "tenant_" + tenant + "_profile_" + profile_id + "_settings.json";
+            files.push_back(settings_path);
+        }
+    }
+    return files;
+}
+
+vector<string>
+ConfigComponent::Impl::fillMultiTenantExpectedConfigFiles(const map<string, set<string>> &active_tenants)
+{
+    vector<string> files;
+    for (const auto &config_file : expected_configuration_files) {
+        for (const auto &type : config_file.second) {
+            if (type == ConfigFileType::RawData) continue;
+            auto global_path = getPolicyConfigPath(config_file.first, type);
+            auto it = find(files.begin(), files.end(), global_path);
+            if (it == files.end()) files.push_back(global_path);
+            for (const pair<string, set<string>> &tenant_profiles : active_tenants) {
+                const string &tenant = tenant_profiles.first;
+                const set<string> &profile_ids = tenant_profiles.second;
+                for (const auto &profile_id : profile_ids) {
+                    auto tenant_path = getPolicyConfigPath(config_file.first, type, tenant, profile_id);
+                    files.push_back(tenant_path);
+                }
+            }
+        }
+    }
+    return files;
+}
+
 bool
 ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_async)
 {
@@ -831,38 +872,20 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
         files.emplace(fullpath, make_shared<ifstream>(fullpath));
     }
 
-    const auto &active_tenants = tenant_mananger ? tenant_mananger->fetchAllActiveTenants() : vector<string>();
+    map<string, set<string>> active_tenants =
+        tenant_manager ? tenant_manager->fetchActiveTenantsAndProfiles() : map<string, set<string>>();
 
     dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size();
 
-    for (const auto &config_file : expected_configuration_files) {
-        for (const auto &type : config_file.second) {
-            if (type == ConfigFileType::RawData) continue;
-            auto global_path = getPolicyConfigPath(config_file.first, type);
-            if (files.find(global_path) == files.end()) {
-                files.emplace(global_path, make_shared<ifstream>(global_path));
-            }
-
-            for (auto &tenant : active_tenants) {
-                const vector<string> &profile_ids =
-                    tenant_mananger ? tenant_mananger->fetchProfileIds(tenant) : vector<string>();
-                for (auto &profile_id : profile_ids) {
-                    auto tenant_path = getPolicyConfigPath(config_file.first, type, tenant, profile_id);
-                    files.emplace(tenant_path, make_shared<ifstream>(tenant_path));
-                }
-            }
-        }
+    const vector<string> &config_files = fillMultiTenantConfigFiles(active_tenants);
+    const vector<string> &expected_config_files = fillMultiTenantExpectedConfigFiles(active_tenants);
+    for (const string &file : config_files) {
+        dbgTrace(D_CONFIG) << "Inserting " << file << " to the list of files to be handled";
+        files.emplace(file, make_shared<ifstream>(file));
     }
-
-    for (const string &tenant : active_tenants) {
-        const vector<string> &profile_ids =
-            tenant_mananger ? tenant_mananger->fetchProfileIds(tenant) : vector<string>();
-        for (auto &profile_id : profile_ids) {
-            string settings_path =
-                config_directory_path + "tenant_" + tenant + "_profile_"+ profile_id + "_settings.json";
-            dbgTrace(D_CONFIG) << "Inserting a settings path: " << settings_path;
-            files.emplace(settings_path, make_shared<ifstream>(settings_path));
-        }
+    for (const string &file : expected_config_files) {
+        dbgTrace(D_CONFIG) << "Inserting " << file << " to the list of files to be handled";
+        files.emplace(file, make_shared<ifstream>(file));
     }
 
     vector<shared_ptr<JSONInputArchive>> archives;
@@ -883,6 +906,7 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
 void
 ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version, uint id)
 {
+    dbgFlow(D_CONFIG) << "Running reloadConfigurationContinuesWrapper. Version: " << version << ", Id: " << id;
     auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();
 
     LoadNewConfigurationStatus in_progress(id, false, false);

core/include/services_sdk/interfaces/i_socket_is.h

@@ -22,7 +22,7 @@
 class I_Socket
 {
 public:
-    enum class SocketType { UNIX, TCP, UDP };
+    enum class SocketType { UNIX, UNIXDG, TCP, UDP };
     using socketFd = int;
 
     virtual Maybe<socketFd>

core/include/services_sdk/interfaces/i_tenant_manager.h

@@ -16,6 +16,8 @@
 
 #include <string>
 #include <vector>
+#include <set>
+#include <map>
 #include <chrono>
 #include <functional>
 
@@ -27,13 +29,14 @@ public:
     virtual void uponNewTenants(const newTenantCB &cb) = 0;
     virtual bool areTenantAndProfileActive(const std::string &tenant_id, const std::string &profile_id) const = 0;
 
-    virtual std::vector<std::string> fetchActiveTenants() const = 0;
-    virtual std::vector<std::string> fetchAllActiveTenants() const = 0;
-    virtual std::vector<std::string> getInstances(
+    virtual std::set<std::string> fetchAllActiveTenants() const = 0;
+    virtual std::set<std::string> fetchActiveTenants() const = 0;
+    virtual std::set<std::string> getInstances(
         const std::string &tenant_id,
         const std::string &profile_id
     ) const = 0;
-    virtual std::vector<std::string> fetchProfileIds(const std::string &tenant_id) const = 0;
+    virtual std::map<std::string, std::set<std::string>> fetchActiveTenantsAndProfiles() const = 0;
+    virtual std::set<std::string> fetchProfileIds(const std::string &tenant_id) const = 0;
 
     virtual void deactivateTenant(const std::string &tenant_id, const std::string &profile_id) = 0;
 
@@ -41,7 +44,7 @@ public:
 
     virtual std::chrono::microseconds getTimeoutVal() const = 0;
 
-    virtual std::vector<std::string> getProfileId(
+    virtual std::set<std::string> getProfileIdsForRegionAccount(
         const std::string &tenant_id,
         const std::string &region,
         const std::string &account_id = ""

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h

@@ -81,6 +81,7 @@ IntelligenceQuery<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
         unsigned int valid_idx = 0;
         const auto &valid_response = bulk_response.getValid();
         const auto &errors = bulk_response.getErrors();
+        responses.clear();
         responses.reserve(requests.size());
         dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items";
         for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) {

core/include/services_sdk/interfaces/mock/mock_tenant_manager.h

@@ -15,19 +15,20 @@ class MockTenantManager : public Singleton::Provide<I_TenantManager>::From<MockP
 public:
     MOCK_METHOD1(uponNewTenants,                    void(const I_TenantManager::newTenantCB &cb));
 
-    MOCK_CONST_METHOD0(fetchActiveTenants,          std::vector<std::string>());
-    MOCK_CONST_METHOD0(fetchAllActiveTenants,       std::vector<std::string>());
-    MOCK_CONST_METHOD1(fetchProfileIds,             std::vector<std::string>(const std::string &));
+    MOCK_CONST_METHOD0(fetchActiveTenantsAndProfiles,     std::map<std::string, std::set<std::string>>());
+    MOCK_CONST_METHOD0(fetchActiveTenants,          std::set<std::string>());
+    MOCK_CONST_METHOD0(fetchAllActiveTenants,       std::set<std::string>());
+    MOCK_CONST_METHOD1(fetchProfileIds,             std::set<std::string>(const std::string &));
     MOCK_CONST_METHOD2(
         getInstances,
-        std::vector<std::string>(const std::string &, const std::string &)
+        std::set<std::string>(const std::string &, const std::string &)
     );
     MOCK_CONST_METHOD2(areTenantAndProfileActive,   bool(const std::string &, const std::string &));
     MOCK_METHOD2(addActiveTenantAndProfile,         void(const std::string &, const std::string &));
     MOCK_METHOD2(deactivateTenant,                  void(const std::string &, const std::string &));
     MOCK_CONST_METHOD3(
-        getProfileId,
-        std::vector<std::string>(const std::string &, const std::string &, const std::string &)
+        getProfileIdsForRegionAccount,
+        std::set<std::string>(const std::string &, const std::string &, const std::string &)
     );
 
     MOCK_CONST_METHOD0(getTimeoutVal,               std::chrono::microseconds());

core/include/services_sdk/resources/debug_flags.h

@@ -137,6 +137,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
         DEFINE_FLAG(D_NGINX_MESSAGE_READER, D_REVERSE_PROXY)
         DEFINE_FLAG(D_ERROR_REPORTER, D_REVERSE_PROXY)
         DEFINE_FLAG(D_UPSTREAM_KEEPALIVE, D_REVERSE_PROXY)
+        DEFINE_FLAG(D_FORWARD_PROXY, D_REVERSE_PROXY)
 
     DEFINE_FLAG(D_IDA, D_COMPONENT)
 
@@ -166,6 +167,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_URL_FILTERING, D_COMPONENT)
     DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
+    DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
 
 DEFINE_FLAG(D_FLOW, D_ALL)
     DEFINE_FLAG(D_DROP, D_FLOW)

core/include/services_sdk/resources/report/report_enums.h

@@ -62,6 +62,7 @@ enum class Tags {
     DEPLOYMENT_EMBEDDED,
     DEPLOYMENT_K8S,
     LAYER_7_ACCESS_CONTROL,
+    HORIZON_TELEMETRY_METRICS,
 
     COUNT
 };
@@ -76,6 +77,7 @@ enum class AudienceTeam
     SIGNATURE_DEVELOPERS,
     FILE_UPLOAD,
     IDENTITY_AWARENESS,
+    HORIZON_TELEMETRY,
     NONE,
 
     COUNT
@@ -147,7 +149,8 @@ enum class IssuingEngine {
     IOT_NEXT,
     SDWAN,
     FILE_UPLOAD,
-    IDA_NEXT
+    IDA_NEXT,
+    HORIZON_TELEMETRY_METRICS
 };
 
 } // namespace ReportIS

core/include/services_sdk/utilities/agent_core_utilities.h

@@ -42,6 +42,8 @@ bool deleteFile(const std::string &path);
 
 std::string convertToHumanReadable(uint64_t size_in_bytes);
 
+std::string getFileName(const std::string &path);
+
 }// namespace Filesystem
 
 namespace Regex

core/report/tag_and_enum_management.cc

@@ -104,7 +104,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
         {"Kong Server", ReportIS::Tags::WEB_SERVER_KONG},
         {"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED},
         {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S},
-        {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL}
+        {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL},
+        {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}
     };
 
     auto report_is_tag = strings_to_tags.find(tag);
@@ -264,6 +265,7 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine)
         case IssuingEngine::SDWAN: return "sdwanGwSharing";
         case IssuingEngine::FILE_UPLOAD: return "fileUpload";
         case IssuingEngine::IDA_NEXT: return "quantumMetaNotifyIdn";
+        case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics";
     }
 
     dbgAssert(false) << "Reached impossible engine value of: " << static_cast<int>(issuing_engine);
@@ -302,7 +304,8 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
     "Kong Server",
     "Embedded Deployment",
     "Kubernetes Deployment",
-    "Layer 7 Access Control"
+    "Layer 7 Access Control",
+    "Horizon Telemetry Metrics"
 };
 
 EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {
@@ -312,5 +315,6 @@ EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation
     "Agent Intelligence",
     "cpviewMonitoring",
     "Signature Developers",
-    "Identity Awareness"
+    "Identity Awareness",
+    "unifiedMonitoring"
 };

core/socket_is/socket_is.cc

@@ -489,6 +489,109 @@ private:
     struct sockaddr_un server;
 };
 
+class UnixDGSocket : public SocketInternal
+{
+public:
+    static Maybe<unique_ptr<UnixDGSocket>>
+    connectSock(bool _is_blocking, bool _is_server, const string &_address)
+    {
+        unique_ptr<UnixDGSocket> unix_socket(make_unique<UnixDGSocket>(_is_blocking, _is_server));
+        if (unix_socket->getSocket() <= 0) return genError("Failed to create socket");
+
+        unix_socket->server.sun_family = AF_UNIX;
+        strncpy(unix_socket->server.sun_path, _address.c_str(), sizeof(unix_socket->server.sun_path) - 1);
+
+        if (!unix_socket->isServerSock()) {
+            if (connect(
+                    unix_socket->getSocket(),
+                    reinterpret_cast<struct sockaddr *>(&unix_socket->server),
+                    sizeof(struct sockaddr_un)
+                ) == -1
+            ) {
+                return genError("Failed to connect socket");
+            }
+            return move(unix_socket);
+        }
+
+        static const int on = 1;
+        if (setsockopt(unix_socket->getSocket(), SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0) {
+            dbgWarning(D_SOCKET) << "Failed to set the socket descriptor as reusable";
+            return genError("Failed to set the socket descriptor as reusable");
+        }
+
+        const int priority = 6;
+        if (setsockopt(unix_socket->getSocket(), SOL_SOCKET, SO_PRIORITY, (char *)&priority, sizeof(priority)) < 0) {
+            dbgWarning(D_SOCKET) << "Failed to set the socket priority to highest";
+            return genError("Failed to set the socket priority to highest");
+        }
+
+        if (ioctl(unix_socket->getSocket(), FIONBIO, (char *)&on) < 0) {
+            dbgWarning(D_SOCKET) << "Failed to set the socket as non-blocking";
+            return genError("Failed to set the socket as non-blocking");
+        }
+
+        unlink(unix_socket->server.sun_path);
+        if (bind(
+                unix_socket->getSocket(),
+                reinterpret_cast<struct sockaddr *>(&unix_socket->server),
+                sizeof(struct sockaddr_un)
+            ) == -1) {
+            dbgWarning(D_SOCKET) << "Failed to bind the socket: " << strerror(errno);
+            return genError("Failed to bind the socket");
+        }
+
+        chmod(unix_socket->server.sun_path, 0666);
+
+        return move(unix_socket);
+    }
+
+    void cleanServer() override
+    {
+        unlink(server.sun_path);
+    }
+
+    Maybe<vector<char>>
+    receiveDataBlocking(uint data_size) override
+    {
+        return receiveDGData(data_size, MSG_DONTWAIT);
+    }
+
+    Maybe<vector<char>>
+    receiveDataNonBlocking(uint data_size) override
+    {
+        return receiveDGData(data_size, 0);
+    }
+
+    Maybe<vector<char>>
+    receiveDGData(uint data_size, int flag)
+    {
+        if (data_size == 0) data_size = udp_max_packet_size;
+        dbgDebug(D_SOCKET) << "data_size: " << data_size;
+        vector<char> param_to_read(data_size, 0);
+        int res = recv(socket_int, param_to_read.data(), data_size, flag);
+
+        if (res == -1) {
+            string error_message = strerror(errno);
+            dbgWarning(D_SOCKET) << "Failed to read data, Error: " + error_message;
+            return genError(
+                "Failed to read data, Error: " + error_message
+            );
+        }
+        param_to_read.resize(res);
+        return param_to_read;
+    }
+
+    UnixDGSocket(bool _is_blocking, bool _is_server_socket)
+        :
+        SocketInternal(_is_blocking, _is_server_socket)
+    {
+        socket_int = socket(AF_UNIX, SOCK_DGRAM, 0);
+    }
+
+private:
+    struct sockaddr_un server;
+};
+
 class SocketIS::Impl
         :
     Singleton::Provide<I_Socket>::From<SocketIS>
@@ -527,6 +630,11 @@ SocketIS::Impl::genSocket(
         if (!unix_sock.ok()) return unix_sock.passErr();
         new_sock = unix_sock.unpackMove();
         socketTypeName = "UNIX";
+    } else if (type ==  SocketType::UNIXDG) {
+        Maybe<unique_ptr<SocketInternal>> unix_dg_sock = UnixDGSocket::connectSock(is_blocking, is_server, address);
+        if (!unix_dg_sock.ok()) return unix_dg_sock.passErr();
+        new_sock = unix_dg_sock.unpackMove();
+        socketTypeName = "UNIXDG";
     } else if (type == SocketType::TCP) {
         Maybe<unique_ptr<SocketInternal>> tcp_sock = TCPSocket::connectSock(is_blocking, is_server, address);
         if (!tcp_sock.ok()) return tcp_sock.passErr();

core/tenant_manager/tenant_manager.cc

@@ -78,10 +78,11 @@ public:
     void uponNewTenants(const newTenantCB &cb) override;
     bool areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const override;
 
-    vector<string> fetchAllActiveTenants() const override;
-    vector<string> fetchActiveTenants() const override;
-    vector<string> getInstances(const string &tenant_id, const string &profile_id) const override;
-    vector<string> fetchProfileIds(const string &tenant_id) const override;
+    map<string, set<string>> fetchActiveTenantsAndProfiles() const override;
+    set<string> fetchAllActiveTenants() const override;
+    set<string> fetchActiveTenants() const override;
+    set<string> getInstances(const string &tenant_id, const string &profile_id) const override;
+    set<string> fetchProfileIds(const string &tenant_id) const override;
 
     void addActiveTenantAndProfile(const string &tenant_id, const string &profile_id) override;
 
@@ -89,7 +90,11 @@ public:
 
     chrono::microseconds getTimeoutVal() const override;
 
-    vector<string> getProfileId(const string &tenant_id, const string &region, const string &account) const override;
+    set<string> getProfileIdsForRegionAccount(
+        const string &tenant_id,
+        const string &region,
+        const string &account
+    ) const override;
 
     void
     addInstance(const string &tenant_id, const string &profile_id, const string &instace_id)
@@ -111,9 +116,9 @@ public:
 private:
     void runUponNewTenants(const vector<string> &new_tenants);
     void sendTenantAndProfile(const string &tenant_id, const string &profile_id);
-    vector<string> getAllTenants() const;
-    vector<string> fetchAllProfileIds(const string &tenant_id) const;
-    vector<string> getProfileIds(const string &profile_id) const;
+    set<string> getAllTenants() const;
+    set<string> fetchAllProfileIds(const string &tenant_id) const;
+    set<string> getProfileIds(const string &tenant_id) const;
     bool sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port);
 
     TemporaryCache<TenantProfilePair, void> active_tenants;
@@ -169,7 +174,7 @@ public:
         active_tenants = Singleton::Consume<I_TenantManager>::from<TenantManager>()->fetchAllActiveTenants();
     }
 
-    S2C_PARAM(std::vector<std::string>, active_tenants);
+    S2C_PARAM(set<string>, active_tenants);
 };
 
 class GetActiveTenants : public ClientRest
@@ -179,7 +184,7 @@ public:
 
     Maybe<string> genJson() const { return string("{}"); };
 
-    S2C_PARAM(vector<string>, active_tenants);
+    S2C_PARAM(set<string>, active_tenants);
 };
 
 class FetchProfileIds : public ServerRest
@@ -191,7 +196,7 @@ public:
         profile_ids = Singleton::Consume<I_TenantManager>::from<TenantManager>()->fetchProfileIds(tenant_id);
     }
 
-    S2C_PARAM(vector<string>, profile_ids);
+    S2C_PARAM(set<string>, profile_ids);
     C2S_PARAM(string, tenant_id);
 };
 
@@ -200,7 +205,7 @@ class GetProfileIds : public ClientRest
 public:
     GetProfileIds(const string &_tenant_id) : profile_ids(), tenant_id(_tenant_id) {};
 
-    S2C_PARAM(vector<string>, profile_ids);
+    S2C_PARAM(set<string>, profile_ids);
     C2S_PARAM(string, tenant_id);
 };
 
@@ -318,7 +323,7 @@ TenantManager::Impl::sendTenantAndProfile(const string &tenant_id, const string
     }
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::getAllTenants() const
 {
     dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active tenants";
@@ -348,7 +353,7 @@ TenantManager::Impl::getAllTenants() const
     return active_tenant.active_tenants.get();
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::getProfileIds(const string &_tenant_id) const
 {
     dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active profiles";
@@ -379,22 +384,25 @@ TenantManager::Impl::getProfileIds(const string &_tenant_id) const
 }
 
 
-vector<string>
-TenantManager::Impl::getProfileId(const string &tenant_id, const string &region, const string &account_id = "") const
+set<string>
+TenantManager::Impl::getProfileIdsForRegionAccount(
+    const string &tenant_id,
+    const string &region,
+    const string &account_id = "") const
 {
     if (region.empty()) {
         dbgWarning(D_TENANT_MANAGER) << "Can't find the profile ID. Region is empty";
-        return vector<string>();
+        return set<string>();
     }
 
-    vector<string> profile_ids = fetchProfileIds(tenant_id);
+    set<string> profile_ids = fetchProfileIds(tenant_id);
 
     dbgTrace(D_TENANT_MANAGER) << "Fetched " << profile_ids.size() << " profiles";
 
     auto i_env = Singleton::Consume<I_Environment>::by<TenantManager>();
     auto unset_tenant_on_exit = make_scope_exit([&]() { i_env->unsetActiveTenantAndProfile(); });
 
-    vector<string> profiles_to_return;
+    set<string> profiles_to_return;
     for (const string &profile_id : profile_ids) {
         string account_dbg = account_id.empty() ? "" : (" in the account " + account_id);
         dbgDebug(D_TENANT_MANAGER)
@@ -413,20 +421,20 @@ TenantManager::Impl::getProfileId(const string &tenant_id, const string &region,
             auto account_region_set = maybe_account_region_set.unpack().getAccoutRegionPairs();
             if (account_region_set.empty()) {
                 dbgTrace(D_TENANT_MANAGER) << "Old profile with new hook. Resolving to profile ID: " << profile_id;
-                profiles_to_return.push_back(profile_id);
+                profiles_to_return.insert(profile_id);
                 return profiles_to_return;
             }
             for (const AccountRegionPair &account : account_region_set) {
                 if (region == account.getRegion() && (account_id.empty() || account_id == account.getAccountID())) {
                     dbgTrace(D_TENANT_MANAGER) << "Found a corresponding profile ID: " << profile_id;
-                    profiles_to_return.push_back(profile_id);
+                    profiles_to_return.insert(profile_id);
                 }
             }
         } else {
             auto maybe_region = getSetting<string>("region");
             if (maybe_region.ok() && region == maybe_region.unpack()) {
                 dbgDebug(D_TENANT_MANAGER) << "The region corresponds to profile ID " << profile_id;
-                profiles_to_return.push_back(profile_id);
+                profiles_to_return.insert(profile_id);
                 return profiles_to_return;
             } else {
                 if (maybe_region.ok()) {
@@ -448,7 +456,7 @@ TenantManager::Impl::getProfileId(const string &tenant_id, const string &region,
     }
 
     dbgWarning(D_TENANT_MANAGER) << "Found no corresponding profile ID";
-    return vector<string>();
+    return set<string>();
 }
 
 void
@@ -490,58 +498,69 @@ TenantManager::Impl::deactivateTenant(const string &tenant_id, const string &pro
     active_tenants.deleteEntry(TenantProfilePair(tenant_id, profile_id));
 }
 
-vector<string>
+map<string, set<string>>
+TenantManager::Impl::fetchActiveTenantsAndProfiles() const
+{
+    dbgFlow(D_TENANT_MANAGER) << "Fetching active teants and profiles map";
+    map<string, set<string>> active_tenants_and_profiles;
+    set<string> tenants = fetchAllActiveTenants();
+    for (const string &tenant : tenants) {
+        active_tenants_and_profiles[tenant] = fetchProfileIds(tenant);
+    }
+
+    return active_tenants_and_profiles;
+}
+
+set<string>
 TenantManager::Impl::fetchAllActiveTenants() const
 {
     dbgFlow(D_TENANT_MANAGER) << "Fetching all active tenants";
     return (type == TenantManagerType::CLIENT) ? getAllTenants() : fetchActiveTenants();
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::fetchActiveTenants() const
 {
     dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a server. Fetching active tenants";
-    vector<string> tenants;
-    tenants.reserve(active_tenants.size());
-    for (auto iter = begin(active_tenants); iter != end(active_tenants); iter++) {
-        dbgDebug(D_TENANT_MANAGER) << "Found a tenant to return. Tenant ID: " << iter->first.getTenantId();
-        tenants.push_back(iter->first.getTenantId());
+    set<string> tenants;
+    for (const auto &iter : active_tenants) {
+        dbgDebug(D_TENANT_MANAGER) << "Found a tenant to return. Tenant ID: " << iter.first.getTenantId();
+        tenants.insert(iter.first.getTenantId());
     }
 
     return tenants;
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::getInstances(const string &tenant_id, const string &profile_id) const
 {
-    vector<string> instances;
+    set<string> instances;
     auto tenant_profile_pair = TenantProfilePair(tenant_id, profile_id);
     auto tenant_instance_cache = mapper.find(tenant_profile_pair);
 
     if (tenant_instance_cache == mapper.end()) return instances;
 
-    instances.reserve(tenant_instance_cache->second.size());
     for (auto iter = begin(tenant_instance_cache->second); iter != end(tenant_instance_cache->second); iter++) {
-        instances.push_back(iter->first);
+        instances.insert(iter->first);
     }
     return instances;
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::fetchAllProfileIds(const string &tenant_id) const
 {
-    vector<string> tenant_profile_ids;
+    set<string> tenant_profile_ids;
 
     for (auto iter = begin(active_tenants); iter != end(active_tenants); iter++) {
         if (iter->first.getTenantId() == tenant_id) {
             dbgTrace(D_TENANT_MANAGER) << "Returning a fetched profile ID: " << iter->first.getProfileId();
-            tenant_profile_ids.push_back(iter->first.getProfileId());
+            tenant_profile_ids.insert(iter->first.getProfileId());
         }
     }
     return tenant_profile_ids;
 }
 
-vector<string>
+set<string>
 TenantManager::Impl::fetchProfileIds(const string &tenant_id) const
 {
     dbgFlow(D_TENANT_MANAGER) << "Fetching all profile IDs for tenant " << tenant_id;

events/include/new_table_entry.h

@@ -0,0 +1,10 @@
+#ifndef __NEW_TABLE_ENTRY_H__
+#define __NEW_TABLE_ENTRY_H__
+
+#include "event.h"
+
+class NewTableEntry : public Event<NewTableEntry>
+{
+};
+
+#endif // __NEW_TABLE_ENTRY_H__

events/include/parsed_context.h

@@ -0,0 +1,25 @@
+#ifndef __PARSED_CONTEXT_H__
+#define __PARSED_CONTEXT_H__
+
+#include <string>
+
+#include "event.h"
+#include "buffer.h"
+
+enum class ParsedContextReply { ACCEPT, DROP };
+
+class ParsedContext : public Event<ParsedContext, ParsedContextReply>
+{
+public:
+    ParsedContext(const Buffer &_buf, const std::string &_name, uint _id) : buf(_buf), name(_name), id(_id) {}
+    const Buffer & getBuffer() const { return buf; }
+    const std::string & getName() const { return name; }
+    uint getId() const  { return id; }
+
+private:
+    Buffer buf;
+    std::string name;
+    uint id;
+};
+
+#endif // __PARSED_CONTEXT_H__

nodes/http_transaction_handler/CMakeLists.txt

@@ -32,6 +32,8 @@ target_link_libraries(cp-nano-http-transaction-handler
 	waap
 	waap_clib
 	reputation
+        ips
+        keywords
 	-Wl,--end-group
 )
 

nodes/http_transaction_handler/main.cc

@@ -17,6 +17,8 @@
 #include "gradual_deployment.h"
 #include "http_manager.h"
 #include "waap.h"
+#include "ips_comp.h"
+#include "keyword_comp.h"
 
 int
 main(int argc, char **argv)
@@ -26,7 +28,9 @@ main(int argc, char **argv)
         NginxAttachment,
         GradualDeployment,
         HttpManager,
-	WaapComponent
+	WaapComponent,
+        IPSComp,
+        KeywordComp
     > comps;
 
     comps.registerGlobalValue<bool>("Is Rest primary routine", true);

nodes/orchestration/package/cpnano_debug/cpnano_debug.cc

@@ -95,6 +95,7 @@ enum class Service {
     HELLO_WORLD,
     IDA,
     IOT_ACCESS_CONTROL,
+    HORIZON_TELEMETRY,
 
     COUNT
 };
@@ -171,6 +172,7 @@ getServiceString(const Service service)
         case (Service::HELLO_WORLD): return "hello-world";
         case (Service::IDA): return "identity-awareness";
         case (Service::IOT_ACCESS_CONTROL): return "iot-access-control";
+        case (Service::HORIZON_TELEMETRY): return "horizon-telemetry";
         default:
             cerr
                 << "Internal Error: the provided service ("
@@ -359,6 +361,11 @@ getServiceConfig (const Service service)
                 filesystem_path + "/conf/cp-nano-iot-access-control-debug-conf.json",
                 log_files_path + "/nano_agent/cp-nano-iot-access-control.dbg"
             );
+        case (Service::HORIZON_TELEMETRY):
+            return ServiceConfig(
+                filesystem_path + "/conf/cp-nano-horizon-telemetry-debug-conf.json",
+                log_files_path + "/nano_agent/cp-nano-horizon-telemetry.dbg"
+            );
         default:
             cerr
                 << "Internal Error: the provided service ("
@@ -1287,6 +1294,8 @@ extractServices(const vector<string> &args)
             services.push_back(Service::IDA);
         } else if (getServiceString(Service::IOT_ACCESS_CONTROL).find(maybe_service) == 0) {
             services.push_back(Service::IOT_ACCESS_CONTROL);
+        } else if (getServiceString(Service::HORIZON_TELEMETRY).find(maybe_service) == 0) {
+            services.push_back(Service::HORIZON_TELEMETRY);
         } else {
             break;
         }