diff --git a/CMakeLists.txt b/CMakeLists.txt index bb356b2..fbe21a2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,7 @@ include_directories(core/include/services_sdk/interfaces) include_directories(core/include/services_sdk/resources) include_directories(core/include/services_sdk/utilities) include_directories(core/include/attachments) +include_directories(events/include) include_directories(components/include) add_subdirectory(build_system) diff --git a/components/include/i_downloader.h b/components/include/i_downloader.h index f39262a..15116af 100755 --- a/components/include/i_downloader.h +++ b/components/include/i_downloader.h @@ -39,6 +39,8 @@ public: Package::ChecksumTypes checksum_type, const std::string &service_name ) const = 0; + + virtual std::string getProfileFromMap(const std::string &tenant_id) const = 0; }; #endif // __I_DOWNLOADER_H__ diff --git a/components/include/i_keywords_rule.h b/components/include/i_keywords_rule.h new file mode 100644 index 0000000..fe3ce69 --- /dev/null +++ b/components/include/i_keywords_rule.h @@ -0,0 +1,30 @@ +#ifndef ___I_KEYWORDS_RULE_H__ +#define ___I_KEYWORDS_RULE_H__ + +#include +#include + +#include "maybe_res.h" + +class I_KeywordsRule +{ +public: + class VirtualRule + { + public: + virtual ~VirtualRule() {}; + virtual bool isMatch() const = 0; + }; + + virtual Maybe> genRule(const std::string &rule) = 0; + + static const std::string & getKeywordsRuleTag() { return keywords_tag; } + +protected: + virtual ~I_KeywordsRule() {} + +private: + static std::string keywords_tag; +}; + +#endif // ___I_KEYWORDS_RULE_H__ diff --git a/components/include/i_orchestration_tools.h b/components/include/i_orchestration_tools.h index 8b7813d..630ce88 100755 --- a/components/include/i_orchestration_tools.h +++ b/components/include/i_orchestration_tools.h @@ -111,6 +111,10 @@ public: virtual bool removeFile(const std::string &path) const = 0; virtual bool copyFile(const std::string &src_path, const std::string &dst_path) const = 0; virtual bool doesFileExist(const std::string &file_path) const = 0; + virtual void fillKeyInJson( + const std::string &filename, + const std::string &_key, + const std::string &_val) const = 0; virtual bool createDirectory(const std::string &directory_path) const = 0; virtual bool doesDirectoryExist(const std::string &dir_path) const = 0; virtual bool executeCmd(const std::string &cmd) const = 0; diff --git a/components/include/ips_comp.h b/components/include/ips_comp.h new file mode 100644 index 0000000..599a5b2 --- /dev/null +++ b/components/include/ips_comp.h @@ -0,0 +1,35 @@ +#ifndef __IPS_COMP_H__ +#define __IPS_COMP_H__ + +#include "singleton.h" +#include "i_keywords_rule.h" +#include "i_table.h" +#include "i_http_manager.h" +#include "i_environment.h" +#include "http_inspection_events.h" +#include "i_generic_rulebase.h" +#include "component.h" + +class IPSComp + : + public Component, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume +{ +public: + IPSComp(); + ~IPSComp(); + + void preload(); + + void init(); + void fini(); + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __IPS_COMP_H__ diff --git a/components/include/keyword_comp.h b/components/include/keyword_comp.h new file mode 100644 index 0000000..cfde6b5 --- /dev/null +++ b/components/include/keyword_comp.h @@ -0,0 +1,28 @@ +#ifndef __KEYWORD_COMP__ +#define __KEYWORD_COMP__ + +#include + +#include "singleton.h" +#include "i_environment.h" +#include "i_table.h" +#include "i_keywords_rule.h" +#include "component.h" + +class KeywordComp + : + public Component, + Singleton::Provide, + Singleton::Consume, + Singleton::Consume +{ +public: + KeywordComp(); + ~KeywordComp(); + +private: + class Impl; + std::unique_ptr pimpl; +}; + +#endif // __KEYWORD_COMP__ diff --git a/components/include/output.h b/components/include/output.h new file mode 100644 index 0000000..a21ee4a --- /dev/null +++ b/components/include/output.h @@ -0,0 +1,13 @@ +#ifndef __OUTPUT_H__ +#define __OUTPUT_H__ + +#include + +namespace infra +{ + +std::string printChar(char ch); + +} // namespace infra + +#endif // __OUTPUT_H__ diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index 68e330c..ca6c3ad 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(ips) add_subdirectory(layer_7_access_control) add_subdirectory(orchestration) add_subdirectory(waap) diff --git a/components/security_apps/ips/CMakeLists.txt b/components/security_apps/ips/CMakeLists.txt new file mode 100644 index 0000000..a454b87 --- /dev/null +++ b/components/security_apps/ips/CMakeLists.txt @@ -0,0 +1,17 @@ +include_directories(include) + +add_library(ips + ips_comp.cc + ips_entry.cc + ips_signatures.cc + simple_protection.cc + compound_protection.cc + ips_configuration.cc + helper_open_source.cc + ips_basic_policy.cc + snort_basic_policy.cc + ips_metric.cc + ips_common_types.cc +) + +add_subdirectory(ips_ut) diff --git a/components/security_apps/ips/compound_protection.cc b/components/security_apps/ips/compound_protection.cc new file mode 100644 index 0000000..792385f --- /dev/null +++ b/components/security_apps/ips/compound_protection.cc @@ -0,0 +1,216 @@ +#include "compound_protection.h" + +#include + +#include "rule_detection.h" +#include "ips_entry.h" +#include "ips_comp.h" +#include "debug.h" + +using namespace std; +using MatchType = IPSSignatureSubTypes::BaseSignature::MatchType; + +USE_DEBUG_FLAG(D_IPS); + +CompoundProtection::Impl::Impl(const string &_sig_name, SignaturesVector &&sig_vec, Operation oper) + : + sig_name(_sig_name), + sub_signatures(move(sig_vec)), + operation(oper), + table(Singleton::Consume::by()) +{ + for (const auto &sig : sub_signatures) { + const auto &sub_sig_context = sig->getContext(); + for (auto &new_context : sub_sig_context) { + bool is_new_context = true; + for (auto &old_context : contexts) { + if (new_context == old_context) { + is_new_context = false; + break; + } + } + if (is_new_context) contexts.push_back(new_context); + } + } +} + +MatchType +CompoundProtection::Impl::getMatch(const set &matched) const +{ + switch (operation) { + case Operation::OR: return getMatchOr(matched); + case Operation::AND: return getMatchAnd(matched); + case Operation::ORDERED_AND: return getMatchOrderedAnd(matched); + } + + dbgAssert(false) << "Unknown compound operation: " << static_cast(operation); + return MatchType::NO_MATCH; +} + +set +CompoundProtection::Impl::patternsInSignature() const +{ + set res; + + for (auto &sig : sub_signatures) { + const auto &sub_sig_patterns = sig->patternsInSignature(); + for (auto &pat : sub_sig_patterns) { + res.insert(pat); + } + } + + return res; +} + +MatchType +CompoundProtection::Impl::getMatchOr(const set &matched) const +{ + MatchType res = MatchType::NO_MATCH; + for (auto &sig : sub_signatures) { + switch (getSubMatch(sig, matched)) { + case MatchType::NO_MATCH: break; + case MatchType::CACHE_MATCH: { + res = MatchType::CACHE_MATCH; + break; + } + case MatchType::MATCH: return MatchType::MATCH; + } + } + return res; +} + +MatchType +CompoundProtection::Impl::getMatchAnd(const set &matched) const +{ + MatchType res = MatchType::CACHE_MATCH; + for (auto &sig : sub_signatures) { + switch (getSubMatch(sig, matched)) { + case MatchType::NO_MATCH: { + res = MatchType::NO_MATCH; + break; + } + case MatchType::CACHE_MATCH: break; + case MatchType::MATCH: { + if (res == MatchType::CACHE_MATCH) res = MatchType::MATCH; + break; + } + } + } + return res; +} + +MatchType +CompoundProtection::Impl::getMatchOrderedAnd(const set &matched) const +{ + MatchType res = MatchType::CACHE_MATCH; + for (auto &sig : sub_signatures) { + switch (getSubMatch(sig, matched)) { + case MatchType::NO_MATCH: return MatchType::NO_MATCH; + case MatchType::CACHE_MATCH: break; + case MatchType::MATCH: { + res = MatchType::MATCH; + break; + } + } + } + return res; +} + +static bool +isStringInVector(const Maybe &str, const vector &vec) +{ + if (!str.ok()) return false; + return find(vec.begin(), vec.end(), *str) != vec.end(); +} + +MatchType +CompoundProtection::Impl::getSubMatch( + const std::shared_ptr &sub_sig, + const set &matched +) const +{ + if (isFlagSet(sub_sig->getSigId())) return MatchType::CACHE_MATCH; + + auto env = Singleton::Consume::by(); + auto curr_ctx = env->get(I_KeywordsRule::getKeywordsRuleTag()); + if (!isStringInVector(curr_ctx, sub_sig->getContext())) return MatchType::NO_MATCH; + + auto res = sub_sig->getMatch(matched); + if (res != MatchType::NO_MATCH) setFlag(sub_sig->getSigId()); + return res; +} + +bool +CompoundProtection::Impl::isFlagSet(const std::string &id) const +{ + if (!table->hasState()) { + dbgWarning(D_IPS) << "No entry was found, limited compound functionality"; + return false; + } + return table->getState().isFlagSet(id); +} + +void +CompoundProtection::Impl::setFlag(const std::string &id) const +{ + if (!table->hasState()) { + dbgWarning(D_IPS) << "No entry was found, limited compound functionality"; + return; + } + table->getState().setFlag(id); +} + +class OperandsReader +{ +public: + OperandsReader(const string &sig_name) : base_sig_name(sig_name) {} + + void + load(cereal::JSONInputArchive &ar) + { + cereal::size_type cereal_size; + ar(cereal::make_size_tag(cereal_size)); + rules.resize(static_cast(cereal_size)); + + uint index = 0; + for (auto &rule : rules) { + stringstream ss; + ss << base_sig_name << "##" << index; + ++index; + RuleDetection detection(ss.str()); + ar(detection); + rule = detection.getRule(); + } + } + + vector> && extrackRules() { return move(rules); } + +private: + string base_sig_name; + vector> rules; +}; + +shared_ptr +CompoundProtection::get(const string &sig_name, cereal::JSONInputArchive &ar) +{ + string operation; + OperandsReader operands(sig_name); + + ar( + cereal::make_nvp("operation", operation), + cereal::make_nvp("operands", operands) + ); + + return make_shared(sig_name, operands.extrackRules(), getOperation(operation)); +} + +CompoundProtection::Operation +CompoundProtection::getOperation(const string &operation) +{ + if (operation == "or") return Operation::OR; + if (operation == "and") return Operation::AND; + if (operation == "ordered_and") return Operation::ORDERED_AND; + + reportConfigurationError("Unknown compound operation: " + operation); + return Operation::OR; +} diff --git a/components/security_apps/ips/helper_open_source.cc b/components/security_apps/ips/helper_open_source.cc new file mode 100644 index 0000000..dfe750c --- /dev/null +++ b/components/security_apps/ips/helper_open_source.cc @@ -0,0 +1,24 @@ +#include "helper.h" + +#include "config.h" + +using namespace std; + +namespace IPSHelper +{ + +string +deobfuscateString(const string &str) +{ + if (str.substr(0, 7) == "M^AGI$C") reportConfigurationError("Deobfuscation isn't available in open-source mode"); + return str; +} + +string +deobfuscateKeyword(const string &str) +{ + if (str.substr(0, 7) == "M^AGI$C") reportConfigurationError("Deobfuscation isn't available in open-source mode"); + return str; +} + +} // IPSHelper diff --git a/components/security_apps/ips/include/compound_protection.h b/components/security_apps/ips/include/compound_protection.h new file mode 100644 index 0000000..c251f72 --- /dev/null +++ b/components/security_apps/ips/include/compound_protection.h @@ -0,0 +1,49 @@ +#ifndef __COMPOUND_PROTECTION_H__ +#define __COMPOUND_PROTECTION_H__ + +#include + +#include "ips_signatures.h" +#include "i_table.h" + +class CompoundProtection +{ + enum class Operation { OR, AND, ORDERED_AND }; + using BaseSignature = IPSSignatureSubTypes::BaseSignature; + + class Impl : public IPSSignatureSubTypes::BaseSignature + { + using SignaturesVector = std::vector>; + + public: + Impl(const std::string &sig_name, SignaturesVector &&sig_vec, Operation oper); + + const std::string & getSigId() const override { return sig_name; } + MatchType getMatch(const std::set &matched) const override; + std::set patternsInSignature() const override; + const std::vector & getContext() const override { return contexts; } + + private: + MatchType getMatchOr(const std::set &matched) const; + MatchType getMatchAnd(const std::set &matched) const; + MatchType getMatchOrderedAnd(const std::set &matched) const; + + MatchType getSubMatch(const std::shared_ptr &sub_sig, const std::set &matched) const; + bool isFlagSet(const std::string &id) const; + void setFlag(const std::string &id) const; + + std::string sig_name; + SignaturesVector sub_signatures; + std::vector contexts; + Operation operation; + I_Table *table; + }; + +public: + static std::shared_ptr get(const std::string &sig_name, cereal::JSONInputArchive &ar); + +private: + static Operation getOperation(const std::string &operation); +}; + +#endif // __COMPOUND_PROTECTION_H__ diff --git a/components/security_apps/ips/include/helper.h b/components/security_apps/ips/include/helper.h new file mode 100644 index 0000000..340e294 --- /dev/null +++ b/components/security_apps/ips/include/helper.h @@ -0,0 +1,14 @@ +#ifndef __HELPER_H__ +#define __HELPER_H__ + +#include + +namespace IPSHelper +{ + +std::string deobfuscateString(const std::string &str); +std::string deobfuscateKeyword(const std::string &str); + +} // IPSHelper + +#endif // __HELPER_H__ diff --git a/components/security_apps/ips/include/i_first_tier_agg.h b/components/security_apps/ips/include/i_first_tier_agg.h new file mode 100644 index 0000000..492fb36 --- /dev/null +++ b/components/security_apps/ips/include/i_first_tier_agg.h @@ -0,0 +1,19 @@ +#ifndef __I_FIRST_TIER_AGG_H__ +#define __I_FIRST_TIER_AGG_H__ + +#include +#include +#include + +#include "pm_hook.h" + +class I_FirstTierAgg +{ +public: + virtual std::shared_ptr getHook(const std::string &context_name, const std::set &patterns) = 0; + +protected: + virtual ~I_FirstTierAgg() {} +}; + +#endif // __I_FIRST_TIER_AGG_H__ diff --git a/components/security_apps/ips/include/ips_basic_policy.h b/components/security_apps/ips/include/ips_basic_policy.h new file mode 100644 index 0000000..c57f586 --- /dev/null +++ b/components/security_apps/ips/include/ips_basic_policy.h @@ -0,0 +1,58 @@ +#ifndef __IPS_BASIC_POLICY_H__ +#define __IPS_BASIC_POLICY_H__ + +#include +#include +#include +#include "ips_enums.h" +#include "debug.h" +#include "maybe_res.h" +#include "ips_signatures.h" + +class RuleSelector +{ +public: + class Rule + { + public: + void serialize(cereal::JSONInputArchive &ar); + + bool isSignaturedMatched(const IPSSignatureSubTypes::CompleteSignature &signature) const; + const IPSSignatureSubTypes::SignatureAction & getAction() const { return action; }; + void readAction(cereal::JSONInputArchive &ar, const std::string &action_type); + void print(std::ostream &os) const; + + private: + void readPerformanceImpact(cereal::JSONInputArchive &ar); + void readSeverityLevel(cereal::JSONInputArchive &ar); + void readConfidenceLevel(cereal::JSONInputArchive &ar); + void readServerProtections(cereal::JSONInputArchive &ar); + void readClientProtections(cereal::JSONInputArchive &ar); + void readProtectionsFromYear(cereal::JSONInputArchive &ar); + void readProtectionTags(cereal::JSONInputArchive &ar); + void readProtectionIds(cereal::JSONInputArchive &ar); + + IPSSignatureSubTypes::SignatureAction action = IPSSignatureSubTypes::SignatureAction::IGNORE; + Maybe performance_impact = genError("undefined"); + Maybe severity_level = genError("undefined"); + Maybe confidence_level = genError("undefined"); + Maybe server_protections = genError("undefined"); + Maybe client_protections = genError("undefined"); + Maybe protections_from_year = genError("undefined"); + Maybe> protection_tags = genError("undefined"); + Maybe> protection_ids = genError("undefined"); + }; + +public: + std::vector selectSignatures() const; + void print(std::ostream &os) const; + void load(cereal::JSONInputArchive &ar); + +private: + void readRules(cereal::JSONInputArchive &ar); + void readDefaultAction(cereal::JSONInputArchive &ar); + + std::vector rules; +}; + +#endif // __IPS_BASIC_POLICY_H__ diff --git a/components/security_apps/ips/include/ips_common_types.h b/components/security_apps/ips/include/ips_common_types.h new file mode 100644 index 0000000..f0c8af0 --- /dev/null +++ b/components/security_apps/ips/include/ips_common_types.h @@ -0,0 +1,11 @@ +#ifndef __IPS_COMMON_TYPES__ +#define __IPS_COMMON_TYPES__ + +#include "buffer.h" + +struct IPSCommonTypes +{ + static const Buffer requests_header_for_log; +}; + +#endif //__IPS_COMMON_TYPES__ diff --git a/components/security_apps/ips/include/ips_configuration.h b/components/security_apps/ips/include/ips_configuration.h new file mode 100644 index 0000000..4b280db --- /dev/null +++ b/components/security_apps/ips/include/ips_configuration.h @@ -0,0 +1,36 @@ +#ifndef __IPS_CONFIGURATION_H__ +#define __IPS_CONFIGURATION_H__ + +#include "config.h" + +class IPSConfiguration +{ +public: + enum class ContextType { NORMAL, KEEP, HISTORY }; + + class Context { + public: + Context() : type(ContextType::NORMAL), history_size(0) {} + Context(ContextType type, uint history); + + ContextType getType() const { return type; } + uint getHistorySize() const; + + private: + ContextType type; + uint history_size; + }; + + IPSConfiguration() {} + IPSConfiguration(const std::map &initial_conf) : context_config(initial_conf) {} + + void load(cereal::JSONInputArchive &ar); + + Context getContext(const std::string &name) const; + uint getHistorySize(const std::string &name) const; + +private: + std::map context_config; +}; + +#endif // __IPS_CONFIGURATION_H__ diff --git a/components/security_apps/ips/include/ips_entry.h b/components/security_apps/ips/include/ips_entry.h new file mode 100644 index 0000000..4c23518 --- /dev/null +++ b/components/security_apps/ips/include/ips_entry.h @@ -0,0 +1,56 @@ +#ifndef __IPS_ENTRY_H__ +#define __IPS_ENTRY_H__ + +#include +#include + +#include "table_opaque.h" +#include "parsed_context.h" +#include "buffer.h" +#include "context.h" + +class IPSEntry : public TableOpaqueSerialize, public Listener +{ +public: + IPSEntry(); + + void upon(const ParsedContext &) override; + ParsedContextReply respond(const ParsedContext &ctx) override; + std::string getListenerName() const override { return name(); } + + template + void serialize(T &, uint32_t) {} + static std::string name(); + static std::unique_ptr prototype(); + static uint currVer(); + static uint minVer(); + + void uponEnteringContext() override { registerListener(); } + void uponLeavingContext() override { unregisterListener(); } + + void setFlag(const std::string &flag) { flags.insert(flag); } + void unsetFlag(const std::string &flag) { flags.erase(flag); } + bool isFlagSet(const std::string &flag) const { return flags.count(flag) != 0; } + + Buffer getBuffer(const std::string &name) const; + void setTransactionData(const Buffer &key, const Buffer &value); + Maybe getTransactionData(const Buffer &key) const; + + void addPendingContext(const std::string &name, const Buffer &buffer); + const std::vector> getPendingContexts() const { return pending_contexts; } + void clearPendingContexts() { pending_contexts.clear(); } + + void setDrop() { is_drop = true; } + bool isDrop() const { return is_drop; } + +private: + std::map past_contexts; + std::set flags; + Context ctx; + std::map transaction_data; + std::vector> pending_contexts; + + bool is_drop = false; +}; + +#endif // __IPS_ENTRY_H__ diff --git a/components/security_apps/ips/include/ips_enums.h b/components/security_apps/ips/include/ips_enums.h new file mode 100644 index 0000000..71c06f0 --- /dev/null +++ b/components/security_apps/ips/include/ips_enums.h @@ -0,0 +1,26 @@ +#ifndef __IPS_ENUMS_H__ +#define __IPS_ENUMS_H__ + +namespace IPSSignatureSubTypes +{ + +enum class SignatureAction +{ + PREVENT, + DETECT, + IGNORE +}; + +enum class IPSLevel +{ + VERY_LOW, + LOW, + MEDIUM_LOW, + MEDIUM, + MEDIUM_HIGH, + HIGH, + CRITICAL +}; + +} // IPSSignatureSubTypes +#endif // __IPS_ENUMS_H__ diff --git a/components/security_apps/ips/include/ips_metric.h b/components/security_apps/ips/include/ips_metric.h new file mode 100644 index 0000000..7283c71 --- /dev/null +++ b/components/security_apps/ips/include/ips_metric.h @@ -0,0 +1,35 @@ +#ifndef __IPS_METRIC_H__ +#define __IPS_METRIC_H__ + +#include "ips_signatures.h" +#include "generic_metric.h" + +namespace IPSSignatureSubTypes +{ + +class MatchEvent : public Event +{ +public: + MatchEvent(const std::shared_ptr &sig, SignatureAction act) : signature(sig), action(act) {} + + const SignatureAction & getAction() const { return action; } + +private: + std::shared_ptr signature; + SignatureAction action; +}; + +class IPSMetric : public GenericMetric, public Listener +{ +public: + void upon(const MatchEvent &event) override; + +private: + MetricCalculations::Counter prevented{this, "preventEngineMatchesSample"}; + MetricCalculations::Counter detected{this, "detectEngineMatchesSample"}; + MetricCalculations::Counter ignored{this, "ignoreEngineMatchesSample"}; +}; + +} // IPSSignatureSubTypes + +#endif // __IPS_METRIC_H__ diff --git a/components/security_apps/ips/include/ips_signatures.h b/components/security_apps/ips/include/ips_signatures.h new file mode 100644 index 0000000..9b9bb1d --- /dev/null +++ b/components/security_apps/ips/include/ips_signatures.h @@ -0,0 +1,238 @@ +#ifndef __IPS_SIGNATURES_H__ +#define __IPS_SIGNATURES_H__ + +#include + +#include "config.h" +#include "parsed_context.h" +#include "log_generator.h" +#include "pm_hook.h" +#include "ips_enums.h" +#include "ips_entry.h" +#include "i_first_tier_agg.h" + +namespace IPSSignatureSubTypes +{ +using ActionResults = std::tuple>; + +class BaseSignature +{ +public: + enum class MatchType { NO_MATCH, CACHE_MATCH, MATCH }; + + virtual const std::string & getSigId() const = 0; + virtual MatchType getMatch(const std::set &matched) const = 0; + virtual std::set patternsInSignature() const = 0; + virtual const std::vector & getContext() const = 0; +}; + +class IPSSignatureMetaData +{ +public: + void load(cereal::JSONInputArchive &ar); + void setIndicators(const std::string &source, const std::string &version); + + const std::string & getId() const { return protection_id; } + const std::string & getName() const { return sig_name; } + const std::string & getUpdateVersion() const { return update; } + const std::string & getLogTitle() const { return event_log; } + const std::string & getSource() const { return source; } + const std::string & getFeedVersion() const { return version; } + const std::vector & getCveList() const { return cve_list; } + IPSLevel getSeverity() const { return severity; } + std::string getSeverityString() const; + IPSLevel getConfidence() const { return confidence; } + std::string getConfidenceString() const; + IPSLevel getPerformance() const { return performance; } + std::string getPerformanceString() const; + bool isSilent() const { return is_silent; } + std::string getIncidentType() const; + bool isYearAtLeast(const Maybe &year) const; + Maybe getYear() const; + +private: + std::string protection_id; + std::string sig_name; + std::string event_log; + std::string update; + std::string source; + std::string version; + std::vector cve_list; + std::vector tag_list; + IPSLevel severity; + IPSLevel confidence; + IPSLevel performance; + bool is_silent = false; +}; + +class CompleteSignature +{ +public: + void load(cereal::JSONInputArchive &ar); + BaseSignature::MatchType getMatch(const std::set &matches) const; + std::set patternsInSignature() const; + void setIndicators(const std::string &source, const std::string &version); + + const std::vector & getContext() const { return rule->getContext(); } + const std::string & getId() const { return metadata.getId(); } + const std::string & getLogTitle() const { return metadata.getLogTitle(); } + const std::string & getName() const { return metadata.getName(); } + const std::string & getUpdateVersion() const { return metadata.getUpdateVersion(); } + const std::string & getSource() const { return metadata.getSource(); } + const std::string & getFeedVersion() const { return metadata.getFeedVersion(); } + const std::vector & getCveList() const { return metadata.getCveList(); } + IPSLevel getSeverity() const { return metadata.getSeverity(); } + std::string getSeverityString() const { return metadata.getSeverityString(); } + IPSLevel getConfidence() const { return metadata.getConfidence(); } + std::string getConfidenceString() const { return metadata.getConfidenceString(); } + IPSLevel getPerformance() const { return metadata.getPerformance(); } + std::string getPerformanceString() const { return metadata.getPerformanceString(); } + bool isSilent() const { return metadata.isSilent(); } + std::string getIncidentType() const { return metadata.getIncidentType(); } + + bool isYearAtLeast(const Maybe &year) const { return metadata.isYearAtLeast(year); } + Maybe getYear() const { return metadata.getYear(); } + +private: + IPSSignatureMetaData metadata; + std::shared_ptr rule; +}; + +class SignatureAndAction +{ +public: + SignatureAndAction(std::shared_ptr _signature, SignatureAction _action) + : + signature(_signature), + action(_action) + { + } + + bool isMatchedPrevent(const Buffer &context_buffer, const std::set &pattern) const; + bool matchSilent(const Buffer &context_buffer) const; + std::set patternsInSignature() const { return signature->patternsInSignature(); } + const std::vector & getContext() const { return signature->getContext(); } + +private: + ActionResults getAction(const IPSEntry &ips_state) const; + std::shared_ptr signature; + SignatureAction action; +}; +} // IPSSignatureSubTypes + +class IPSSignaturesPerContext : public Singleton::Consume +{ +public: + void addSignature(const IPSSignatureSubTypes::SignatureAndAction &sig); + bool isMatchedPrevent(const Buffer &context_buffer) const; + void calcFirstTier(const std::string &ctx_name); + +private: + std::set getFirstTierMatches(const Buffer &buffer) const; + + std::map> signatures_per_lss; + std::vector signatures_without_lss; + std::shared_ptr first_tier; +}; + +class IPSSignaturesResource +{ +public: + void load(cereal::JSONInputArchive &ar); + + const std::vector> & + getSignatures() const + { + return all_signatures; + } + +private: + std::vector> all_signatures; +}; + +class SnortSignaturesResourceFile +{ +public: + void load(cereal::JSONInputArchive &ar); + bool isFile(const std::string &file_name) const { return file_name == name; } + const std::vector> & + getSignatures() const + { + return all_signatures; + } + +private: + std::string name; + std::vector> all_signatures; +}; + +class SnortSignaturesResource +{ +public: + void load(cereal::JSONInputArchive &ar); + + const std::vector> & + getSignatures(const std::string &file_name) const + { + for (auto &file : files) { + if (file.isFile(file_name)) return file.getSignatures(); + } + return empty; + } + +private: + std::vector> empty; + std::vector files; +}; + +class IPSSignatures +{ + std::set getFirstTier(const ParsedContext &context); + +public: + void load(cereal::JSONInputArchive &ar); + bool isMatchedPrevent(const std::string &context_name, const Buffer &context_buffer) const; + bool isEmpty() const { return signatures_per_context.empty(); } + bool isEmpty(const std::string &context) const; + + const std::string & getAsset() const { return asset_name; } + const std::string & getAssetId() const { return asset_id; } + const std::string & getPractice() const { return practice_name; } + const std::string & getPracticeId() const { return practice_id; } + const std::string & getSourceIdentifier() const { return source_id; } + +private: + std::map signatures_per_context; + std::string asset_name; + std::string asset_id; + std::string practice_name; + std::string practice_id; + std::string source_id; +}; + +class SnortSignatures +{ + std::set getFirstTier(const ParsedContext &context); + +public: + void load(cereal::JSONInputArchive &ar); + bool isMatchedPrevent(const std::string &context_name, const Buffer &context_buffer) const; + bool isEmpty() const { return signatures_per_context.empty(); } + bool isEmpty(const std::string &context) const; + + const std::string & getAsset() const { return asset_name; } + const std::string & getAssetId() const { return asset_id; } + const std::string & getPractice() const { return practice_name; } + const std::string & getPracticeId() const { return practice_id; } + const std::string & getSourceIdentifier() const { return source_id; } + +private: + std::map signatures_per_context; + std::string asset_name; + std::string asset_id; + std::string practice_name; + std::string practice_id; + std::string source_id; +}; + +#endif // __IPS_SIGNATURES_H__ diff --git a/components/security_apps/ips/include/rule_detection.h b/components/security_apps/ips/include/rule_detection.h new file mode 100644 index 0000000..a15050a --- /dev/null +++ b/components/security_apps/ips/include/rule_detection.h @@ -0,0 +1,30 @@ +#ifndef __RULE_DETECTION_H__ +#define __RULE_DETECTION_H__ + +#include "simple_protection.h" +#include "compound_protection.h" + +class RuleDetection +{ +public: + RuleDetection(const std::string &_sig_name) : sig_name(_sig_name) {} + + template + void serialize(T &ar) + { + std::string type; + ar(cereal::make_nvp("type", type)); + + if (type == "simple") rule = SimpleProtection::get(sig_name, ar); + else if (type == "compound") rule = CompoundProtection::get(sig_name, ar); + else reportConfigurationError("Unknown rule type: " + type); + }; + + std::shared_ptr getRule() { return rule; } + +private: + std::shared_ptr rule; + std::string sig_name; +}; + +#endif // __RULE_DETECTION_H__ diff --git a/components/security_apps/ips/include/simple_protection.h b/components/security_apps/ips/include/simple_protection.h new file mode 100644 index 0000000..076e469 --- /dev/null +++ b/components/security_apps/ips/include/simple_protection.h @@ -0,0 +1,49 @@ +#ifndef __SIMPLE_PROTECTION_H__ +#define __SIMPLE_PROTECTION_H__ + +#include + +#include "ips_signatures.h" +#include "i_keywords_rule.h" + +class SimpleProtection +{ + class Impl : public IPSSignatureSubTypes::BaseSignature + { + public: + Impl( + const std::string &sig_name, + const std::string &ssm, + const std::string &keyword, + const std::vector &context + ); + + const std::string & getSigId() const override { return sig_name; } + MatchType getMatch(const std::set &matched) const override; + std::set patternsInSignature() const override; + const std::vector & getContext() const override { return context; } + + private: + std::string sig_name; + std::vector context; + std::shared_ptr rule; + PMPattern pattern; + }; + +public: + template + static std::shared_ptr get(const std::string &sig_name, Archive &ar) + { + std::string ssm, keyword; + std::vector context; + + ar( + cereal::make_nvp("SSM", ssm), + cereal::make_nvp("keywords", keyword), + cereal::make_nvp("context", context) + ); + + return std::make_shared(sig_name, ssm, keyword, context); + } +}; +#endif // __SIMPLE_PROTECTION_H__ diff --git a/components/security_apps/ips/include/snort_basic_policy.h b/components/security_apps/ips/include/snort_basic_policy.h new file mode 100644 index 0000000..96d11b2 --- /dev/null +++ b/components/security_apps/ips/include/snort_basic_policy.h @@ -0,0 +1,22 @@ +#ifndef __SNORT_BASIC_POLICY_H__ +#define __SNORT_BASIC_POLICY_H__ + +#include +#include +#include + +#include "ips_enums.h" +#include "ips_signatures.h" + +class SnortRuleSelector +{ +public: + std::vector selectSignatures() const; + void load(cereal::JSONInputArchive &ar); + +private: + IPSSignatureSubTypes::SignatureAction action = IPSSignatureSubTypes::SignatureAction::IGNORE; + std::vector file_names; +}; + +#endif // __SNORT_BASIC_POLICY_H__ diff --git a/components/security_apps/ips/ips_basic_policy.cc b/components/security_apps/ips/ips_basic_policy.cc new file mode 100644 index 0000000..25d7eb8 --- /dev/null +++ b/components/security_apps/ips/ips_basic_policy.cc @@ -0,0 +1,254 @@ +#include "ips_basic_policy.h" +#include +#include +#include +#include +#include +#include "ips_signatures.h" +#include "helper.h" +#include "config.h" +#include "common.h" + +USE_DEBUG_FLAG(D_IPS); + +using namespace std; + +void +RuleSelector::load(cereal::JSONInputArchive &ar) +{ + readRules(ar); + readDefaultAction(ar); +} + +vector +RuleSelector::selectSignatures() const +{ + vector res; + + auto all_signatures = getResource("IPS", "protections"); + if (!all_signatures.ok()) return res; + auto signatures_version = getResourceWithDefault("", "IPS", "VersionId"); + + for (auto &signature : (*all_signatures).getSignatures()) { + for (auto &rule : rules) { + if (rule.isSignaturedMatched(*signature)) { + if (rule.getAction() != IPSSignatureSubTypes::SignatureAction::IGNORE) { + signature->setIndicators("Check Point", signatures_version); + res.emplace_back(signature, rule.getAction()); + } + break; + } + } + } + + return res; +} + +void +RuleSelector::readRules(cereal::JSONInputArchive &ar) +{ + ar(cereal::make_nvp("rules", rules)); +} + +void +RuleSelector::readDefaultAction(cereal::JSONInputArchive &ar) +{ + Rule rule; + rule.readAction(ar, "defaultAction"); + rules.push_back(rule); +} + +void +RuleSelector::Rule::serialize(cereal::JSONInputArchive &ar) +{ + readAction(ar, "action"); + readPerformanceImpact(ar); + readSeverityLevel(ar); + readConfidenceLevel(ar); + readServerProtections(ar); + readClientProtections(ar); + readProtectionsFromYear(ar); + readProtectionTags(ar); + readProtectionIds(ar); +} + +bool +RuleSelector::Rule::isSignaturedMatched(const IPSSignatureSubTypes::CompleteSignature &signature) const +{ + if (confidence_level.ok() && signature.getConfidence() != confidence_level.unpack()) return false; + if (severity_level.ok() && signature.getSeverity() < severity_level.unpack()) return false; + if (performance_impact.ok() && signature.getPerformance() > performance_impact.unpack()) return false; + if (!signature.isYearAtLeast(protections_from_year)) return false; + + return true; +} + +static ostream & +operator<<(ostream &os, const RuleSelector::Rule & rule) +{ + rule.print(os); + return os; +} + +void +RuleSelector::Rule::print(ostream &os) const +{ + os << "[Rule] " << "action: " << static_cast(action); + if (performance_impact.ok()) { + os << " performanceImpact: " << static_cast(performance_impact.unpack()); + } + if (severity_level.ok()) { + os << " severityLevel: " << static_cast(severity_level.unpack()); + } + if (confidence_level.ok()) { + os << " confidenceLevel: " << static_cast(confidence_level.unpack()); + } + if (server_protections.ok()) { + os << boolalpha << " serverProtections: " << server_protections.unpack(); + } + if (client_protections.ok()) { + os << boolalpha << " clientProtections: " << client_protections.unpack(); + } + if (protections_from_year.ok()) { + os << " protectionsFromYear: " << protections_from_year.unpack(); + } + if (protection_ids.ok()) { + os << " protectionIds: " << makeSeparatedStr(protection_ids.unpack(), ", "); + } + if (protection_tags.ok()) { + os << " protectionTags: " << makeSeparatedStr(protection_tags.unpack(), ", "); + } +} + +void +RuleSelector::Rule::readAction(cereal::JSONInputArchive &ar, const string &action_type) +{ + string str; + ar(cereal::make_nvp(action_type, str)); + + if (str == "Inactive") action = IPSSignatureSubTypes::SignatureAction::IGNORE; + else if (str == "Detect") action = IPSSignatureSubTypes::SignatureAction::DETECT; + else if (str == "Prevent") action = IPSSignatureSubTypes::SignatureAction::PREVENT; + else reportConfigurationError("invalid action value " + str); +} + +void +RuleSelector::Rule::readPerformanceImpact(cereal::JSONInputArchive &ar) +{ + try { + string str; + ar(cereal::make_nvp("performanceImpact", str)); + + if (str == "Very low") performance_impact = IPSSignatureSubTypes::IPSLevel::VERY_LOW; + else if (str == "Low or lower") performance_impact = IPSSignatureSubTypes::IPSLevel::LOW; + else if (str == "Medium or lower") performance_impact = IPSSignatureSubTypes::IPSLevel::MEDIUM; + else if (str == "High or lower") performance_impact = IPSSignatureSubTypes::IPSLevel::HIGH; + else reportConfigurationError("invalid performanceImpact value " + str); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readSeverityLevel(cereal::JSONInputArchive &ar) +{ + try { + string str; + ar(cereal::make_nvp("severityLevel", str)); + + if (str == "Critical") severity_level = IPSSignatureSubTypes::IPSLevel::CRITICAL; + else if (str == "High or above") severity_level = IPSSignatureSubTypes::IPSLevel::HIGH; + else if (str == "Medium or above") severity_level = IPSSignatureSubTypes::IPSLevel::MEDIUM; + else if (str == "Low or above") severity_level = IPSSignatureSubTypes::IPSLevel::LOW; + else reportConfigurationError("invalid severityLevel value " + str); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readConfidenceLevel(cereal::JSONInputArchive &ar) +{ + try { + string str; + ar(cereal::make_nvp("confidenceLevel", str)); + + if (str == "Low") confidence_level = IPSSignatureSubTypes::IPSLevel::LOW; + else if (str == "Medium") confidence_level = IPSSignatureSubTypes::IPSLevel::MEDIUM; + else if (str == "High") confidence_level = IPSSignatureSubTypes::IPSLevel::HIGH; + else reportConfigurationError("invalid confidenceLevel value " + str); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readServerProtections(cereal::JSONInputArchive &ar) +{ + try { + bool _server_protections; + ar(cereal::make_nvp("serverProtections", _server_protections)); + server_protections = _server_protections; + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readClientProtections(cereal::JSONInputArchive &ar) +{ + try { + bool _client_protections; + ar(cereal::make_nvp("clientProtections", _client_protections)); + client_protections = _client_protections; + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readProtectionsFromYear(cereal::JSONInputArchive &ar) +{ + try { + int year; + ar(cereal::make_nvp("protectionsFromYear", year)); + if (year < 1999 || year > 2021) { + reportConfigurationError("invalid protectionsFromYear value " + to_string(year)); + } + else { + protections_from_year = year; + } + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readProtectionTags(cereal::JSONInputArchive &ar) +{ + try { + vector _protection_tags; + ar(cereal::make_nvp("protectionTags", _protection_tags)); + protection_tags = _protection_tags; + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::Rule::readProtectionIds(cereal::JSONInputArchive &ar) +{ + try { + vector _protection_ids; + ar(cereal::make_nvp("protectionIds", _protection_ids)); + protection_ids = _protection_ids; + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + } +} + +void +RuleSelector::print(ostream &os) const +{ + os << makeSeparatedStr(rules, ";"); +} diff --git a/components/security_apps/ips/ips_common_types.cc b/components/security_apps/ips/ips_common_types.cc new file mode 100644 index 0000000..ad0e3b1 --- /dev/null +++ b/components/security_apps/ips/ips_common_types.cc @@ -0,0 +1,4 @@ +#include "ips_common_types.h" + +const Buffer IPSCommonTypes::requests_header_for_log = + Buffer("agent-found-Requests-header-for-log", 35, Buffer::MemoryType::STATIC); diff --git a/components/security_apps/ips/ips_comp.cc b/components/security_apps/ips/ips_comp.cc new file mode 100644 index 0000000..125c143 --- /dev/null +++ b/components/security_apps/ips/ips_comp.cc @@ -0,0 +1,392 @@ +#include "ips_comp.h" + +#include + +#include "debug.h" +#include "new_table_entry.h" +#include "ips_entry.h" +#include "ips_configuration.h" +#include "ips_signatures.h" +#include "ips_metric.h" +#include "generic_rulebase/parameters_config.h" +#include "config.h" +#include "virtual_modifiers.h" +#include "helper.h" +#include "ips_common_types.h" +#include "nginx_attachment_common.h" + +using namespace std; + +USE_DEBUG_FLAG(D_IPS); + +static const Buffer header_sep(": ", 2, Buffer::MemoryType::STATIC); +static const Buffer line_sep("\r\n", 2, Buffer::MemoryType::STATIC); +static const Buffer space(" ", 1, Buffer::MemoryType::STATIC); +static const Buffer x_forworded_for_key("x-forworded-for", 15, Buffer::MemoryType::STATIC); +static const Buffer log_sep(", ", 2, Buffer::MemoryType::STATIC); +static const Buffer empty_buffer("", 0, Buffer::MemoryType::STATIC); + +static const string cookie("cookie"); +static const string oauth("_oauth2_proxy"); +static const string jsessionid("jsessionid"); +static const string xff("x-forwarded-for"); +static const string header("header"); +static const string source_ip("source ip"); + +class IPSComp::Impl + : + public Singleton::Provide::SelfInterface, + public Listener, + public Listener, + public Listener, + public Listener, + public Listener, + public Listener, + public Listener, + public Listener, + public Listener +{ + static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + + class SigsFirstTierAgg + { + public: + const shared_ptr & + getHook(const set &new_pat) + { + auto old_size = pats.size(); + pats.insert(new_pat.begin(), new_pat.end()); + + if (pats.size() != old_size) { + if (!hook->prepare(pats).ok()) { + reportConfigurationError("failed to compile first tier"); + } + } + + return hook; + } + + private: + set pats; + shared_ptr hook = make_shared(); + }; + +public: + void + preload() + { + function cb = [&](){ clearAggCache(); }; + registerConfigPrepareCb(cb); + registerConfigLoadCb(cb); + registerConfigAbortCb(cb); + } + + void + init() + { + ips_metric.init( + "IPS Stats", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + std::chrono::minutes(10), + true, + ReportIS::Audience::SECURITY + ); + ips_metric.registerListener(); + registerListener(); + table = Singleton::Consume::by(); + env = Singleton::Consume::by(); + } + + void + fini() + { + unregisterListener(); + } + + void + upon(const NewTableEntry &) override + { + if (isSignatureListsEmpty()) return; + auto table = Singleton::Consume::by(); + table->createState(); + table->getState().uponEnteringContext(); + } + + string getListenerName() const override { return "ips application"; } + + EventVerdict + respond(const NewHttpTransactionEvent &event) override + { + if (isSignatureListsEmpty()) return ACCEPT; + table->createState(); + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + Buffer method(event.getHttpMethod()); + ips_state.addPendingContext("HTTP_METHOD", method); + + Buffer uri(event.getURI()); + ips_state.addPendingContext("HTTP_COMPLETE_URL_ENCODED", uri); + + auto decoder = makeVirtualContainer>(event.getURI()); + vector decoded_url(decoder.begin(), decoder.end()); + auto start = find(decoded_url.begin(), decoded_url.end(), static_cast('?')); + + if (start != decoded_url.end()) { + vector query(start + 1, decoded_url.end()); + ips_state.addPendingContext("HTTP_QUERY_DECODED", Buffer(move(query))); + } + vector path(decoded_url.begin(), start); + ips_state.addPendingContext("HTTP_PATH_DECODED", Buffer(move(path))); + ips_state.addPendingContext("HTTP_COMPLETE_URL_DECODED", Buffer(move(decoded_url))); + + Buffer protocol(event.getHttpProtocol()); + ips_state.addPendingContext("HTTP_PROTOCOL", protocol); + + auto full_line = method + space + uri + space + protocol + line_sep; + ips_state.addPendingContext("HTTP_RAW", full_line); + + return INSPECT; + } + + static string + getHeaderContextName(const Buffer &name) + { + string name_str = name; + transform(name_str.begin(), name_str.end(), name_str.begin(), ::toupper); + return "HTTP_REQUEST_HEADER_" + name_str; + } + + EventVerdict + respond(const HttpRequestHeaderEvent &event) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + auto header_value = event.getKey() + header_sep + event.getValue(); + ips_state.addPendingContext("HTTP_REQUEST_ONE_HEADER", header_value); + auto full_header = header_value + line_sep; + ips_state.addPendingContext("HTTP_REQUEST_HEADER", full_header); + ips_state.addPendingContext(getHeaderContextName(event.getKey()), event.getValue()); + ips_state.addPendingContext("HTTP_RAW", full_header); + + auto max_size = getConfigurationWithDefault(1536, "IPS", "Max Field Size"); + + // Add request header for log + auto maybe_req_headers_for_log = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log); + if (!maybe_req_headers_for_log.ok()) { + ips_state.setTransactionData(IPSCommonTypes::requests_header_for_log, header_value); + } else if ((maybe_req_headers_for_log.unpack()).size() + log_sep.size() + header_value.size() < max_size) { + Buffer request_headers_for_log = maybe_req_headers_for_log.unpack() + log_sep; + request_headers_for_log += header_value; + ips_state.setTransactionData(IPSCommonTypes::requests_header_for_log, request_headers_for_log); + } + + addRequestHdr(event.getKey(), event.getValue()); + if (event.isLastHeader()) { + for (auto &context : ips_state.getPendingContexts()) { + if (isDropContext(context.first, context.second)) setDrop(ips_state); + } + ips_state.clearPendingContexts(); + if (isDrop(ips_state)) return DROP; + } + + return INSPECT; + } + + EventVerdict + respond(const HttpRequestBodyEvent &event) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + if (isDropContext("HTTP_REQUEST_BODY", event.getData())) setDrop(ips_state); + + if (!ips_state.isFlagSet("HttpRequestData")) { + ips_state.setFlag("HttpRequestData"); + auto data = + ips_state.getBuffer("HTTP_METHOD") + + space + + ips_state.getBuffer("HTTP_COMPLETE_URL_DECODED") + + space + + ips_state.getBuffer("HTTP_PROTOCOL") + + line_sep + + ips_state.getBuffer("HTTP_REQUEST_HEADER") + + line_sep + + event.getData(); + if (isDropContext("HTTP_REQUEST_DATA", data)) setDrop(ips_state); + } + + if (isDropContext("HTTP_RAW", event.getData())) setDrop(ips_state); + + return INSPECT; + } + + EventVerdict + respond(const EndRequestEvent &) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + if (!ips_state.isFlagSet("HttpRequestData")) { + ips_state.setFlag("HttpRequestData"); + auto data = + ips_state.getBuffer("HTTP_METHOD") + + space + + ips_state.getBuffer("HTTP_COMPLETE_URL_DECODED") + + space + + ips_state.getBuffer("HTTP_PROTOCOL") + + line_sep + + ips_state.getBuffer("HTTP_REQUEST_HEADER") + + line_sep; + if (isDropContext("HTTP_REQUEST_DATA", data)) return DROP; + } + + if (isDrop(ips_state)) return DROP; + + if (isContextActive("HTTP_RESPONSE_HEADER")) return INSPECT; + if (isContextActive("HTTP_RESPONSE_BODY")) return INSPECT; + return ACCEPT; + } + + EventVerdict + respond(const ResponseCodeEvent &event) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + Buffer buf(reinterpret_cast(&event), sizeof(event), Buffer::MemoryType::VOLATILE); + if (isDropContext("HTTP_RESPONSE_CODE", buf)) return DROP; + + return INSPECT; + } + + EventVerdict + respond(const HttpResponseHeaderEvent &event) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + if (isDropContext("HTTP_RESPONSE_HEADER", event.getValue())) return DROP; + + return INSPECT; + } + + EventVerdict + respond(const HttpResponseBodyEvent &event) override + { + if (!table->hasState()) return ACCEPT; + + auto &ips_state = table->getState(); + ips_state.uponEnteringContext(); + auto leave_context = make_scope_exit([&ips_state] () { ips_state.uponLeavingContext(); }); + + if (isDropContext("HTTP_RESPONSE_BODY", event.getData())) return DROP; + + return event.isLastChunk() ? ACCEPT : INSPECT; + } + + EventVerdict respond (const EndTransactionEvent &) override { return ACCEPT; } + +private: + static void setDrop(IPSEntry &state) { state.setDrop(); } + static bool isDrop(const IPSEntry &state) { return state.isDrop(); } + + bool + isDropContext(const string &name, const Buffer &buf) + { + auto responeses = ParsedContext(buf, name, 0).query(); + for (auto &reponse : responeses) { + if (reponse == ParsedContextReply::DROP) return true; + } + return false; + } + + static bool + isContextActive(const string &context) + { + return !getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections").isEmpty(context) || + !getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections").isEmpty(context); + } + + static bool + isSignatureListsEmpty() + { + return getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections").isEmpty() && + getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections").isEmpty(); + } + + void + addRequestHdr(const Buffer &name, const Buffer &value) + { + auto &ips_state = table->getState(); + ips_state.setTransactionData(name, value); + } + + shared_ptr + getHook(const string &context_name, const set &patterns) override + { + return tier_aggs[context_name].getHook(patterns); + } + + void clearAggCache() { tier_aggs.clear(); } + + I_Table *table = nullptr; + I_Environment *env = nullptr; + IPSSignatureSubTypes::IPSMetric ips_metric; + map tier_aggs; +}; + +IPSComp::IPSComp() : Component("IPSComp"), pimpl(make_unique()) {} + +IPSComp::~IPSComp() {} + +void +IPSComp::preload() +{ + registerExpectedResource("IPS", "protections"); + registerExpectedResource("IPS", "VersionId"); + registerExpectedResource("IPSSnortSigs", "protections"); + registerExpectedConfiguration("IPS", "IpsConfigurations"); + registerExpectedConfiguration("IPS", "Max Field Size"); + registerExpectedConfiguration("IPS", "IpsProtections"); + registerExpectedConfiguration("IPSSnortSigs", "SnortProtections"); + registerExpectedConfigFile("ips", Config::ConfigFileType::Policy); + registerExpectedConfigFile("ips", Config::ConfigFileType::Data); + registerExpectedConfigFile("snort", Config::ConfigFileType::Policy); + + ParameterException::preload(); + + pimpl->preload(); +} + +void +IPSComp::init() +{ + pimpl->init(); +} + +void +IPSComp::fini() +{ + pimpl->fini(); +} diff --git a/components/security_apps/ips/ips_configuration.cc b/components/security_apps/ips/ips_configuration.cc new file mode 100644 index 0000000..80f4bf1 --- /dev/null +++ b/components/security_apps/ips/ips_configuration.cc @@ -0,0 +1,74 @@ +#include "ips_configuration.h" +#include "debug.h" + +using namespace std; + +IPSConfiguration::Context::Context(ContextType _type, uint history) : type(_type), history_size(history) {} + +uint +IPSConfiguration::Context::getHistorySize() const +{ + dbgAssert(type == ContextType::HISTORY) << "Try to access history size for non-history context"; + return history_size; +} + +static const map type_convertor = { + { "normal", IPSConfiguration::ContextType::NORMAL }, + { "keep", IPSConfiguration::ContextType::KEEP }, + { "history", IPSConfiguration::ContextType::HISTORY } +}; + +class ContextConfigurationJSON +{ +public: + void + load(cereal::JSONInputArchive &ar) + { + string type_name; + ar( + cereal::make_nvp("name", name), + cereal::make_nvp("type", type_name) + ); + + auto type_pointer = type_convertor.find(type_name); + if (type_pointer == type_convertor.end()) reportConfigurationError("Unknown IPS context type: " + type_name); + type = type_pointer->second; + + if (type == IPSConfiguration::ContextType::HISTORY) ar(cereal::make_nvp("historySize", size)); + } + + string getName() const { return name; } + IPSConfiguration::Context getContext() const { return IPSConfiguration::Context(type, size); } + +private: + string name; + IPSConfiguration::ContextType type; + uint size = 0; +}; + +void +IPSConfiguration::load(cereal::JSONInputArchive &ar) +{ + vector config; + ar(cereal::make_nvp("contextsConfiguration", config)); + + for (auto &context : config) { + context_config.emplace(context.getName(), context.getContext()); + } +} + +IPSConfiguration::Context +IPSConfiguration::getContext(const string &name) const +{ + auto context = context_config.find(name); + if (context == context_config.end()) return IPSConfiguration::Context(); + return context->second; +} + +uint +IPSConfiguration::getHistorySize(const string &name) const +{ + auto context = context_config.find(name); + dbgAssert(context != context_config.end()) << "Try to access history size for non-exiting context"; + return context->second.getHistorySize(); +} diff --git a/components/security_apps/ips/ips_entry.cc b/components/security_apps/ips/ips_entry.cc new file mode 100644 index 0000000..66345b2 --- /dev/null +++ b/components/security_apps/ips/ips_entry.cc @@ -0,0 +1,143 @@ +#include "ips_entry.h" +#include "ips_signatures.h" +#include "ips_configuration.h" +#include "config.h" +#include "debug.h" +#include "common.h" +#include "i_keywords_rule.h" +#include "helper.h" + +using namespace std; +using namespace IPSHelper; + +USE_DEBUG_FLAG(D_IPS); + +static const map default_conf_mapping = { + { "HTTP_METHOD", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_COMPLETE_URL_DECODED", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_PATH_DECODED", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_QUERY_DECODED", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_PROTOCOL", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_REQUEST_HEADER", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_REQUEST_BODY", IPSConfiguration::Context(IPSConfiguration::ContextType::HISTORY, 1000) }, + { "HTTP_RESPONSE_CODE", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_RESPONSE_HEADER", IPSConfiguration::Context(IPSConfiguration::ContextType::KEEP, 0) }, + { "HTTP_RESPONSE_BODY", IPSConfiguration::Context(IPSConfiguration::ContextType::HISTORY, 1000) } +}; + +static const IPSConfiguration default_conf(default_conf_mapping); + +IPSEntry::IPSEntry() : TableOpaqueSerialize(this) {} + +void +IPSEntry::upon(const ParsedContext &) +{ +} + +ParsedContextReply +IPSEntry::respond(const ParsedContext &parsed) +{ + const auto &name = parsed.getName(); + auto buf = parsed.getBuffer(); + + dbgDebug(D_IPS) << "Entrying context " << name; + dbgTrace(D_IPS) << "Context Content " << dumpHex(buf); + + auto config = getConfigurationWithDefault(default_conf, "IPS", "IpsConfigurations").getContext(name); + if (config.getType() == IPSConfiguration::ContextType::HISTORY) { + buf = past_contexts[name] + buf; + } + ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), name); + ctx.registerValue(name, buf); + + ctx.activate(); + auto &signatures = getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections"); + bool should_drop = signatures.isMatchedPrevent(parsed.getName(), buf); + auto &snort_signatures = getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections"); + should_drop |= snort_signatures.isMatchedPrevent(parsed.getName(), buf); + ctx.deactivate(); + + switch(config.getType()) { + case IPSConfiguration::ContextType::NORMAL: { + ctx.unregisterKey(name); + break; + } + case IPSConfiguration::ContextType::KEEP: { + past_contexts[name] += buf; + ctx.registerValue(name, past_contexts[name]); + break; + } + case IPSConfiguration::ContextType::HISTORY: { + if (buf.size() > config.getHistorySize()) buf.keepTail(config.getHistorySize()); + ctx.registerValue(name, buf); + past_contexts[name] = buf; + break; + } + } + + dbgDebug(D_IPS) << "Return " << (should_drop ? "drop" : "continue"); + + return should_drop ? ParsedContextReply::DROP : ParsedContextReply::ACCEPT; +} + +Buffer +IPSEntry::getBuffer(const string &name) const +{ + auto elem = past_contexts.find(name); + if (elem != past_contexts.end()) return elem->second; + + for (auto &p : pending_contexts) { + if (p.first == name) return p.second; + } + + return Buffer(); +} + +void +IPSEntry::setTransactionData(const Buffer &key, const Buffer &value) +{ + transaction_data[key] = value; +} + +Maybe +IPSEntry::getTransactionData(const Buffer &key) const +{ + map::const_iterator iter = transaction_data.find(key); + + if (iter == transaction_data.end()) { + return genError("Http header value not found"); + } + + return iter->second; + +} + +string +IPSEntry::name() +{ + return "IPS"; +} + +unique_ptr +IPSEntry::prototype() +{ + return make_unique(); +} + +uint +IPSEntry::currVer() +{ + return 0; +} + +uint +IPSEntry::minVer() +{ + return 0; +} + +void +IPSEntry::addPendingContext(const std::string &name, const Buffer &buffer) +{ + pending_contexts.emplace_back(name, buffer); +} diff --git a/components/security_apps/ips/ips_metric.cc b/components/security_apps/ips/ips_metric.cc new file mode 100644 index 0000000..f6ba767 --- /dev/null +++ b/components/security_apps/ips/ips_metric.cc @@ -0,0 +1,20 @@ +#include "ips_metric.h" + +void +IPSSignatureSubTypes::IPSMetric::upon(const MatchEvent &event) +{ + switch (event.getAction()) { + case SignatureAction::PREVENT: { + prevented.report(1); + break; + } + case SignatureAction::DETECT: { + detected.report(1); + break; + } + case SignatureAction::IGNORE: { + ignored.report(1); + break; + } + } +} diff --git a/components/security_apps/ips/ips_signatures.cc b/components/security_apps/ips/ips_signatures.cc new file mode 100644 index 0000000..e781c42 --- /dev/null +++ b/components/security_apps/ips/ips_signatures.cc @@ -0,0 +1,809 @@ +#include "ips_signatures.h" + +#include +#include + +#include "ips_comp.h" +#include "ips_basic_policy.h" +#include "snort_basic_policy.h" +#include "generic_rulebase/parameters_config.h" +#include "generic_rulebase/triggers_config.h" +#include "rule_detection.h" +#include "helper.h" +#include "config.h" +#include "context.h" +#include "ips_entry.h" +#include "ips_metric.h" +#include "ips_common_types.h" + +USE_DEBUG_FLAG(D_IPS); + +using namespace IPSSignatureSubTypes; +using namespace ReportIS; +using namespace std; +using MatchType = BaseSignature::MatchType; + +static const map severities = { + { IPSLevel::CRITICAL, Severity::CRITICAL }, + { IPSLevel::HIGH, Severity::HIGH }, + { IPSLevel::MEDIUM_HIGH, Severity::HIGH }, + { IPSLevel::MEDIUM, Severity::MEDIUM }, + { IPSLevel::MEDIUM_LOW, Severity::LOW }, + { IPSLevel::LOW, Severity::LOW }, + { IPSLevel::VERY_LOW, Severity::INFO } +}; + +static const map levels = { + { "Critical", IPSLevel::CRITICAL }, + { "High", IPSLevel::HIGH }, + { "Medium High", IPSLevel::MEDIUM_HIGH }, + { "Medium", IPSLevel::MEDIUM }, + { "Medium Low", IPSLevel::MEDIUM_LOW }, + { "Low", IPSLevel::LOW }, + { "Very Low", IPSLevel::VERY_LOW } +}; + +static IPSLevel +getLevel(const string &level_string, const string &attr_name) +{ + auto index = levels.find(level_string); + if (index == levels.end()) { + reportConfigurationError( + "Unknown level: '" + level_string + "' in attribute " + attr_name + ); + } + return index->second; +} + +void +IPSSignatureMetaData::setIndicators(const string &_source, const string &_version) +{ + source = _source; + version = _version; +} + +string +IPSSignatureMetaData::getSeverityString() const +{ + switch (severity) { + case IPSLevel::VERY_LOW: + return "Very Low"; + case IPSLevel::LOW: + return "Low"; + case IPSLevel::MEDIUM_LOW: + return "Medium Low"; + case IPSLevel::MEDIUM: + return "Medium"; + case IPSLevel::MEDIUM_HIGH: + return "Medium High"; + case IPSLevel::HIGH: + return "High"; + case IPSLevel::CRITICAL: + return "Critical"; + } + + dbgAssert(false) << "Illegal severity value: " << static_cast(severity); + return "Critical"; +} + +string +IPSSignatureMetaData::getConfidenceString() const +{ + if (confidence <= IPSLevel::LOW) return "Low"; + if (confidence >= IPSLevel::HIGH) return "High"; + return "Medium"; +} + +string +IPSSignatureMetaData::getPerformanceString() const +{ + switch (performance) { + case IPSLevel::VERY_LOW: + return "Very Low"; + case IPSLevel::LOW: + return "Low"; + case IPSLevel::MEDIUM_LOW: + return "Medium Low"; + case IPSLevel::MEDIUM: + return "Medium"; + case IPSLevel::MEDIUM_HIGH: + return "Medium High"; + case IPSLevel::HIGH: + return "High"; + case IPSLevel::CRITICAL: + return "Critical"; + } + + dbgAssert(false) << "Illegal performance value: " << static_cast(performance); + return "Critical"; +} + +void +IPSSignatureMetaData::load(cereal::JSONInputArchive &ar) +{ + string severity_string, confidence_string, performance_string; + ar( + cereal::make_nvp("maintrainId", protection_id), + cereal::make_nvp("protectionName", sig_name), + cereal::make_nvp("severity", severity_string), + cereal::make_nvp("lastUpdate", update), + cereal::make_nvp("confidenceLevel", confidence_string), + cereal::make_nvp("performanceImpact", performance_string), + cereal::make_nvp("cveList", cve_list), + cereal::make_nvp("tags", tag_list) + ); + + severity = getLevel(severity_string, "severity"); + confidence = getLevel(confidence_string, "confidence"); + performance = getLevel(performance_string, "performance"); + + try { + ar(cereal::make_nvp("logAttackName", event_log)); + } catch (cereal::Exception &) { + event_log = "IPS Signature '" + sig_name + "' Found"; + ar.setNextName(nullptr); + } + + try { + ar(cereal::make_nvp("silent", is_silent)); + } catch (cereal::Exception &) { + ar.setNextName(nullptr); + } +} + +static const size_t protection_type_pos = strlen("Protection_Type_"); +static const size_t vul_type_pos = strlen("Vul_Type_"); + +string +IPSSignatureSubTypes::IPSSignatureMetaData::getIncidentType() const +{ + for (auto &tag : tag_list) { + if (tag.compare(0, vul_type_pos, "Vul_Type_") == 0) { + auto incident_type = tag.substr(vul_type_pos); + replace(incident_type.begin(), incident_type.end(), '_', ' '); + if (incident_type == "Vulnerability") return "Vulnerability exploit attempt"; + return incident_type; + } + } + + for (auto &tag : tag_list) { + if (tag.compare(0, protection_type_pos, "Protection_Type_") == 0) { + auto incident_type = tag.substr(protection_type_pos); + replace(incident_type.begin(), incident_type.end(), '_', ' '); + if (incident_type == "Vulnerability") return "Vulnerability exploit attempt"; + return incident_type; + } + } + + return ""; +} + +static const size_t year_start_pos = strlen("Threat_Year_"); + +bool +IPSSignatureMetaData::isYearAtLeast(const Maybe &year) const +{ + if (!year.ok()) return true; + auto protection_year = getYear(); + if (!protection_year.ok()) return true; + + return *protection_year >= *year; +} + +Maybe +IPSSignatureMetaData::getYear() const +{ + for (auto &tag : tag_list) { + if (tag.compare(0, year_start_pos, "Threat_Year_") == 0) { + if (tag.size() != year_start_pos + 4) { + dbgWarning(D_IPS) << "Threat year tag (" << tag << ") doen't meet expected format"; + return false; + } + int protection_year = + (tag[year_start_pos] - '0') * 1000 + + (tag[year_start_pos + 1] - '0') * 100 + + (tag[year_start_pos + 2] - '0') * 10 + + (tag[year_start_pos + 3] - '0'); + return protection_year; + } + } + + return genError("Year not found"); +} + +void +CompleteSignature::load(cereal::JSONInputArchive &ar) +{ + ar(cereal::make_nvp("protectionMetadata", metadata)); + RuleDetection rule_detection(metadata.getName()); + ar(cereal::make_nvp("detectionRules", rule_detection)); + rule = rule_detection.getRule(); +} + +MatchType +CompleteSignature::getMatch(const set &matches) const +{ + return rule->getMatch(matches); +} + +set +CompleteSignature::patternsInSignature() const +{ + return rule->patternsInSignature(); +} + +void +CompleteSignature::setIndicators(const string &source, const string &version) +{ + metadata.setIndicators(source, version); +} + +template +static string +getSubString(const Maybe &buf, uint max_size = 0) +{ + if (max_size == 0) max_size = buf.unpack().size(); + const Buffer &real_buf = buf.unpack(); + auto res = real_buf.size() <= max_size ? real_buf : real_buf.getSubBuffer(0, max_size); + return static_cast(res); +} + +ActionResults +SignatureAndAction::getAction(const IPSEntry &ips_state) const +{ + dbgDebug(D_IPS) << "matching exceptions"; + + unordered_map> exceptions_dict; + exceptions_dict["protectionName"].insert(signature->getName()); + + ScopedContext ctx; + ctx.registerValue("protectionName", signature->getName()); + + auto env = Singleton::Consume::by(); + auto host = env->get(HttpTransactionData::host_name_ctx); + if (host.ok()) exceptions_dict["hostName"].insert(*host); + + auto client_ip = env->get(HttpTransactionData::client_ip_ctx); + if (client_ip.ok()) { + stringstream client_ip_str; + client_ip_str << client_ip.unpack(); + exceptions_dict["sourceIP"].insert(client_ip_str.str()); + } + + auto path = ips_state.getBuffer("HTTP_PATH_DECODED"); + if (path.size()) exceptions_dict["url"].insert(static_cast(path)); + + auto env_source_identifier = env->get(HttpTransactionData::source_identifier); + if (env_source_identifier.ok()) { + exceptions_dict["sourceIdentifier"].insert(*env_source_identifier); + } + + I_GenericRulebase *i_rulebase = Singleton::Consume::by(); + auto behaviors = i_rulebase->getBehavior(exceptions_dict); + + set override_actions; + vector override_ids; + for (auto const &behavior : behaviors) { + if (behavior.getKey() == BehaviorKey::ACTION) { + override_actions.insert(behavior.getValue()); + const string &override_id = behavior.getId(); + if (!override_id.empty()) override_ids.push_back(override_id); + } + } + + if (override_actions.find(BehaviorValue::IGNORE) != override_actions.end()) { + dbgDebug(D_IPS) << "Exception matched - action=Detect"; + return make_tuple(IPSSignatureSubTypes::SignatureAction::DETECT, string("Skip"), override_ids); + } + if (override_actions.find(BehaviorValue::ACCEPT) != override_actions.end()) { + dbgDebug(D_IPS) << "Exception matched - action=Detect"; + return make_tuple(IPSSignatureSubTypes::SignatureAction::DETECT, string("Accept"), override_ids); + } + if (override_actions.find(BehaviorValue::REJECT) != override_actions.end()) { + dbgDebug(D_IPS) << "Exception matched - action=Prevent"; + return make_tuple(IPSSignatureSubTypes::SignatureAction::PREVENT, string("Drop"), override_ids); + } + return make_tuple(action, string("None"), override_ids); +} + +static const auto req_body = LogTriggerConf::WebLogFields::webBody; +static const auto headers = LogTriggerConf::WebLogFields::webHeaders; +static const auto url_path = LogTriggerConf::WebLogFields::webUrlPath; +static const auto url_query = LogTriggerConf::WebLogFields::webUrlQuery; +static const auto res_body = LogTriggerConf::WebLogFields::responseBody; +static const auto res_code = LogTriggerConf::WebLogFields::responseCode; + +bool +SignatureAndAction::matchSilent(const Buffer &sample) const +{ + dbgTrace(D_IPS) << "Matched silent signature"; + MatchEvent(signature, IPSSignatureSubTypes::SignatureAction::IGNORE).notify(); + + ScopedContext ctx; + ctx.registerValue("Audience Team", AudienceTeam::SIGNATURE_DEVELOPERS); + + LogGen log( + "Silent Protection", + Audience::INTERNAL, + Severity::INFO, + Priority::MEDIUM, + LogField("practiceType", "Threat Prevention"), + Tags::IPS, + StreamType::JSON_FOG + ); + log << LogField("signatureVersion", signature->getUpdateVersion()) + << LogField("protectionId", signature->getName()) + << LogField("indicatorsSource", signature->getSource()) + << LogField("indicatorsVersion", signature->getFeedVersion()) + << LogField("incidentType", signature->getIncidentType()) + << LogField("matchedSample", static_cast(sample), LogFieldOption::XORANDB64); + + auto env = Singleton::Consume::by(); + auto table = Singleton::Consume::by(); + auto &ips_state = table->getState(); + + auto method = env->get(HttpTransactionData::method_ctx); + if (method.ok()) log << LogField("httpMethod", method.unpack()); + + auto path = env->get("HTTP_PATH_DECODED"); + if (path.ok()) log << LogField("httpUriPath", getSubString(path, 1536), LogFieldOption::XORANDB64); + + auto req_header = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log); + if (req_header.ok()) log << LogField("httpRequestHeaders", getSubString(req_header), LogFieldOption::XORANDB64); + + auto res_code = env->get("HTTP_RESPONSE_CODE"); + if (res_code.ok()) log << LogField("httpResponseCode", static_cast(res_code.unpack())); + + auto req_body = env->get("HTTP_REQUEST_BODY"); + auto res_body = env->get("HTTP_RESPONSE_BODY"); + uint req_size = req_body.ok() ? req_body.unpack().size() : 0; + uint res_size = res_body.ok() ? res_body.unpack().size() : 0; + if (req_size + res_size > 1536) { + if (req_size + 500 > 1536) { + res_size = std::min(500u, res_size); + req_size = 1536 - res_size; + } else { + res_size = 1536 - req_size; + } + } + if (req_size) log << LogField("httpRequestBody", getSubString(req_body, req_size), LogFieldOption::XORANDB64); + if (res_size) log << LogField("httpResponseBody", getSubString(res_body, res_size), LogFieldOption::XORANDB64); + + return false; +} + +bool +SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set &pattern) const +{ + if (signature->getMatch(pattern) != MatchType::MATCH) { + dbgTrace(D_IPS) << "Signature doesn't match"; + return false; + } + + if (signature->isSilent()) return matchSilent(context_buffer); + + auto table = Singleton::Consume::by(); + auto &ips_state = table->getState(); + + auto override_action = getAction(ips_state); + + MatchEvent(signature, get<0>(override_action)).notify(); + + if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::IGNORE) { + dbgDebug(D_IPS) << "Ignored signature"; + return false; + } + + dbgDebug(D_IPS) << "Signature matched - sending log"; + + auto &trigger = getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log"); + bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT; + + auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL; + if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::DETECT) severity = Severity::INFO; + + LogGen log = trigger( + "Web Request", + LogTriggerConf::SecurityType::ThreatPrevention, + severity, + Priority::HIGH, + is_prevent, + LogField("practiceType", "Threat Prevention"), + Tags::IPS + ); + log + << LogField("matchedSignatureConfidence", signature->getConfidenceString()) + << LogField("matchedSignaturePerformance", signature->getPerformanceString()) + << LogField("matchedSignatureSeverity", signature->getSeverityString()) + << LogField("matchedSignatureCVE", makeSeparatedStr(signature->getCveList(), ", ")) + << LogField("signatureVersion", signature->getUpdateVersion()) + << LogField("protectionId", signature->getName()) + << LogField("indicatorsSource", signature->getSource()) + << LogField("indicatorsVersion", signature->getFeedVersion()) + << LogField("waapIncidentType", signature->getIncidentType()); + + if (context_buffer.size() < 1024) { + log << LogField("matchedSample", static_cast(context_buffer), LogFieldOption::XORANDB64); + } else { + auto sample = context_buffer; + sample.keepHead(1024); + log << LogField("matchedSample", static_cast(sample), LogFieldOption::XORANDB64); + } + + auto year = signature->getYear(); + if (year.ok()) log << LogField("matchedSignatureYear", to_string(*year)); + + auto env = Singleton::Consume::by(); + auto host = env->get(HttpTransactionData::host_name_ctx); + if (host.ok()) log << LogField("httpHostName", host.unpack()); + auto client_ip = env->get(HttpTransactionData::client_ip_ctx); + + if (client_ip.ok()) { + stringstream client_ip_str; + client_ip_str << client_ip.unpack(); + log << LogField("sourceIP", client_ip_str.str()); + } + + auto proxy_ip = env->get(HttpTransactionData::proxy_ip_ctx); + if (proxy_ip.ok()) { + log << LogField("proxyIP", static_cast(proxy_ip.unpack())); + } + + auto source_identifier = env->get(HttpTransactionData::source_identifier); + if (source_identifier.ok()) { + log << LogField("httpSourceId", static_cast(source_identifier.unpack())); + } + + auto req_header = ips_state.getTransactionData(IPSCommonTypes::requests_header_for_log); + if (req_header.ok() && trigger.isWebLogFieldActive(headers)) { + log << LogField("httpRequestHeaders", static_cast(req_header.unpack()), LogFieldOption::XORANDB64); + } + + auto client_port = env->get(HttpTransactionData::client_port_ctx); + if (client_port.ok()) log << LogField("sourcePort", client_port.unpack()); + auto method = env->get(HttpTransactionData::method_ctx); + if (method.ok()) log << LogField("httpMethod", method.unpack()); + uint max_size = getConfigurationWithDefault(1536, "IPS", "Max Field Size"); + auto path = env->get("HTTP_PATH_DECODED"); + if (path.ok() && trigger.isWebLogFieldActive(url_path)) { + log << LogField("httpUriPath", getSubString(path, max_size), LogFieldOption::XORANDB64); + } + auto query = env->get("HTTP_QUERY_DECODED"); + if (query.ok() && trigger.isWebLogFieldActive(url_query)) { + log << LogField("httpUriQuery", getSubString(query, max_size), LogFieldOption::XORANDB64); + } + + auto res_code = env->get("HTTP_RESPONSE_CODE"); + if (res_code.ok() && trigger.isWebLogFieldActive(::res_code)) { + log << LogField("httpResponseCode", static_cast(res_code.unpack())); + } + + auto req_body = env->get("HTTP_REQUEST_BODY"); + auto res_body = env->get("HTTP_RESPONSE_BODY"); + uint req_size = req_body.ok() && trigger.isWebLogFieldActive(::req_body) ? req_body.unpack().size() : 0; + uint res_size = res_body.ok() && trigger.isWebLogFieldActive(::res_body) ? res_body.unpack().size() : 0; + if (req_size + res_size > max_size) { + if (req_size + 500 > max_size) { + res_size = std::min(500u, res_size); + req_size = max_size - res_size; + } else { + res_size = max_size - req_size; + } + } + if (req_size) log << LogField("httpRequestBody", getSubString(req_body, req_size), LogFieldOption::XORANDB64); + if (res_size) log << LogField("httpResponseBody", getSubString(res_body, res_size), LogFieldOption::XORANDB64); + + log << LogField("waapOverride", get<1>(override_action)); + + if (!get<2>(override_action).empty()) log.addToOrigin(LogField("exceptionIdList", get<2>(override_action))); + + log << LogField("securityAction", is_prevent ? "Prevent" : "Detect"); + + return is_prevent; +} + +void +IPSSignaturesResource::load(cereal::JSONInputArchive &ar) +{ + vector sigs; + cereal::load(ar, sigs); + + all_signatures.reserve(sigs.size()); + for (auto &sig : sigs) { + all_signatures.emplace_back(make_shared(move(sig))); + } +} + +class CompleteSignatureWrapper +{ +public: + void + load(cereal::JSONInputArchive &ar) + { + try { + sig.load(ar); + is_loaded = true; + } catch (const cereal::Exception &e) { + ar.finishNode(); + reportError(e.what()); + } catch (const Config::ConfigException &e) { + ar.finishNode(); + reportError(e.getError()); + } + } + + bool isOk() const { return is_loaded; } + void setIndicators(const string &source, const string &version) { sig.setIndicators(source, version); } + shared_ptr getPtr() { return make_shared(move(sig)); } + +private: + void + reportError(const string &err) + { + dbgError(D_IPS) << "Failed to load signature due to: " << err; + + if (sig.getName() != "") { + string remediation = + "Verify the validity of the '" + + sig.getName() + + "' signature."; + + LogGen( + "Could not load a Snort signature from configured file", + ReportIS::Level::ACTION, + ReportIS::Audience::SECURITY, + ReportIS::Severity::CRITICAL, + ReportIS::Priority::URGENT, + LogField("EventTopic", "Snort Signatures"), + ReportIS::Tags::POLICY_INSTALLATION + ) << LogField("EventRemediation", remediation); + } + } + + CompleteSignature sig; + bool is_loaded = false; +}; + +void +SnortSignaturesResourceFile::load(cereal::JSONInputArchive &ar) +{ + string time; + vector sigs; + ar( + cereal::make_nvp("modificationTime", time), + cereal::make_nvp("name", name), + cereal::make_nvp("protections", sigs) + ); + + all_signatures.reserve(sigs.size()); + for (auto &sig : sigs) { + if (sig.isOk()) { + sig.setIndicators(name, time); + all_signatures.emplace_back(sig.getPtr()); + } + } +} + +void +SnortSignaturesResource::load(cereal::JSONInputArchive &ar) +{ + cereal::load(ar, files); +} + +void +IPSSignaturesPerContext::addSignature(const IPSSignatureSubTypes::SignatureAndAction &sig) +{ + auto patterns = sig.patternsInSignature(); + + if (patterns.empty()) { + signatures_without_lss.push_back(sig); + return; + } + + for (auto &pat : patterns) { + signatures_per_lss[pat].push_back(sig); + } +} + +void +IPSSignaturesPerContext::calcFirstTier(const string &ctx_name) +{ + std::set patterns; + for (const auto &lss_to_sig : signatures_per_lss) { + patterns.emplace(lss_to_sig.first); + } + + first_tier = Singleton::Consume::by()->getHook(ctx_name, patterns); +} + +set +IPSSignaturesPerContext::getFirstTierMatches(const Buffer &buffer) const +{ + return first_tier->ok() ? first_tier->scanBuf(buffer) : set(); +} + +bool +IPSSignaturesPerContext::isMatchedPrevent(const Buffer &context_buffer) const +{ + auto first_tier_res = getFirstTierMatches(context_buffer); + + for (auto &pat : first_tier_res) { + auto find = signatures_per_lss.find(pat); + if (find == signatures_per_lss.end()) continue; + for (auto &sig : find->second) { + if (sig.isMatchedPrevent(context_buffer, first_tier_res)) return true; + } + } + + for (auto &sig : signatures_without_lss) { + if (sig.isMatchedPrevent(context_buffer, first_tier_res)) return true; + } + + return false; +} + +void +IPSSignatures::load(cereal::JSONInputArchive &ar) +{ + ar( + cereal::make_nvp("assetName", asset_name), + cereal::make_nvp("practiceName", practice_name) + ); + + try { + ar(cereal::make_nvp("assetId", asset_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + asset_id = ""; + } + + try { + ar(cereal::make_nvp("practiceId", practice_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + practice_id = ""; + } + + try { + ar(cereal::make_nvp("sourceIdentifier", source_id)); + for (auto &ch : source_id) { + ch = tolower(ch); + } + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + source_id = ""; + } + + RuleSelector ruleSelector; + ruleSelector.load(ar); + std::vector signatures = ruleSelector.selectSignatures(); + + if (signatures.empty()) { + dbgDebug(D_IPS) << "[IPS] Could not find any match between rules and signatures."; + return; + } + + for (const auto &sig : signatures) { + auto &sig_contexts = sig.getContext(); + for (auto &sig_context : sig_contexts) { + signatures_per_context[sig_context].addSignature(sig); + } + } + + for (auto &sig_per_ctx : signatures_per_context) { + sig_per_ctx.second.calcFirstTier(sig_per_ctx.first); + } +} + +bool +IPSSignatures::isMatchedPrevent(const string &context_name, const Buffer &context_buffer) const +{ + auto curr_sig = signatures_per_context.find(context_name); + + if (curr_sig == signatures_per_context.end()) { + dbgDebug(D_IPS) << "[IPS] No signatures for " << context_name; + return false; + } + + auto &config = getConfiguration("IPS", "IpsProtections"); + ScopedContext ctx; + auto SOURCE = EnvKeyAttr::LogSection::SOURCE; + if (config.ok()) { + ctx.registerValue("practiceName", (*config).getPractice(), SOURCE); + ctx.registerValue("practiceId", (*config).getPracticeId(), SOURCE); + } + ctx.registerValue("practiceSubType", "Web IPS", SOURCE); + auto is_matched = curr_sig->second.isMatchedPrevent(context_buffer); + + return is_matched; +} + +bool +IPSSignatures::isEmpty(const std::string &context) const +{ + return signatures_per_context.find(context) == signatures_per_context.end(); +} + +void +SnortSignatures::load(cereal::JSONInputArchive &ar) +{ + ar( + cereal::make_nvp("assetName", asset_name), + cereal::make_nvp("practiceName", practice_name) + ); + + try { + ar(cereal::make_nvp("assetId", asset_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + asset_id = ""; + } + + try { + ar(cereal::make_nvp("practiceId", practice_id)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + practice_id = ""; + } + + try { + ar(cereal::make_nvp("sourceIdentifier", source_id)); + for (auto &ch : source_id) { + ch = tolower(ch); + } + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + source_id = ""; + } + + SnortRuleSelector ruleSelector; + ruleSelector.load(ar); + std::vector signatures = ruleSelector.selectSignatures(); + + if (signatures.empty()) { + dbgDebug(D_IPS) << "[Snort] Could not find any match between rules and signatures."; + return; + } + + for (const auto &sig : signatures) { + auto &sig_contexts = sig.getContext(); + for (auto &sig_context : sig_contexts) { + signatures_per_context[sig_context].addSignature(sig); + } + } + + for (auto &sig_per_ctx: signatures_per_context) { + sig_per_ctx.second.calcFirstTier(sig_per_ctx.first); + } +} + +bool +SnortSignatures::isMatchedPrevent(const string &context_name, const Buffer &context_buffer) const +{ + auto curr_sig = signatures_per_context.find(context_name); + + if (curr_sig == signatures_per_context.end()) { + dbgDebug(D_IPS) << "[Snort] No signatures for " << context_name; + return false; + } + + auto &config = getConfiguration("IPSSnortSigs", "SnortProtections"); + ScopedContext ctx; + auto SOURCE = EnvKeyAttr::LogSection::SOURCE; + if (config.ok()) { + ctx.registerValue("assetName", (*config).getAsset(), SOURCE); + ctx.registerValue("assetId", (*config).getAssetId(), SOURCE); + ctx.registerValue("practiceName", (*config).getPractice(), SOURCE); + ctx.registerValue("practiceId", (*config).getPracticeId(), SOURCE); + } + ctx.registerValue("practiceSubType", "Web Snort", SOURCE); + auto is_matched = curr_sig->second.isMatchedPrevent(context_buffer); + + return is_matched; +} + +bool +SnortSignatures::isEmpty(const std::string &context) const +{ + return signatures_per_context.find(context) == signatures_per_context.end(); +} diff --git a/components/security_apps/ips/ips_ut/CMakeLists.txt b/components/security_apps/ips/ips_ut/CMakeLists.txt new file mode 100644 index 0000000..2a36d27 --- /dev/null +++ b/components/security_apps/ips/ips_ut/CMakeLists.txt @@ -0,0 +1,7 @@ +link_directories(${CMAKE_BINARY_DIR}/core/shmem_ipc) + +add_unit_test( + ips_ut + "signatures_ut.cc;entry_ut.cc;component_ut.cc;configuration.cc;rule_selector_ut.cc;compound_ut.cc;resource_ut.cc" + "ips;keywords;pcre2-8;intelligence_is_v2;logging;compression_utils;agent_details;time_proxy;event_is;table;http_transaction_data;nginx_attachment;connkey;pm;metric;encryptor;generic_rulebase;generic_rulebase_evaluators;compression_utils;ip_utilities;-lboost_regex;-lcrypto;-lz" +) diff --git a/components/security_apps/ips/ips_ut/component_ut.cc b/components/security_apps/ips/ips_ut/component_ut.cc new file mode 100644 index 0000000..dd6be98 --- /dev/null +++ b/components/security_apps/ips/ips_ut/component_ut.cc @@ -0,0 +1,913 @@ +#include "ips_comp.h" + +#include + +#include "cptest.h" +#include "ips_entry.h" +#include "new_table_entry.h" +#include "keyword_comp.h" +#include "environment.h" +#include "mock/mock_table.h" +#include "config.h" +#include "http_manager.h" +#include "config_component.h" +#include "agent_details.h" +#include "mock/mock_logging.h" +#include "mock/mock_time_get.h" +#include "mock/mock_mainloop.h" +#include "encryptor.h" +#include "generic_rulebase/generic_rulebase.h" +#include "generic_rulebase/triggers_config.h" + +using namespace testing; +using namespace std; + +class ComponentTest : public Test +{ +public: + ComponentTest() + { + comp.preload(); + comp.init(); + } + + ~ComponentTest() + { + comp.fini(); + } + + void + loadPolicy(const string &config) + { + stringstream ss; + ss << config; + Singleton::Consume::from(conf)->loadConfiguration(ss); + } + + void + setTrigger() + { + string log_trigger( + "{" + " \"context\": \"triggerId(5eaeefde6765c30010bae8b6)\"," + " \"triggerName\": \"Logging Trigger\"," + " \"triggerType\": \"log\"," + " \"urlForSyslog\": \"\"," + " \"urlForCef\": \"128.1.1.1:333\"," + " \"acAllow\": false," + " \"acDrop\": true," + " \"complianceViolations\": true," + " \"complianceWarnings\": true," + " \"logToAgent\": true," + " \"logToCloud\": true," + " \"logToSyslog\": false," + " \"logToCef\": true," + " \"tpDetect\": true," + " \"tpPrevent\": true," + " \"verbosity\": \"Standard\"," + " \"webBody\": true," + " \"webHeaders\": true," + " \"webRequests\": true," + " \"webUrlPath\": true," + " \"webUrlQuery\": true" + "}" + ); + + stringstream ss(log_trigger); + cereal::JSONInputArchive ar(ss); + LogTriggerConf trigger; + trigger.load(ar); + + setConfiguration(trigger, "rulebase", "log"); + } + + IPSComp comp; + StrictMock table; + IPSEntry entry; + + GenericRulebase generic_rulebase; + ConfigComponent conf; + Encryptor encryptor; + KeywordComp keywords; + ::Environment env; + AgentDetails details; + NiceMock logs; + NiceMock time; + NiceMock mainloop; + static const EventVerdict inspect; + static const EventVerdict accept; + static const EventVerdict drop; + + HttpHeader end_headers{Buffer(""), Buffer(""), 0, true}; +}; + +static bool +operator ==(const EventVerdict &first, const EventVerdict &second) +{ + return first.getVerdict() == second.getVerdict(); +} + +const EventVerdict ComponentTest::inspect(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT); + +const EventVerdict ComponentTest::accept(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT); + +const EventVerdict ComponentTest::drop(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); + +TEST_F(ComponentTest, check_init_fini_do_not_crush) +{ +} + +TEST_F(ComponentTest, new_table_entry_with_empty_configuration) +{ + NewTableEntry().notify(); +} + +TEST_F(ComponentTest, new_table_entry_with_configuration) +{ + string config = + "{" + "\"IPS\": {" + "\"configurations\": [" + "{" + "\"context\": \"\"," + "\"contextsConfiguration\": [" + "{" + "\"type\": \"keep\"," + "\"name\": \"HTTP_REQUEST_BODY\"" + "}" + "]" + "}" + "]," + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Prevent\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + + loadPolicy(config); + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillOnce(Return(&entry)); + NewTableEntry().notify(); +} + +TEST_F(ComponentTest, events) +{ + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + IPSEntry entry; + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + + HttpTransactionData transaction; + EXPECT_THAT(NewHttpTransactionEvent(transaction).query(), ElementsAre(accept)); + HttpHeader header_req(Buffer("key"), Buffer("val"), 1); + EXPECT_THAT(HttpRequestHeaderEvent(header_req).query(), ElementsAre(inspect)); + HttpBody body_req(Buffer("data"), 0, true); + EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(accept)); + + EXPECT_THAT(ResponseCodeEvent(200).query(), ElementsAre(inspect)); + HttpHeader header_res(Buffer("key"), Buffer("val"), 2); + EXPECT_THAT(HttpResponseHeaderEvent(header_res).query(), ElementsAre(inspect)); + HttpBody body_res(Buffer("data"), true, 0); + EXPECT_THAT(HttpResponseBodyEvent(body_res, Buffer()).query(), ElementsAre(accept)); + EXPECT_THAT(EndTransactionEvent().performNamedQuery(), ElementsAre(Pair("ips application", accept))); +} + +TEST_F(ComponentTest, check_url_decoding) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"d d\\\";\"," + "\"context\": [\"HTTP_COMPLETE_URL_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Prevent\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(drop)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop)); +} + +TEST_F(ComponentTest, check_query) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"g=#\\\";\"," + "\"context\": [\"HTTP_QUERY_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Prevent\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(drop)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop)); +} + +TEST_F(ComponentTest, check_query_detect_mode) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"d d\\\";\"," + "\"context\": [\"HTTP_QUERY_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Detect\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillOnce(Return(&entry)); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + EXPECT_THAT(EndTransactionEvent().query(), ElementsAre(accept)); +} + +TEST_F(ComponentTest, check_query_inactive_mode) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"g=#\\\";\"," + "\"context\": [\"HTTP_QUERY_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Prevent\"," + "\"rules\": [" + "{" + "\"action\": \"Inactive\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(accept)); +} + +TEST_F(ComponentTest, check_query_silent_mode) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"silent\": true," + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"g=#\\\";\"," + "\"context\": [\"HTTP_QUERY_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Prevent\"," + "\"rules\": []" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + EXPECT_THAT(HttpRequestHeaderEvent(end_headers).query(), ElementsAre(inspect)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(accept)); +} + +TEST_F(ComponentTest, check_filtering_by_year) +{ + string config = + "{" + "\"IPS\": {" + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": [ \"ggg\", \"Threat_Year_2014\", \"hhh\" ]," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"g=#\\\";\"," + "\"context\": [\"HTTP_QUERY_DECODED\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Prevent\"," + "\"rules\": [" + "{" + "\"action\": \"Inactive\"," + "\"protectionsFromYear\": 2013" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + + HttpTransactionData new_transaction( + "1.1", + "GET", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(accept)); +} + +TEST_F(ComponentTest, log_fields) +{ + string config = + "{" + "\"IPS\": {" + "\"Max Field Size\": [" + "{" + "\"value\": 25" + "}" + "]," + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Prevent\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + setTrigger(); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + Report report; + EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report)); + + HttpTransactionData new_transaction( + "1.1", + "POST", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 1); + EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect)); + HttpHeader header_req2(Buffer("key2"), Buffer("val2"), 2); + EXPECT_THAT(HttpRequestHeaderEvent(header_req2).query(), ElementsAre(inspect)); + HttpHeader header_req3(Buffer("key3"), Buffer("val3"), 3); + EXPECT_THAT(HttpRequestHeaderEvent(header_req3).query(), ElementsAre(inspect)); + string body_str("data: ddd"); + HttpBody body_req(Buffer(body_str), 0, true); + EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop)); + + EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestHeaders=\"key1: val1, key2: val2\"")); + EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestBody=\"" + body_str + "\"")); + EXPECT_THAT(report.getSyslog(), HasSubstr("signatureVersion=\"20210420\"")); +} + +TEST_F(ComponentTest, log_field_httpRequestHeader) +{ + string config = + "{" + "\"IPS\": {" + "\"Max Field Size\": [" + "{" + "\"value\": 25" + "}" + "]," + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}" + "]," + "\"IpsProtections\": [" + "{" + "\"context\": \"\"," + "\"ruleName\": \"rule1\"," + "\"assetName\": \"asset1\"," + "\"assetId\": \"1-1-1\"," + "\"practiceId\": \"2-2-2\"," + "\"practiceName\": \"practice1\"," + "\"defaultAction\": \"Detect\"," + "\"rules\": [" + "{" + "\"action\": \"Prevent\"," + "\"severityLevel\": \"Low or above\"," + "\"performanceImpact\": \"High or lower\"," + "\"confidenceLevel\": \"Low\"" + "}" + "]" + "}" + "]" + "}" + "}"; + loadPolicy(config); + setTrigger(); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + IPSEntry entry; + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + Report report; + EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report)); + + HttpTransactionData new_transaction( + "1.1", + "POST", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 1); + EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect)); + HttpBody body_req(Buffer("data: ddd"), 0, true); + EXPECT_THAT(HttpRequestBodyEvent(body_req, Buffer()).query(), ElementsAre(inspect)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop)); + + EXPECT_THAT(report.getSyslog(), HasSubstr("httpRequestHeaders=\"key1: val1\"")); + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + IPSEntry entry1; + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry1)); + Report report1; + EXPECT_CALL(logs, sendLog(_)).WillOnce(SaveArg<0>(&report1)); + + HttpTransactionData new_transaction2( + "1.1", + "POST", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "d%20d?g=%23", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction2).query(), ElementsAre(inspect)); + HttpHeader header_req2(Buffer("key2"), Buffer("val2"), 1); + EXPECT_THAT(HttpRequestHeaderEvent(header_req2).query(), ElementsAre(inspect)); + + HttpBody body_req2(Buffer("data: ddd"), 0, true); + EXPECT_THAT(HttpRequestBodyEvent(body_req2, Buffer()).query(), ElementsAre(inspect)); + EXPECT_THAT(EndRequestEvent().query(), ElementsAre(drop)); + + EXPECT_THAT(report1.getSyslog(), HasSubstr("httpRequestHeaders=\"key2: val2\"")); +} + +TEST_F(ComponentTest, prxeem_exception_bug) +{ + generic_rulebase.preload(); + generic_rulebase.init(); + string config = + "{" + " \"IPS\": {" + " \"protections\": [" + " {" + " \"protectionMetadata\": {" + " \"protectionName\": \"Null HTTP Encodings\"," + " \"maintrainId\": \"101\"," + " \"severity\": \"Low\"," + " \"confidenceLevel\": \"Low\"," + " \"performanceImpact\": \"Medium High\"," + " \"lastUpdate\": \"20210420\"," + " \"tags\": []," + " \"cveList\": []" + " }," + " \"detectionRules\": {" + " \"type\": \"simple\"," + " \"SSM\": \"\"," + " \"keywords\": \"data: \\\"|25|00\\\"; data: \\\"?\\\";\"," + " \"context\": [\"HTTP_COMPLETE_URL_ENCODED\"]" + " }" + " }" + " ]," + " \"IpsProtections\": [" + " {" + " \"context\": \"\"," + " \"ruleName\": \"rule1\"," + " \"assetName\": \"asset1\"," + " \"assetId\": \"1-1-1\"," + " \"practiceId\": \"2-2-2\"," + " \"practiceName\": \"practice1\"," + " \"defaultAction\": \"Prevent\"," + " \"rules\": []" + " }" + " ]" + " }," + " \"rulebase\": {" + " \"rulesConfig\": [" + " {" + " \"context\": \"All()\"," + " \"priority\": 1," + " \"ruleId\": \"5eaef0726765c30010bae8bb\"," + " \"ruleName\": \"Acme web API\"," + " \"assetId\": \"5e243effd858007660b758ad\"," + " \"assetName\": \"Acme Power API\"," + " \"parameters\": [" + " {" + " \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\"," + " \"parameterType\": \"exceptions\"," + " \"parameterName\": \"exception\"" + " }" + " ]," + " \"zoneId\": \"\"," + " \"zoneName\": \"\"" + " }" + " ]," + " \"exception\": [" + " {" + " \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764003)\"," + " \"match\": {" + " \"type\": \"operator\"," + " \"op\": \"and\"," + " \"items\": [{" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"url\"," + " \"value\": [\"(/en|/de)?/admin/helpdesk/dashboard/operator/advanced_search.*\"]" + " }, {" + " \"type\": \"operator\"," + " \"op\": \"or\"," + " \"items\": [{" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"protectionName\"," + " \"value\": [\"Null HTTP Encodings\"]" + " }, {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"parameterName\"," + " \"value\": [\"op\\\\.submit\\\\.reset\"]" + " }]" + " }]" + " }," + " \"behavior\": {" + " \"key\": \"action\"," + " \"value\": \"accept\"" + " }" + " }" + " ]" + " }" + "}"; + loadPolicy(config); + + + EXPECT_CALL(table, createStateRValueRemoved(_, _)); + IPSEntry entry; + EXPECT_CALL(table, getState(_)).WillRepeatedly(Return(&entry)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(Return(true)); + + HttpTransactionData new_transaction( + "1.1", + "POST", + "ffff", + IPAddr::createIPAddr("0.0.0.0").unpack(), + 80, + "/admin/helpdesk/dashboard/operator/advanced_search?order=created&stuff=%00", + IPAddr::createIPAddr("1.1.1.1").unpack(), + 5428 + ); + + EXPECT_THAT(NewHttpTransactionEvent(new_transaction).query(), ElementsAre(inspect)); + HttpHeader header_req1(Buffer("key1"), Buffer("val1"), 0, true); + EXPECT_THAT(HttpRequestHeaderEvent(header_req1).query(), ElementsAre(inspect)); +} diff --git a/components/security_apps/ips/ips_ut/compound_ut.cc b/components/security_apps/ips/ips_ut/compound_ut.cc new file mode 100644 index 0000000..05633d0 --- /dev/null +++ b/components/security_apps/ips/ips_ut/compound_ut.cc @@ -0,0 +1,155 @@ +#include "compound_protection.h" + +#include "cptest.h" +#include "ips_entry.h" +#include "mock/mock_table.h" +#include "environment.h" +#include "i_keywords_rule.h" + +using namespace testing; +using namespace std; + +class CompoundTest : public Test +{ +public: + CompoundTest() + { + ON_CALL(table, hasState(_)).WillByDefault(Return(true)); + ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state)); + } + + template + shared_ptr + loadSig(const string &name, const string &operation, PatternContextPair ... pat_ctx) + { + stringstream ss; + ss << "{" + << "\"type\": \"compound\"," + << "\"operation\": \"" << operation << "\"," + << "\"operands\": ["; + getSginatureStream(ss, pat_ctx ...); + ss << "]" << "}"; + + cereal::JSONInputArchive ar(ss); + + return CompoundProtection::get(name, ar); + } + + template + set + turnToPatternSet(const Strings & ... strings) + { + pat_set.clear(); + populatePatternSet(strings ...); + return pat_set; + } + + void setActiveContext(const string &name) { ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), name); } + +private: + void populatePatternSet() {} + + template + void + populatePatternSet(const string &pat, const Strings & ... strings) + { + pat_set.emplace(pat, false, false); + populatePatternSet(strings ...); + } + + ostream & + getSginatureStream(ostream &ss, const string &pattern, const string &context) + { + ss << "{" + << "\"type\": \"simple\"," + << "\"SSM\": \"" << pattern << "\"," + << "\"keywords\": \"\"," + << "\"context\": [" + << "\"" << context << "\"" + << "]" + << "}"; + return ss; + } + + template + ostream & + getSginatureStream(ostream &ss, const string &pattern, const string &context, PatternContextPair ... pat_ctx) + { + return getSginatureStream(getSginatureStream(ss, pattern, context) << ",", pat_ctx ...); + } + + NiceMock table; + IPSEntry ips_state; + set pat_set; + ::Environment env; + ScopedContext ctx; +}; + +TEST_F(CompoundTest, BasicLoading) +{ + auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + EXPECT_NE(sig, nullptr); + EXPECT_EQ(sig->getSigId(), "Test"); + EXPECT_THAT(sig->getContext(), ElementsAre("HTTP_REQUEST_DATA", "HTTP_RESPONSE_DATA")); + EXPECT_EQ(sig->patternsInSignature(), turnToPatternSet("aaa", "bbb")); +} + +TEST_F(CompoundTest, BasicOrTest) +{ + auto sig = loadSig("Test", "or", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("NO_CONTEXT"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH); + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("ddd")), IPSSignatureSubTypes::BaseSignature::MatchType::CACHE_MATCH); +} + +TEST_F(CompoundTest, BasicOrOrderTest) +{ + auto sig = loadSig("Test", "or", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("HTTP_RESPONSE_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH); +} + +TEST_F(CompoundTest, BasicAndTest) +{ + auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); + setActiveContext("HTTP_RESPONSE_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH); +} + +TEST_F(CompoundTest, BasicAndOrderTest) +{ + auto sig = loadSig("Test", "and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("HTTP_RESPONSE_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH); +} + +TEST_F(CompoundTest, BasicOrderedAndTest) +{ + auto sig = loadSig("Test", "ordered_and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); + setActiveContext("HTTP_RESPONSE_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::MATCH); +} + +TEST_F(CompoundTest, BasicOrderedAndOrderTest) +{ + auto sig = loadSig("Test", "ordered_and", "aaa", "HTTP_REQUEST_DATA", "bbb", "HTTP_RESPONSE_DATA"); + + setActiveContext("HTTP_RESPONSE_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("bbb")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); + setActiveContext("HTTP_REQUEST_DATA"); + EXPECT_EQ(sig->getMatch(turnToPatternSet("aaa")), IPSSignatureSubTypes::BaseSignature::MatchType::NO_MATCH); +} diff --git a/components/security_apps/ips/ips_ut/configuration.cc b/components/security_apps/ips/ips_ut/configuration.cc new file mode 100644 index 0000000..c831742 --- /dev/null +++ b/components/security_apps/ips/ips_ut/configuration.cc @@ -0,0 +1,55 @@ +#include "ips_configuration.h" +#include "cptest.h" + +TEST(configuration, basic_context) +{ + cptestPrepareToDie(); + + IPSConfiguration::Context ctx1(IPSConfiguration::ContextType::HISTORY, 254); + EXPECT_EQ(ctx1.getType(), IPSConfiguration::ContextType::HISTORY); + EXPECT_EQ(ctx1.getHistorySize(), 254); + + IPSConfiguration::Context ctx2(IPSConfiguration::ContextType::NORMAL, 0); + EXPECT_EQ(ctx2.getType(), IPSConfiguration::ContextType::NORMAL); + EXPECT_DEATH(ctx2.getHistorySize(), "Try to access history size for non-history context"); +} + + +TEST(configuration, read_configuration) +{ + cptestPrepareToDie(); + + std::stringstream conf_str; + conf_str << + "{" + "\"contextsConfiguration\": [" + "{" + "\"name\": \"HTTP_REQUEST_BODY\"," + "\"type\": \"history\"," + "\"historySize\": 100" + "}," + "{" + "\"name\": \"HTTP_REQUEST_HEADER\"," + "\"type\": \"keep\"" + "}" + "]" + "}"; + + cereal::JSONInputArchive ar(conf_str); + + IPSConfiguration conf; + conf.load(ar); + + auto body = conf.getContext("HTTP_REQUEST_BODY"); + EXPECT_EQ(body.getType(), IPSConfiguration::ContextType::HISTORY); + EXPECT_EQ(conf.getHistorySize("HTTP_REQUEST_BODY"), 100); + + auto header = conf.getContext("HTTP_REQUEST_HEADER"); + EXPECT_EQ(header.getType(), IPSConfiguration::ContextType::KEEP); + EXPECT_DEATH(conf.getHistorySize("HTTP_REQUEST_HEADER"), "Try to access history size for non-history context"); + + auto line = conf.getContext("HTTP_REQUEST_LINE"); + EXPECT_EQ(line.getType(), IPSConfiguration::ContextType::NORMAL); + + EXPECT_DEATH(conf.getHistorySize("NO_CONTEXT"), "Try to access history size for non-exiting context"); +} diff --git a/components/security_apps/ips/ips_ut/entry_ut.cc b/components/security_apps/ips/ips_ut/entry_ut.cc new file mode 100644 index 0000000..b12722c --- /dev/null +++ b/components/security_apps/ips/ips_ut/entry_ut.cc @@ -0,0 +1,268 @@ +#include "ips_entry.h" + +#include + +#include "ips_signatures.h" +#include "cptest.h" +#include "keyword_comp.h" +#include "config.h" +#include "config_component.h" +#include "environment.h" +#include "agent_details.h" +#include "mock/mock_logging.h" +#include "mock/mock_time_get.h" +#include "mock/mock_mainloop.h" +#include "mock/mock_table.h" +#include "generic_rulebase/generic_rulebase.h" + +using namespace std; +using namespace testing; + +ostream & +operator<<(ostream &os, const ParsedContextReply &action) +{ + return os << (action==ParsedContextReply::ACCEPT ? "ACCEPT" : "DROP"); +} + +class MockAgg : Singleton::Provide::SelfInterface +{ + shared_ptr + getHook(const string &, const set &pats) override + { + auto hook = make_shared(); + hook->prepare(pats); + return hook; + } +}; + +class EntryTest : public Test +{ +public: + EntryTest() + { + ON_CALL(table, getState(_)).WillByDefault(Return(ptr)); + } + + void + loadSignatures(const string &sigs) + { + { + stringstream ss; + ss << "[" << sigs << "]"; + cereal::JSONInputArchive ar(ss); + IPSSignaturesResource resource; + resource.load(ar); + setResource(resource, "IPS", "protections"); + } + { + stringstream ss; + ss << "{"; + ss << "\"context\": \"\","; + ss << "\"ruleName\": \"rule1\","; + ss << "\"assetName\": \"asset1\","; + ss << "\"assetId\": \"1-1-1\","; + ss << "\"practiceId\": \"2-2-2\","; + ss << "\"practiceName\": \"practice1\","; + ss << "\"defaultAction\": \"Detect\","; + ss << "\"rules\": ["; + ss << "{"; + ss << "\"action\": \"Prevent\","; + ss << "\"performanceImpact\": \"High or lower\","; + ss << "\"severityLevel\": \"Low or above\","; + ss << "\"confidenceLevel\": \"Low\""; + ss << "}"; + ss << "]"; + ss << "}"; + cereal::JSONInputArchive ar(ss); + IPSSignatures signatures; + signatures.load(ar); + setConfiguration(signatures, "IPS", "IpsProtections"); + } + } + + void + loadSnortSignatures(const string &sigs) + { + { + stringstream ss; + ss << "[{ \"modificationTime\": \"22/02/08\", \"name\": \"rules1\", \"protections\": [" << sigs << "] }]"; + cereal::JSONInputArchive ar(ss); + SnortSignaturesResource resource; + resource.load(ar); + setResource(resource, "IPSSnortSigs", "protections"); + } + { + stringstream ss; + ss << "{"; + ss << "\"context\": \"\","; + ss << "\"assetName\": \"asset1\","; + ss << "\"assetId\": \"1-1-1\","; + ss << "\"practiceId\": \"2-2-2\","; + ss << "\"practiceName\": \"practice1\","; + ss << "\"files\": [ \"rules1\" ],"; + ss << "\"mode\": \"Prevent\""; + ss << "}"; + cereal::JSONInputArchive ar(ss); + SnortSignatures signatures; + signatures.load(ar); + setConfiguration(signatures, "IPSSnortSigs", "SnortProtections"); + } + } + + ParsedContextReply + repondToContext(const string &buf_str, const string &name) + { + Buffer buf(buf_str); + ScopedContext ctx; + ctx.registerValue(name, buf); + return entry.respond(ParsedContext(buf, name, 0)); + } + + IPSEntry entry; + TableOpaqueBase *ptr = &entry; + +private: + NiceMock mock_mainloop; + NiceMock time; + ::Environment env; + GenericRulebase generic_rulebase; + ConfigComponent conf; + KeywordComp keywords; + AgentDetails details; + NiceMock logs; + NiceMock table; + MockAgg mock_agg; +}; + +TEST_F(EntryTest, basic_inherited_functions) +{ + EXPECT_EQ(IPSEntry::name(), "IPS"); + EXPECT_EQ(IPSEntry::currVer(), 0); + EXPECT_EQ(IPSEntry::minVer(), 0); + EXPECT_NE(IPSEntry::prototype(), nullptr); + EXPECT_EQ(entry.getListenerName(), IPSEntry::name()); + + stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + entry.serialize(ar, 0); + } + EXPECT_EQ(ss.str(), ""); + + // Just make sure it doesn't crush + entry.upon(ParsedContext(Buffer(), "Nothing", 0)); +} + +TEST_F(EntryTest, check_listenning) +{ + EXPECT_TRUE(Listener::empty()); + ptr->uponEnteringContext(); + EXPECT_FALSE(Listener::empty()); + ptr->uponLeavingContext(); + EXPECT_TRUE(Listener::empty()); +} + +TEST_F(EntryTest, check_signature_invoking) +{ + EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::ACCEPT); + EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT); + + string signature = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + loadSignatures(signature); + + EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::DROP); + EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT); +} + +TEST_F(EntryTest, check_snort_signature_invoking) +{ + EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::ACCEPT); + EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT); + + string signature = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}," + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Bad sig\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: jjjj;\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + loadSnortSignatures(signature); + + EXPECT_EQ(repondToContext("ddd", "HTTP_REQUEST_BODY"), ParsedContextReply::DROP); + EXPECT_EQ(repondToContext("ddd", "HTTP_RESPONSE_BODY"), ParsedContextReply::ACCEPT); +} + +TEST_F(EntryTest, flags_test) +{ + EXPECT_FALSE(entry.isFlagSet("CONTEXT_A")); + EXPECT_FALSE(entry.isFlagSet("CONTEXT_B")); + entry.setFlag("CONTEXT_A"); + EXPECT_TRUE(entry.isFlagSet("CONTEXT_A")); + EXPECT_FALSE(entry.isFlagSet("CONTEXT_B")); + entry.unsetFlag("CONTEXT_A"); + EXPECT_FALSE(entry.isFlagSet("CONTEXT_A")); + EXPECT_FALSE(entry.isFlagSet("CONTEXT_B")); +} + +TEST_F(EntryTest, get_buffer_test) +{ + repondToContext("ddd", "HTTP_REQUEST_BODY"); + EXPECT_EQ(entry.getBuffer("HTTP_REQUEST_BODY"), Buffer("ddd")); + EXPECT_EQ(entry.getBuffer("HTTP_REQUEST_HEADER"), Buffer()); +} + +TEST_F(EntryTest, get_and_set_transaction_data) +{ + EXPECT_FALSE(entry.getTransactionData(Buffer("transaction_key")).ok()); + entry.setTransactionData(Buffer("transaction_key"), Buffer("transaction_value")); + ASSERT_TRUE(entry.getTransactionData(Buffer("transaction_key")).ok()); + EXPECT_EQ(entry.getTransactionData(Buffer("transaction_key")).unpack(), Buffer("transaction_value")); +} diff --git a/components/security_apps/ips/ips_ut/resource_ut.cc b/components/security_apps/ips/ips_ut/resource_ut.cc new file mode 100644 index 0000000..967e917 --- /dev/null +++ b/components/security_apps/ips/ips_ut/resource_ut.cc @@ -0,0 +1,77 @@ +#include "ips_signatures.h" +#include "cptest.h" +#include "environment.h" +#include "config_component.h" + +using namespace std; +using namespace testing; + +static const string basic_resource = + "{" + "\"IPS\": {" + "\"VersionId\": \"1234567\"," + "\"protections\": [" + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Null HTTP Encodings\"," + "\"severity\": \"Medium\"," + "\"confidenceLevel\": \"High\"," + "\"performanceImpact\": \"Medium\"," + "\"lastUpdate\": \"20130101\"," + "\"maintrainId\": \"8576967832\"," + "\"tags\": []," + "\"cveList\": []," + "\"silent\": false" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"aaaa\"," + "\"keywords\": \"\"," + "\"context\": [" + "\"HTTP_COMPLETE_URL_ENCODED\"" + "]" + "}" + "}," + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Null HTTP Encodings\"," + "\"severity\": \"Medium\"," + "\"confidenceLevel\": \"High\"," + "\"performanceImpact\": \"Medium\"," + "\"lastUpdate\": \"20130101\"," + "\"maintrainId\": \"8576967832\"," + "\"tags\": []," + "\"cveList\": []," + "\"silent\": false" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"bbbbb\"," + "\"keywords\": \"\"," + "\"context\": [" + "\"HTTP_COMPLETE_URL_ENCODED\"" + "]" + "}" + "}" + "]" + "}" + "}"; + +TEST(resources, basic_resource) +{ + ConfigComponent conf; + ::Environment env; + + conf.preload(); + + registerExpectedSetting("IPS", "protections"); + registerExpectedSetting("IPS", "VersionId"); + stringstream resource; + resource << basic_resource; + Singleton::Consume::from(conf)->loadConfiguration(resource); + + auto loaded_resources = getSettingWithDefault(IPSSignaturesResource(), "IPS", "protections"); + EXPECT_EQ(loaded_resources.getSignatures().size(), 2); + auto version = getSettingWithDefault("", "IPS", "VersionId"); + EXPECT_EQ(version, "1234567"); +} diff --git a/components/security_apps/ips/ips_ut/rule_selector_ut.cc b/components/security_apps/ips/ips_ut/rule_selector_ut.cc new file mode 100644 index 0000000..2c041ca --- /dev/null +++ b/components/security_apps/ips/ips_ut/rule_selector_ut.cc @@ -0,0 +1,163 @@ +#include +#include "ips_enums.h" +#include "ips_basic_policy.h" +#include "cptest.h" +#include "config.h" + +using namespace testing; +using namespace std; + +ostream & +operator<<(ostream &os, const RuleSelector &selector) +{ + selector.print(os); + return os; +} + +class RuleSelectorTest : public Test +{ +public: + void + load(const string &config) + { + stringstream ss; + ss << config; + cereal::JSONInputArchive ar(ss); + + ruleSelector.load(ar); + } + + string protection = + "{" + "\"defaultAction\": \"Prevent\"," + "\"rules\": [" + "{" + "\"action\": \"Detect\"," + "\"performanceImpact\": \"Medium or lower\"," + "\"severityLevel\": \"Low or above\"," + "\"confidenceLevel\": \"Medium\"," + "\"serverProtections\": false," + "\"clientProtections\": true," + "\"protectionsFromYear\": 2020," + "\"protectionTags\": [" + "\"tag1\"," + "\"tag2\"" + "]," + "\"protectionIds\": [" + "\"id1\"," + "\"id2\"" + "]" + "}," + "{" + "\"action\": \"Prevent\"," + "\"performanceImpact\": \"Very low\"," + "\"severityLevel\": \"Medium or above\"," + "\"confidenceLevel\": \"Low\"," + "\"serverProtections\": true," + "\"clientProtections\": false," + "\"protectionsFromYear\": 1999," + "\"protectionTags\": [" + "\"tag11\"," + "\"tag22\"" + "]," + "\"protectionIds\": [" + "\"id11\"," + "\"id22\"" + "]" + "}" + "]" + "}"; + + string protection2 = + "{" + "\"defaultAction\": \"Inactive\"," + "\"rules\": [" + "{" + "\"action\": \"Detect\"," + "\"performanceImpact\": \"Medium or lower\"," + "\"severityLevel\": \"Low or above\"," + "\"confidenceLevel\": \"Medium\"" + "}," + "{" + "\"action\": \"Prevent\"" + "}" + "]" + "}"; + + string protection3 = + "{" + "\"defaultAction\": \"Prevent\"," + "\"rules\": []" + "}"; + + string protection4 = + "{" + "\"rules\": [" + "{" + "\"action\": \"Detect\"," + "\"performanceImpact\": \"Medium or lower\"," + "\"severityLevel\": \"Low or above\"," + "\"confidenceLevel\": \"Medium\"" + "}," + "{" + "\"action\": \"Prevent\"" + "}" + "]" + "}"; + + RuleSelector ruleSelector; +}; + +TEST_F(RuleSelectorTest, read_rules) +{ + load(protection); + ostringstream stream; + stream << ruleSelector; + string str = stream.str(); + string result = + "[Rule] action: 1 performanceImpact: 3 severityLevel: 1 confidenceLevel: 3 serverProtections: false" + " clientProtections: true protectionsFromYear: 2020 protectionIds: id1, id2 protectionTags: tag1, tag2;" + "[Rule] action: 0 performanceImpact: 0 severityLevel: 3 confidenceLevel: 1 serverProtections: true" + " clientProtections: false protectionsFromYear: 1999 protectionIds: id11, id22 protectionTags: tag11, tag22;" + "[Rule] action: 0"; + + EXPECT_EQ(result, str); +} + +TEST_F(RuleSelectorTest, read_semi_rules) +{ + load(protection2); + ostringstream stream; + stream << ruleSelector; + string str = stream.str(); + string result = + "[Rule] action: 1 performanceImpact: 3 severityLevel: 1 confidenceLevel: 3;" + "[Rule] action: 0;" + "[Rule] action: 2"; + + EXPECT_EQ(result, str); +} + +TEST_F(RuleSelectorTest, read_empty_rules) +{ + try + { + load(protection3); + } + catch(const Config::ConfigException &e) + { + EXPECT_EQ("rules array is empty", e.getError()); + } +} + +TEST_F(RuleSelectorTest, read_no_default_action) +{ + try + { + load(protection4); + } + catch(const cereal::Exception &e) + { + EXPECT_EQ("JSON Parsing failed - provided NVP (defaultAction) not found", string(e.what())); + } +} diff --git a/components/security_apps/ips/ips_ut/signatures_ut.cc b/components/security_apps/ips/ips_ut/signatures_ut.cc new file mode 100644 index 0000000..0024f0a --- /dev/null +++ b/components/security_apps/ips/ips_ut/signatures_ut.cc @@ -0,0 +1,661 @@ +#include "ips_signatures.h" +#include "ips_common_types.h" + +#include +#include +#include "cptest.h" + +#include "keyword_comp.h" +#include "environment.h" +#include "agent_details.h" +#include "mock/mock_logging.h" +#include "time_proxy.h" +#include "config.h" +#include "config_component.h" +#include "i_keywords_rule.h" +#include "mock/mock_mainloop.h" +#include "generic_rulebase/generic_rulebase.h" +#include "generic_rulebase/parameters_config.h" +#include "generic_rulebase/generic_rulebase_context.h" +#include "encryptor.h" +#include "mock/mock_table.h" + +using namespace testing; +using namespace std; + +MATCHER_P(IsLog, IteratableFields, "") +{ + stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + ar(arg); + } + for (const auto &field : IteratableFields) { + if (ss.str().find(field) == string::npos) return false; + } + + return true; +} + +class MockAgg : Singleton::Provide::SelfInterface +{ + shared_ptr + getHook(const string &, const set &pats) override + { + auto hook = make_shared(); + hook->prepare(pats); + return hook; + } +}; + +class SignatureTest : public Test +{ +public: + SignatureTest() + { + generic_rulebase.preload(); + EXPECT_CALL(logs, getCurrentLogId()).Times(AnyNumber()); + ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state)); + { + stringstream ss; + ss << "[" << signature1 << "]"; + cereal::JSONInputArchive ar(ss); + single_signature.load(ar); + } + { + stringstream ss; + ss << "[" << signature3 << "]"; + cereal::JSONInputArchive ar(ss); + single_signature2.load(ar); + } + { + stringstream ss; + ss << "[" << signature1 << ", " << signature2 << ", " << signature3 << "]"; + cereal::JSONInputArchive ar(ss); + multiple_signatures.load(ar); + } + { + stringstream ss; + ss << "[" << signature_performance_very_low << ", " << signature_performance_low << "]"; + cereal::JSONInputArchive ar(ss); + performance_signatures1.load(ar); + } + { + stringstream ss; + ss << "[" << signature_performance_medium_low << ", " << signature_performance_medium << "]"; + cereal::JSONInputArchive ar(ss); + performance_signatures2.load(ar); + } + { + stringstream ss; + ss << "[" << signature_performance_medium_high << ", " << signature_performance_high << "]"; + cereal::JSONInputArchive ar(ss); + performance_signatures3.load(ar); + } + { + stringstream ss; + ss << "[" << signature_high_confidance << ", " << signature_medium_confidance << "]"; + cereal::JSONInputArchive ar(ss); + high_medium_confidance_signatures.load(ar); + } + } + + ~SignatureTest() + { + if (gen_ctx != nullptr) { + gen_ctx->deactivate(); + gen_ctx.reset(); + } + } + + void + loadExceptions() + { + env.preload(); + env.init(); + + BasicRuleConfig::preload(); + registerExpectedConfiguration("rulebase", "exception"); + + string test_config( + "{" + " \"rulebase\": {" + " \"rulesConfig\": [" + " {" + " \"context\": \"All()\"," + " \"priority\": 1," + " \"ruleId\": \"5eaef0726765c30010bae8bb\"," + " \"ruleName\": \"Acme web API\"," + " \"assetId\": \"5e243effd858007660b758ad\"," + " \"assetName\": \"Acme Power API\"," + " \"parameters\": [" + " {" + " \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\"," + " \"parameterType\": \"exceptions\"," + " \"parameterName\": \"exception\"" + " }" + " ]," + " \"zoneId\": \"\"," + " \"zoneName\": \"\"" + " }" + " ]," + " \"exception\": [" + " {" + " \"context\": \"parameterId(6c3867be-4da5-42c2-93dc-8f509a764003)\"," + " \"match\": {" + " \"type\": \"operator\"," + " \"op\": \"or\"," + " \"items\": [{" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"protectionName\"," + " \"value\": [\"Test1\"]" + " }, {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"protectionName\"," + " \"value\": [\"Test2\"]" + " }, {" + " \"type\": \"condition\"," + " \"op\": \"equals\"," + " \"key\": \"sourceIdentifier\"," + " \"value\": [\"1.1.1.1\"]" + " }]" + " }," + " \"behavior\": {" + " \"key\": \"action\"," + " \"value\": \"accept\"" + " }" + " }" + " ]" + " }" + "}" + ); + + istringstream ss(test_config); + auto i_config = Singleton::Consume::from(config); + i_config->loadConfiguration(ss); + + gen_ctx = make_unique(); + gen_ctx->activate(); + } + + void + load(const IPSSignaturesResource &policy, const string &severity, const string &confidence) + { + setResource(policy, "IPS", "protections"); + stringstream ss; + ss << "{"; + ss << "\"ruleName\": \"rule1\", \"assetName\": \"asset1\", \"practiceName\": \"practice1\","; + ss << "\"assetId\": \"1-1-1\", \"practiceId\": \"2-2-2\","; + ss << "\"defaultAction\" : " << "\"Detect\","; + ss << "\"rules\": ["; + ss << "{"; + ss << "\"action\": \"Prevent\","; + ss << "\"performanceImpact\": \"High or lower\","; + ss << "\"severityLevel\": \"" << severity << "\","; + ss << "\"confidenceLevel\": \"" << confidence << "\""; + ss << "}"; + ss << "]"; + ss << "}"; + cereal::JSONInputArchive ar(ss); + sigs.load(ar); + } + + bool + checkData(const string &data, const string &ctx_name = "HTTP_REQUEST_BODY") + { + ParsedContext body(data, ctx_name, 0); + ScopedContext ctx; + ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), ctx_name); + ctx.registerValue(body.getName(), body.getBuffer()); + return sigs.isMatchedPrevent(body.getName(), body.getBuffer()); + } + + template + void + expectLog(const string &field, Strings ...more_fields) + { + vector all_fields; + all_fields.push_back(field); + expectLog(all_fields, more_fields...); + } + + template + void + expectLog(vector all_fields, const string &field, Strings ...more_fields) + { + all_fields.push_back(field); + expectLog(all_fields, more_fields...); + } + + + void + expectLog(vector all_fields) + { + EXPECT_CALL(logs, sendLog(IsLog(all_fields))); + } + + IPSSignatures sigs; + IPSSignaturesResource single_signature; + IPSSignaturesResource single_signature2; + IPSSignaturesResource multiple_signatures; + IPSSignaturesResource high_medium_confidance_signatures; + IPSSignaturesResource performance_signatures1; + IPSSignaturesResource performance_signatures2; + IPSSignaturesResource performance_signatures3; + NiceMock table; + MockAgg mock_agg; + +private: + GenericRulebase generic_rulebase; + unique_ptr gen_ctx; + NiceMock mock_mainloop; + KeywordComp keywords; + TimeProxyComponent time; + ::Environment env; + ConfigComponent config; + Encryptor encryptor; + AgentDetails details; + StrictMock logs; + IPSEntry ips_state; + + string signature1 = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": [\"Protection_Type_Scanning_Tool\"]," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"fff\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + string signature2 = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test2\"," + "\"maintrainId\": \"102\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": [\"Vul_Type_SQL_Injection\"]," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"ddd\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + string signature3 = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test3\"," + "\"maintrainId\": \"102\"," + "\"severity\": \"High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": [\"Protection_Type_Scanning_Tool\", \"Vul_Type_SQL_Injection\"]," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"ggg\"," + "\"keywords\": \"\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + + string signature_high_confidance = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test3\"," + "\"maintrainId\": \"103\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"High\"," + "\"performanceImpact\": \"Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"hhh\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + + string signature_medium_confidance = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test4\"," + "\"maintrainId\": \"104\"," + "\"severity\": \"Low\"," + "\"confidenceLevel\": \"Medium\"," + "\"performanceImpact\": \"Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"mmm\"," + "\"keywords\": \"data: \\\"mmm\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\"]" + "}" + "}"; + + string signature_performance_very_low = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Very Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"aaa\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + + string signature_performance_low = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"bbb\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + + string signature_performance_medium_low = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium Low\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ccc\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + + string signature_performance_medium = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"ddd\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + + string signature_performance_medium_high = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"Medium High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"eee\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; + + string signature_performance_high = + "{" + "\"protectionMetadata\": {" + "\"protectionName\": \"Test1\"," + "\"maintrainId\": \"101\"," + "\"severity\": \"Medium High\"," + "\"confidenceLevel\": \"Low\"," + "\"performanceImpact\": \"High\"," + "\"lastUpdate\": \"20210420\"," + "\"tags\": []," + "\"cveList\": []" + "}," + "\"detectionRules\": {" + "\"type\": \"simple\"," + "\"SSM\": \"\"," + "\"keywords\": \"data: \\\"fff\\\";\"," + "\"context\": [\"HTTP_REQUEST_BODY\", \"HTTP_RESPONSE_BODY\"]" + "}" + "}"; +}; + +TEST_F(SignatureTest, basic_load_of_signatures) +{ + EXPECT_TRUE(sigs.isEmpty()); + load(single_signature, "Low or above", "Low"); + EXPECT_FALSE(sigs.isEmpty()); + EXPECT_TRUE(sigs.isEmpty("NO_CONTEXT")); + EXPECT_FALSE(sigs.isEmpty("HTTP_REQUEST_BODY")); +} + +TEST_F(SignatureTest, single_signature_matching_override) +{ + load(single_signature, "Low or above", "Low"); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\""); + + EXPECT_TRUE(checkData("fffddd")); + + loadExceptions(); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"Info\""); + + EXPECT_FALSE(checkData("fffddd")); +} + +TEST_F(SignatureTest, source_idetifier_exception) +{ + load(single_signature2, "Low or above", "Low"); + + loadExceptions(); + + expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\""); + + EXPECT_TRUE(checkData("gggddd")); + + ScopedContext ctx; + ctx.registerValue("sourceIdentifiers", "1.1.1.1"); + + expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Info\""); + + EXPECT_FALSE(checkData("gggddd")); +} + +TEST_F(SignatureTest, single_signature_matching) +{ + load(single_signature, "Low or above", "Low"); + + EXPECT_FALSE(checkData("ggg")); + + EXPECT_FALSE(checkData("ddd")); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\""); + + EXPECT_TRUE(checkData("fffddd")); +} + +TEST_F(SignatureTest, context_signature_matching) +{ + load(single_signature, "Low or above", "Low"); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\""); + EXPECT_TRUE(checkData("fff", "HTTP_REQUEST_BODY")); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\""); + EXPECT_TRUE(checkData("fff", "HTTP_RESPONSE_BODY")); + + EXPECT_FALSE(checkData("fff", "HTTP_COMPLETE_URL_DECODED")); +} + +TEST_F(SignatureTest, id_to_log_test) +{ + load(single_signature, "Low or above", "Low"); + + expectLog("\"protectionId\": \"Test1\""); + + EXPECT_TRUE(checkData("fffddd")); +} + +TEST_F(SignatureTest, multiple_signatures_matching) +{ + load(multiple_signatures, "Low or above", "Low"); + EXPECT_FALSE(checkData("hhh")); + + expectLog("\"protectionId\": \"Test2\"", "\"eventSeverity\": \"High\""); + EXPECT_TRUE(checkData("ddd")); + + expectLog("\"protectionId\": \"Test1\"", "\"eventSeverity\": \"High\""); + EXPECT_TRUE(checkData("fff")); + + expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\""); + EXPECT_TRUE(checkData("ggg")); + + // Only one signature is caught + expectLog("\"protectionId\": \"Test2\"", "\"eventSeverity\": \"High\""); + EXPECT_TRUE(checkData("fffdddggg")); +} + +TEST_F(SignatureTest, severity_to_log_test) +{ + load(multiple_signatures, "Low or above", "Low"); + + expectLog("\"matchedSignatureSeverity\": \"Medium High\""); + EXPECT_TRUE(checkData("fff")); + + expectLog("\"matchedSignatureSeverity\": \"Low\""); + EXPECT_TRUE(checkData("ddd")); + + expectLog("\"matchedSignatureSeverity\": \"High\""); + EXPECT_TRUE(checkData("ggg")); +} + +TEST_F(SignatureTest, incident_type) +{ + load(multiple_signatures, "Low or above", "Low"); + + expectLog("\"waapIncidentType\": \"Scanning Tool\""); + EXPECT_TRUE(checkData("fff")); + + expectLog("\"waapIncidentType\": \"SQL Injection\""); + EXPECT_TRUE(checkData("ddd")); + + expectLog("\"waapIncidentType\": \"SQL Injection\""); + EXPECT_TRUE(checkData("ggg")); +} + +TEST_F(SignatureTest, performance_to_log_very_low) +{ + load(performance_signatures1, "Low or above", "Low"); + + EXPECT_FALSE(checkData("ggg")); + + expectLog("\"matchedSignaturePerformance\": \"Very Low\""); + + EXPECT_TRUE(checkData("aaa")); + + expectLog("\"matchedSignaturePerformance\": \"Low\""); + + EXPECT_TRUE(checkData("bbb")); +} + +TEST_F(SignatureTest, performance_to_log_medium_low) +{ + load(performance_signatures2, "Low or above", "Low"); + + EXPECT_FALSE(checkData("ggg")); + + expectLog("\"matchedSignaturePerformance\": \"Medium Low\""); + + EXPECT_TRUE(checkData("ccc")); + + expectLog("\"matchedSignaturePerformance\": \"Medium\""); + + EXPECT_TRUE(checkData("ddd")); +} + +TEST_F(SignatureTest, performance_to_log_medium_high) +{ + load(performance_signatures3, "Low or above", "Low"); + + EXPECT_FALSE(checkData("ggg")); + + expectLog("\"matchedSignaturePerformance\": \"Medium High\""); + + EXPECT_TRUE(checkData("eee")); + + expectLog("\"matchedSignaturePerformance\": \"High\""); + + EXPECT_TRUE(checkData("fff")); +} + +TEST_F(SignatureTest, high_confidance_signatures_matching) +{ + load(high_medium_confidance_signatures, "Low or above", "High"); + EXPECT_FALSE(checkData("ggg")); + + expectLog("\"protectionId\": \"Test3\"", "\"matchedSignatureConfidence\": \"High\""); + EXPECT_TRUE(checkData("hhh")); + + expectLog("\"protectionId\": \"Test4\"", "\"matchedSignatureConfidence\": \"Medium\""); + EXPECT_FALSE(checkData("mmm")); +} diff --git a/components/security_apps/ips/simple_protection.cc b/components/security_apps/ips/simple_protection.cc new file mode 100644 index 0000000..347d3ad --- /dev/null +++ b/components/security_apps/ips/simple_protection.cc @@ -0,0 +1,64 @@ +#include "simple_protection.h" +#include "ips_comp.h" +#include "debug.h" +#include "helper.h" + +using namespace std; + +USE_DEBUG_FLAG(D_IPS); + +SimpleProtection::Impl::Impl( + const string &_sig_name, + const string &ssm, + const string &keyword, + const vector &_context + ) + : + sig_name(_sig_name), + context(_context) +{ + string deobfuscated_keyword = IPSHelper::deobfuscateKeyword(keyword); + if (deobfuscated_keyword != "") { + auto compiled = Singleton::Consume::by()->genRule(deobfuscated_keyword); + if (!compiled.ok()) { + reportConfigurationError( + "Failed to complie keywords '" + keyword + "' in signature " + sig_name + ": " + compiled.getErr() + ); + } + rule = compiled.unpackMove(); + } + + auto deobfuscated_ssm = IPSHelper::deobfuscateString(ssm); + if (deobfuscated_ssm != "") { + auto temp_pattern = PMHook::lineToPattern(deobfuscated_ssm); + if (!temp_pattern.ok()) reportConfigurationError("Failed first tier pattern: " + temp_pattern.getErr()); + pattern = temp_pattern.unpackMove(); + } + + if (deobfuscated_keyword == "" && deobfuscated_ssm == "") { + reportConfigurationError("Both Simple String and Keyword are empty in a simple protection " + sig_name); + } +} + +using MatchType = IPSSignatureSubTypes::BaseSignature::MatchType; + +MatchType +SimpleProtection::Impl::getMatch(const set &matches) const +{ + dbgTrace(D_IPS) << "Entering signature"; + if (!pattern.empty() && matches.find(pattern) == matches.end()) return MatchType::NO_MATCH; + + dbgTrace(D_IPS) << "Checking for rule"; + if (!rule) return MatchType::MATCH; + + dbgTrace(D_IPS) << "Running keywords"; + return rule->isMatch() ? MatchType::MATCH : MatchType::NO_MATCH; +} + +set +SimpleProtection::Impl::patternsInSignature() const +{ + set res; + if (!pattern.empty()) res.insert(pattern); + return res; +} diff --git a/components/security_apps/ips/snort_basic_policy.cc b/components/security_apps/ips/snort_basic_policy.cc new file mode 100644 index 0000000..592fbc7 --- /dev/null +++ b/components/security_apps/ips/snort_basic_policy.cc @@ -0,0 +1,45 @@ +#include "snort_basic_policy.h" +#include +#include +#include +#include +#include +#include "ips_signatures.h" +#include "helper.h" +#include "config.h" +#include "common.h" + +USE_DEBUG_FLAG(D_IPS); + +using namespace std; + +void +SnortRuleSelector::load(cereal::JSONInputArchive &ar) +{ + string mode; + ar(cereal::make_nvp("mode", mode), cereal::make_nvp("files", file_names)); + + if (mode == "Inactive") action = IPSSignatureSubTypes::SignatureAction::IGNORE; + else if (mode == "Disabled") action = IPSSignatureSubTypes::SignatureAction::IGNORE; + else if (mode == "Detect") action = IPSSignatureSubTypes::SignatureAction::DETECT; + else if (mode == "Prevent") action = IPSSignatureSubTypes::SignatureAction::PREVENT; + else reportConfigurationError("invalid action value " + mode); +} + +vector +SnortRuleSelector::selectSignatures() const +{ + vector res; + + if (action == IPSSignatureSubTypes::SignatureAction::IGNORE) return res; + + auto signatures = getResource("IPSSnortSigs", "protections"); + if (!signatures.ok()) return res; + + for (auto &file : file_names) { + for (auto &signature : (*signatures).getSignatures(file)) { + res.emplace_back(signature, action); + } + } + return res; +} diff --git a/components/security_apps/orchestration/downloader/downloader.cc b/components/security_apps/orchestration/downloader/downloader.cc index 3d2701c..d0e66de 100755 --- a/components/security_apps/orchestration/downloader/downloader.cc +++ b/components/security_apps/orchestration/downloader/downloader.cc @@ -21,6 +21,11 @@ #include "rest.h" #include "cereal/external/rapidjson/document.h" +#include "customized_cereal_map.h" +#include "cereal/archives/json.hpp" +#include "cereal/types/vector.hpp" +#include "cereal/types/string.hpp" + #include using namespace std; @@ -28,6 +33,31 @@ using namespace rapidjson; USE_DEBUG_FLAG(D_ORCHESTRATOR); +// LCOV_EXCL_START Reason: WA for NSaaS upgrade +class TenantProfileMap +{ +public: + void + load(const string &raw_value) + { + vector tenants_and_profiles; + { + stringstream string_stream(raw_value); + cereal::JSONInputArchive archive(string_stream); + cereal::load(archive, tenants_and_profiles); + } + for (const auto &tenant_profile_pair : tenants_and_profiles) { + value.push_back(tenant_profile_pair); + } + } + + const vector & getValue() const { return value; } +private: + vector value; +}; + +// LCOV_EXCL_STOP + class Downloader::Impl : Singleton::Provide::From { public: @@ -51,6 +81,9 @@ public: const string &service_name ) const override; + void createTenantProfileMap(); + string getProfileFromMap(const string &tenant_id) const override; + private: Maybe downloadFileFromFogByHTTP( const GetResourceFile &resourse_file, @@ -74,6 +107,7 @@ private: tuple splitQuery(const string &query) const; string vectorToPath(const vector &vec) const; string dir_path; + map tenant_profile_map; }; void @@ -111,6 +145,42 @@ Downloader::Impl::downloadFileFromFog( return file_path; } +void +Downloader::Impl::createTenantProfileMap() +{ + dbgFlow(D_ORCHESTRATOR) << "Creating a tenant-profile map from the agent settings"; + tenant_profile_map.clear(); + auto maybe_tenant_profile_map = getProfileAgentSetting("TenantProfileMap"); + if (maybe_tenant_profile_map.ok()) { + dbgTrace(D_ORCHESTRATOR) << "Managed to read the TenantProfileMap agent settings"; + TenantProfileMap tpm = maybe_tenant_profile_map.unpack(); + for (const string &str : tpm.getValue()) { + string delimiter = ":"; + string tenant = str.substr(0, str.find(delimiter)); + string profile = str.substr(str.find(delimiter) + 1); + dbgTrace(D_ORCHESTRATOR) + << "Loading into the map. Tenant: " + << tenant + << " Profile: " + << profile; + tenant_profile_map[tenant] = profile; + } + } else { + dbgTrace(D_ORCHESTRATOR) << "Couldn't load the TenantProfileMap agent settings"; + } +} + +// LCOV_EXCL_START Reason: NSaaS old profiles support +string +Downloader::Impl::getProfileFromMap(const string &tenant_id) const +{ + if (tenant_profile_map.find(tenant_id) == tenant_profile_map.end()) { + return ""; + } + return tenant_profile_map.at(tenant_id); +} +// LCOV_EXCL_STOP + Maybe, string>> Downloader::Impl::downloadVirtualFileFromFog( const GetResourceFile &resourse_file, @@ -130,8 +200,10 @@ Downloader::Impl::downloadVirtualFileFromFog( Document document; document.Parse(downloaded_data.unpack().c_str()); - if (document.HasParseError()) return genError("JSON file is not valid."); - + if (document.HasParseError()) { + dbgWarning(D_ORCHESTRATOR) << "JSON file is not valid"; + return genError("JSON file is not valid."); + } const Value &tenants_data = document[tenants_key.c_str()]; for (Value::ConstValueIterator itr = tenants_data.Begin(); itr != tenants_data.End(); ++itr) { @@ -145,9 +217,21 @@ Downloader::Impl::downloadVirtualFileFromFog( if (artifact_data != itr->MemberEnd()) { auto profile_id_obj = itr->FindMember(profile_id_key.c_str()); - if (profile_id_obj == itr->MemberEnd()) continue; + string profile_id; + if (profile_id_obj == itr->MemberEnd()) { + if (tenant_profile_map.count(tenant_id)) { + dbgWarning(D_ORCHESTRATOR) + << "Forcing profile ID to be " + << getProfileFromMap(tenant_id); + profile_id = getProfileFromMap(tenant_id); + } else { + dbgWarning(D_ORCHESTRATOR) << "Couldn't force profile ID"; + continue; + } + } - string profile_id = profile_id_obj->value.GetString(); + if (profile_id.empty()) profile_id = profile_id_obj->value.GetString(); + dbgTrace(D_ORCHESTRATOR) << "Found a profile ID " << profile_id; string file_path = dir_path + "/" + resourse_file.getFileName() + "_" + @@ -161,6 +245,9 @@ Downloader::Impl::downloadVirtualFileFromFog( if (orchestration_tools->writeFile(buffer.GetString(), file_path)) { res.insert({{tenant_id, profile_id}, file_path}); } + + orchestration_tools->fillKeyInJson(file_path, "profileID", profile_id); + orchestration_tools->fillKeyInJson(file_path, "tenantID", tenant_id); continue; } @@ -387,4 +474,5 @@ Downloader::preload() registerExpectedConfiguration("orchestration", "Default file download path"); registerExpectedConfiguration("orchestration", "Self signed certificates acceptable"); registerExpectedConfiguration("orchestration", "Add tenant suffix"); + registerConfigLoadCb([this]() { pimpl->createTenantProfileMap(); }); } diff --git a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc index 2c59a25..14c6721 100755 --- a/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc +++ b/components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc @@ -345,6 +345,8 @@ TEST_F(DownloaderTest, download_virtual_policy) EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_0000_file, "/tmp/virtualPolicy_0000_profile_1234.download")) .WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, fillKeyInJson(_, _, _)).WillRepeatedly(Return()); + EXPECT_CALL(mock_orchestration_tools, writeFile(tenant_1111_file, "/tmp/virtualPolicy_1111_profile_1235.download")) .WillOnce(Return(true)); @@ -429,6 +431,8 @@ TEST_F(DownloaderTest, download_virtual_settings) ) ).WillOnce(Return(true)); + EXPECT_CALL(mock_orchestration_tools, fillKeyInJson(_, _, _)).WillRepeatedly(Return()); + stringstream file_path; file_path << "/tmp/virtualSettings_4c721b40-85df-4364-be3d-303a10ee9789" "_profile_4c721b40-85df-4364-be3d-303a10ee9780.download"; diff --git a/components/security_apps/orchestration/include/mock/mock_downloader.h b/components/security_apps/orchestration/include/mock/mock_downloader.h index ac2160b..8b357bd 100755 --- a/components/security_apps/orchestration/include/mock/mock_downloader.h +++ b/components/security_apps/orchestration/include/mock/mock_downloader.h @@ -40,6 +40,12 @@ public: downloadFileFromURL, Maybe(const std::string &, const std::string &, Package::ChecksumTypes, const std::string &) ); + + MOCK_CONST_METHOD1( + getProfileFromMap, + std::string(const std::string &) + ); + }; #endif // __MOCK_DOWNLOADER_H__ diff --git a/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h b/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h index 20b6cfb..47c5fe3 100755 --- a/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h +++ b/components/security_apps/orchestration/include/mock/mock_orchestration_tools.h @@ -31,13 +31,6 @@ operator<<(std::ostream &os, const std::map &) return os; } -template -std::ostream & -operator<<(std::ostream &os, const Maybe> &) -{ - return os; -} - class MockOrchestrationTools : public Singleton::Provide::From> @@ -56,6 +49,7 @@ public: Maybe>(const std::string &, const std::string &, const std::string &) ); MOCK_CONST_METHOD1(doesFileExist, bool(const std::string &)); + MOCK_CONST_METHOD3(fillKeyInJson, void(const std::string &, const std::string &, const std::string &)); MOCK_CONST_METHOD1(createDirectory, bool(const std::string &)); MOCK_CONST_METHOD1(doesDirectoryExist, bool(const std::string &)); MOCK_CONST_METHOD1(executeCmd, bool(const std::string &)); diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 2b4cbe9..7941e7c 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1113,17 +1113,26 @@ private: // Download virtual policy bool is_empty = true; GetResourceFile resource_v_policy_file(GetResourceFile::ResourceFileType::VIRTUAL_POLICY); + I_Downloader *downloader = Singleton::Consume::by(); for (const auto &tenant: *updated_policy_tenants) { if (!tenant.getVersion().empty()) { is_empty = false; + + string profile_to_use = tenant.getProfileID().empty() ? + downloader->getProfileFromMap(tenant.getTenantID()) : + tenant.getProfileID(); + dbgTrace(D_ORCHESTRATOR) << "Adding a tenant to the multi-tenant list. Tenant: " - << tenant.getTenantID(); + << tenant.getTenantID() + << " Profile: " + << profile_to_use; auto tenant_manager = Singleton::Consume::by(); - tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), tenant.getProfileID()); + + tenant_manager->addActiveTenantAndProfile(tenant.getTenantID(), profile_to_use); resource_v_policy_file.addTenant( tenant.getTenantID(), - tenant.getProfileID(), + profile_to_use, tenant.getVersion(), tenant.getChecksum() ); @@ -1132,7 +1141,7 @@ private: if (!is_empty) { auto new_virtual_policy_files = - Singleton::Consume::by()->downloadVirtualFileFromFog( + downloader->downloadVirtualFileFromFog( resource_v_policy_file, I_OrchestrationTools::SELECTED_CHECKSUM_TYPE ); @@ -1151,9 +1160,24 @@ private: for (const auto &tenant: *updated_settings_tenants) { if (!tenant.getVersion().empty()) { is_empty = false; + + string profile_to_use = tenant.getProfileID().empty() ? + downloader->getProfileFromMap(tenant.getTenantID()) : + tenant.getProfileID(); + + dbgTrace(D_ORCHESTRATOR) + << "Handling virtual settings: Tenant ID: " + << tenant.getTenantID() + << ", Profile ID: " + << profile_to_use + << ", version: " + << tenant.getVersion() + << ", checksum: " + << tenant.getChecksum(); + resource_v_settings_file.addTenant( tenant.getTenantID(), - tenant.getProfileID(), + profile_to_use, tenant.getVersion(), tenant.getChecksum() ); @@ -1169,6 +1193,11 @@ private: if (new_virtual_settings_files.ok()) { for (const auto &tenant_file: *new_virtual_settings_files) { auto tenant_profile = TenantProfilePair(tenant_file.first.first, tenant_file.first.second); + dbgTrace(D_ORCHESTRATOR) + << "Downloaded a file from the FOG: Tenant ID: " + << tenant_profile.getTenantId() + << ", Profile ID: " + << tenant_profile.getProfileId(); sorted_files[tenant_profile].push_back(tenant_file.second); } } diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index bae869d..6411d73 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -45,6 +45,7 @@ public: bool removeFile(const string &path) const override; bool copyFile(const string &src_path, const string &dst_path) const override; bool doesFileExist(const string &file_path) const override; + void fillKeyInJson(const string &filename, const string &_key, const string &_val) const override; bool createDirectory(const string &directory_path) const override; bool doesDirectoryExist(const string &dir_path) const override; bool executeCmd(const string &cmd) const override; @@ -78,6 +79,41 @@ checkExistence(const string &path, bool is_dir) } } +// LCOV_EXCL_START Reason: NSaaS upgrade WA +void +OrchestrationTools::Impl::fillKeyInJson(const string &filename, const string &_key, const string &_val) const +{ + // Load the JSON file into a string + std::ifstream ifs(filename); + std::string jsonStr((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); + + dbgTrace(D_ORCHESTRATOR) << "Trying to parse " << filename; + // Parse the JSON string + Document doc; + doc.Parse(jsonStr.c_str()); + + // Check if the key exists + if (doc.HasMember(_key.c_str())) { + dbgTrace(D_ORCHESTRATOR) << _key << " already exists."; + return; + } + + // Add the key with value + Value key(_key.c_str(), doc.GetAllocator()); + Value val(_val.c_str(), doc.GetAllocator()); + doc.AddMember(key, val, doc.GetAllocator()); + + // Write the modified JSON to a new file + StringBuffer buffer; + Writer writer(buffer); + doc.Accept(writer); + std::ofstream ofs(filename); + ofs << buffer.GetString() << std::endl; + + dbgTrace(D_ORCHESTRATOR) << _key << " added with val " << _val; +} +// LCOV_EXCL_STOP + bool OrchestrationTools::Impl::doesFileExist(const string &file_path) const { diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index 306e6f6..f925d2a 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -265,14 +265,14 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource) EXPECT_CALL(mock_service_controller, getPolicyVersion()) .Times(2).WillRepeatedly(ReturnRef(first_policy_version)); - vector active_tenants = { "1236", "1235" }; + set active_tenants = { "1236", "1235" }; EXPECT_CALL(tenant_manager, fetchActiveTenants()).WillOnce(Return(active_tenants)); EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1235", "2311")); EXPECT_CALL(tenant_manager, addActiveTenantAndProfile("1236", "2611")); - vector first_tenant_profiles = { "2611" }; - vector second_tenant_profiles = { "2311"}; + set first_tenant_profiles = { "2611" }; + set second_tenant_profiles = { "2311"}; EXPECT_CALL( tenant_manager, fetchProfileIds("1236")).WillRepeatedly(Return(first_tenant_profiles) diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index 61af44d..6639b43 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -1466,8 +1466,8 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles) make_pair("/etc/cp/conf/tenant2_profile_1235_policy.json", "")} }; - vector ids = {"family1_id2"}; - vector empty_ids; + set ids = {"family1_id2"}; + set empty_ids; EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillOnce(Return(ids)); EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillOnce(Return(empty_ids)); @@ -1592,7 +1592,7 @@ TEST_F(ServiceControllerTest, cleanup_virtual_files) "222222\n" "333333\n"; - vector active_tenants = { + set active_tenants = { "222222" }; diff --git a/components/security_apps/waap/include/i_serialize.h b/components/security_apps/waap/include/i_serialize.h index ee1c43d..ec502db 100755 --- a/components/security_apps/waap/include/i_serialize.h +++ b/components/security_apps/waap/include/i_serialize.h @@ -189,10 +189,10 @@ protected: "object sent successfully after " << i << " retry attempts"; return true; } - dbgWarning(D_WAAP) << "Failed to send object. Attempt: " << i; + dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i; mainloop->yield(wait_next_attempt); } - dbgError(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << + dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << max_send_obj_retries; return false; } @@ -243,10 +243,10 @@ protected: "object sent successfully after " << i << " retry attempts"; return true; } - dbgWarning(D_WAAP) << "Failed to send object. Attempt: " << i; + dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i; mainloop->yield(wait_next_attempt); } - dbgError(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << + dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << max_send_obj_retries; return false; } diff --git a/components/security_apps/waap/reputation/reputation_features_agg.cc b/components/security_apps/waap/reputation/reputation_features_agg.cc index 52c5c97..fde179d 100755 --- a/components/security_apps/waap/reputation/reputation_features_agg.cc +++ b/components/security_apps/waap/reputation/reputation_features_agg.cc @@ -348,6 +348,10 @@ ReputationFeaturesAgg::Impl::reportReputationFeatures() I_MainLoop *i_mainLoop = Singleton::Consume::by(); string tenantId = agentDetails->getTenantId(); + if (tenantId.empty()) + { + tenantId = "Elpis"; + } string agentId = agentDetails->getAgentId(); if (Singleton::exists()) { diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc index 9c99890..8003844 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc @@ -137,9 +137,13 @@ bool ConfidenceCalculator::postData() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url; WindowLogPost currentWindow(m_time_window_logger_backup); - return sendNoReplyObjectWithRetry(currentWindow, + bool ok = sendNoReplyObjectWithRetry(currentWindow, I_Messaging::Method::PUT, url); + if (!ok) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url; + } + return ok; } void ConfidenceCalculator::pullData(const std::vector& files) @@ -149,7 +153,7 @@ void ConfidenceCalculator::pullData(const std::vector& files) mergeProcessedFromRemote(); } std::string url = getPostDataUrl(); - std::string sentFile = url.erase(0, url.find_first_of('/') + 1); + std::string sentFile = url.erase(0, strlen("/storage/waap/")); dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "pulling files, skipping: " << sentFile; for (auto file : files) { @@ -159,10 +163,15 @@ void ConfidenceCalculator::pullData(const std::vector& files) } dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Pulling the file: " << file; WindowLogGet getWindow; - sendObjectWithRetry(getWindow, + bool ok = sendObjectWithRetry(getWindow, I_Messaging::Method::GET, getUri() + "/" + file); + if (!ok) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get file: " << file; + continue; + } + KeyValSourcesLogger remoteLogger = getWindow.getWindowLogger().unpack(); for (auto& log : remoteLogger) { @@ -215,6 +224,10 @@ void ConfidenceCalculator::pullProcessedData(const std::vector& fil m_confidence_level = getConfFile.getConfidenceLevels().unpackMove(); } } + // is_first_pull = false -> at least one file was downloaded and merged + if (is_first_pull) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state"; + } } void ConfidenceCalculator::postProcessedData() diff --git a/components/security_apps/waap/waap_clib/ScannerDetector.cc b/components/security_apps/waap/waap_clib/ScannerDetector.cc index 1688042..fd2c3b2 100755 --- a/components/security_apps/waap/waap_clib/ScannerDetector.cc +++ b/components/security_apps/waap/waap_clib/ScannerDetector.cc @@ -100,9 +100,13 @@ bool ScannerDetector::postData() dbgTrace(D_WAAP) << "Sending the data to: " << url; SourcesMonitorPost currentWindow(m_sources_monitor_backup); - return sendNoReplyObjectWithRetry(currentWindow, + bool ok = sendNoReplyObjectWithRetry(currentWindow, I_Messaging::Method::PUT, url); + if (!ok) { + dbgError(D_WAAP) << "Failed to post collected data to: " << url; + } + return ok; } void ScannerDetector::pullData(const std::vector& files) @@ -118,10 +122,15 @@ void ScannerDetector::pullData(const std::vector& files) } dbgTrace(D_WAAP) << "Pulling the file: " << file; SourcesMonitorGet getMonitor; - sendObjectWithRetry(getMonitor, + bool ok = sendObjectWithRetry(getMonitor, I_Messaging::Method::GET, getUri() + "/" + file); + if (!ok) { + dbgError(D_WAAP) << "Failed to get data from: " << file; + continue; + } + SourceKeyValsMap remoteMonitor = getMonitor.getSourcesMonitor().unpack(); for (const auto& srcData : remoteMonitor) { diff --git a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc index 708687a..6822560 100755 --- a/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc +++ b/components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc @@ -96,9 +96,13 @@ bool TrustedSourcesConfidenceCalculator::postData() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url; TrsutedSourcesLogger logger(m_logger); - return sendNoReplyObjectWithRetry(logger, + bool ok = sendNoReplyObjectWithRetry(logger, I_Messaging::Method::PUT, url); + if (!ok) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url; + } + return ok; } void TrustedSourcesConfidenceCalculator::pullData(const std::vector& files) @@ -116,7 +120,12 @@ void TrustedSourcesConfidenceCalculator::pullData(const std::vector bool res = sendObjectWithRetry(getTrustFile, I_Messaging::Method::GET, getUri() + "/" + file); - if (res && getTrustFile.getTrustedLogs().ok()) + if (!res) + { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get file: " << file; + continue; + } + if (getTrustFile.getTrustedLogs().ok()) { mergeFromRemote(getTrustFile.getTrustedLogs().unpack()); } @@ -134,20 +143,22 @@ void TrustedSourcesConfidenceCalculator::updateState(const std::vector& files) -{ +void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector& files) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources"; - for (auto file : files) - { + bool pull_ok = false; + for (auto file: files) { GetTrustedFile getTrustFile; bool res = sendObjectWithRetry(getTrustFile, I_Messaging::Method::GET, getUri() + "/" + file); - if (res && getTrustFile.getTrustedLogs().ok()) - { + if (res && getTrustFile.getTrustedLogs().ok()) { mergeFromRemote(getTrustFile.getTrustedLogs().unpack()); + pull_ok = true; } } + if (!pull_ok && !files.empty()) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to pull state data"; + } } void TrustedSourcesConfidenceCalculator::postProcessedData() diff --git a/components/security_apps/waap/waap_clib/WaapSampleValue.cc b/components/security_apps/waap/waap_clib/WaapSampleValue.cc index c776800..c6ac241 100644 --- a/components/security_apps/waap/waap_clib/WaapSampleValue.cc +++ b/components/security_apps/waap/waap_clib/WaapSampleValue.cc @@ -37,5 +37,6 @@ SampleValue::getSampleString() const void SampleValue::findMatches(const Regex &pattern, std::vector &matches) const { - pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr); + static const size_t maxMatchesPerSignature = 5; + pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr, maxMatchesPerSignature); } diff --git a/components/security_apps/waap/waap_clib/Waf2Regex.cc b/components/security_apps/waap/waap_clib/Waf2Regex.cc index b6dfe02..c6b1ad4 100755 --- a/components/security_apps/waap/waap_clib/Waf2Regex.cc +++ b/components/security_apps/waap/waap_clib/Waf2Regex.cc @@ -140,7 +140,7 @@ bool SingleRegex::hasMatch(const std::string& s) const { return true; } -size_t SingleRegex::findAllMatches(const std::string& s, std::vector& matches) const { +size_t SingleRegex::findAllMatches(const std::string& s, std::vector& matches, size_t maxMatches) const { size_t matchesCount = 0; // Optimized regex that always immediately reports a "simulated" match without spending time to do a scan @@ -234,7 +234,7 @@ size_t SingleRegex::findAllMatches(const std::string& s, std::vector // continue searching for next match starting from end of this match // (first two entries in ov[] are start and end offsets of current full match) startOffset = ov[1]; - } while (true); + } while (matchesCount < maxMatches); return matchesCount; } @@ -418,7 +418,7 @@ bool Regex::hasMatch(const std::string& s) const { } size_t Regex::findAllMatches(const std::string& s, std::vector& matches, - const Waap::RegexPreconditions::PmWordSet *pmWordSet) const { + const Waap::RegexPreconditions::PmWordSet *pmWordSet, size_t maxMatches) const { matches.clear(); if (m_regexPreconditions && pmWordSet) { @@ -442,7 +442,7 @@ size_t Regex::findAllMatches(const std::string& s, std::vector& matc } // Scan only regexes that are enabled by aho-corasick scan - m_sre[regexIndex]->findAllMatches(s, matches); + m_sre[regexIndex]->findAllMatches(s, matches, maxMatches); dbgTrace(D_WAAP_REGEX) << "Regex['" << m_sre[regexIndex]->getName() << "',index=" << regexIndex << "]::findAllMatches(): " << matches.size() << " matches found (so far)"; @@ -453,7 +453,7 @@ size_t Regex::findAllMatches(const std::string& s, std::vector& matc else { // When optimization is disabled - scan all regexes for (SingleRegex* pSingleRegex : m_sre) { - pSingleRegex->findAllMatches(s, matches); + pSingleRegex->findAllMatches(s, matches, maxMatches); dbgTrace(D_WAAP_REGEX) << "Regex['" << m_regexName << "']['" << pSingleRegex->getName() << "']::findAllMatches(): " << matches.size() << " matches found (so far)"; } diff --git a/components/security_apps/waap/waap_clib/Waf2Regex.h b/components/security_apps/waap/waap_clib/Waf2Regex.h index 8b80292..2a15f41 100755 --- a/components/security_apps/waap/waap_clib/Waf2Regex.h +++ b/components/security_apps/waap/waap_clib/Waf2Regex.h @@ -55,7 +55,8 @@ public: const std::string ®exMatchName="", const std::string ®exMatchValue=""); ~SingleRegex(); bool hasMatch(const std::string &s) const; - size_t findAllMatches(const std::string &s, std::vector &matches) const; + size_t findAllMatches(const std::string &s, std::vector &matches, + size_t max_matches = std::string::npos) const; size_t findMatchRanges(const std::string &s, std::vector &matchRanges) const; const std::string &getName() const; private: @@ -76,8 +77,8 @@ public: std::shared_ptr regexPreconditions); ~Regex(); bool hasMatch(const std::string &s) const; - size_t findAllMatches(const std::string &v, std::vector &maches, - const Waap::RegexPreconditions::PmWordSet *pmWordSet=nullptr) const; + size_t findAllMatches(const std::string &v, std::vector &matches, + const Waap::RegexPreconditions::PmWordSet *pmWordSet=nullptr, size_t max_matches = std::string::npos) const; std::string sub(const std::string &s, const std::string &repl="") const; // Run regex search, and for each found match - run callback. // The callback can cancel replacement of the match (leave source match "as-is"), provide a replacement string, diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index 48eed04..32b02f9 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -1,3 +1,4 @@ add_subdirectory(http_transaction_data) add_subdirectory(ip_utilities) +add_subdirectory(keywords) add_subdirectory(pm) diff --git a/components/utils/keywords/CMakeLists.txt b/components/utils/keywords/CMakeLists.txt new file mode 100644 index 0000000..2ff136f --- /dev/null +++ b/components/utils/keywords/CMakeLists.txt @@ -0,0 +1,2 @@ +add_library(keywords keywords_rule.cc single_keyword.cc data_keyword.cc pcre_keyword.cc length_keyword.cc byte_extract_keyword.cc compare_keyword.cc jump_keyword.cc stateop_keyword.cc no_match_keyword.cc) +add_subdirectory(keywords_ut) diff --git a/components/utils/keywords/byte_extract_keyword.cc b/components/utils/keywords/byte_extract_keyword.cc new file mode 100644 index 0000000..8de609c --- /dev/null +++ b/components/utils/keywords/byte_extract_keyword.cc @@ -0,0 +1,319 @@ +#include "single_keyword.h" +#include "output.h" +#include "debug.h" + +#include +#include +#include "limits.h" + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class ByteExtractKeyword : public SingleKeyword +{ +public: + explicit ByteExtractKeyword(const vector &attr, VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState *prev) const override; + +private: + enum class BaseId + { + BIN, + HEX = 16, + DEC = 10, + OCT = 8 + }; + + void + setOffset(const KeywordAttr &attr, const VariablesMapping &vars) + { + offset.setAttr(attr, vars, "byte_extract"); + } + + void + setRelative(const KeywordAttr &attr, const VariablesMapping &) + { + is_relative.setAttr(attr, "byte_extract"); + } + + void + setLittleEndian(const KeywordAttr &attr, const VariablesMapping &) + { + is_little_end.setAttr(attr, "byte_extract"); + } + + void + setDataType(const KeywordAttr &attr, const VariablesMapping &) + { + if (data_type != BaseId::BIN) { + throw KeywordError("Double definition of the data type in the 'byte_extract' keyword"); + } + + auto &vec = attr.getParams(); + if (vec.size() != 2) throw KeywordError("Malformed data type in the 'byte_extract' keyword"); + + if (vec[1] == "hex") { + data_type = BaseId::HEX; + } else if (vec[1] == "dec") { + data_type = BaseId::DEC; + } else if (vec[1] == "oct") { + data_type = BaseId::OCT; + } else { + throw KeywordError("Unknown data type in the 'byte_extract' keyword: " + vec[1]); + } + } + + void + setContext(const KeywordAttr &attr, const VariablesMapping &) + { + ctx.setAttr(attr, "byte_extract"); + } + + void + setAlign(const KeywordAttr &attr, const VariablesMapping &) + { + if (align != 1) throw KeywordError("Double definition of the 'align' in the 'byte_extract' keyword"); + auto &vec = attr.getParams(); + if (vec.size() != 2) throw KeywordError("Malformed 'align' in the 'byte_extract' keyword"); + + if (vec[1] == "2") { + align = 2; + } else if (vec[1] == "4") { + align = 4; + } else { + throw KeywordError("Unknown 'align' in the 'byte_extract' keyword: " + vec[1]); + } + } + + bool + isConstant() const + { + return !is_relative && bytes.isConstant() && offset.isConstant(); + } + + pair getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const; + uint applyAlignment(uint value) const; + Maybe readValue(uint start, uint length, const Buffer &buf) const; + Maybe readStringValue(uint start, uint length, const Buffer &buf) const; + + NumericAttr bytes; + uint var_id; + NumericAttr offset; + BoolAttr is_relative; + BoolAttr is_little_end; + BaseId data_type = BaseId::BIN; + int align = 1; + CtxAttr ctx; + + static const map setops; +}; + +const map +ByteExtractKeyword::setops = { + { "offset", &ByteExtractKeyword::setOffset }, + { "relative", &ByteExtractKeyword::setRelative }, + { "little_endian", &ByteExtractKeyword::setLittleEndian }, + { "string", &ByteExtractKeyword::setDataType }, + { "part", &ByteExtractKeyword::setContext }, + { "align", &ByteExtractKeyword::setAlign } +}; + +ByteExtractKeyword::ByteExtractKeyword(const vector &attrs, VariablesMapping &vars) + : + offset() +{ + //two requied attributes - number of bytes and var name + if (attrs.size() < 2) throw KeywordError("Invalid number of attributes in the 'byte_extract' keyword"); + + //parisng first attribute (Required) - number of bytes + auto &bytes_param = attrs[0].getParams(); + if (bytes_param.size() != 1) { + throw KeywordError("More than one element in the 'bytes' in the 'byte_extract' keyword"); + } + bytes.setAttr("bytes", bytes_param[0], vars, "byte_extract", static_cast(BaseId::DEC), true); + if (bytes.isConstant() && bytes.evalAttr(nullptr) == 0) { + throw KeywordError("Number of bytes is zero in the 'byte_extract' keyword"); + } + + //parisng second attribute (Required) - variable name + auto &var_name_param = attrs[1].getParams(); + if (var_name_param.size() != 1) { + throw KeywordError("More than one element in the variable name in the 'byte_extract' keyword"); + } + const string &var_name = var_name_param[0]; + auto curr = setops.find(var_name); + if (curr != setops.end()) { + throw KeywordError("'" + var_name + "' cannot be the variable name in the 'byte_extract' keyword"); + } + if (isdigit(var_name[0]) || var_name[0] == '-') { + throw KeywordError("Malformed variable name in the 'byte_extract' keyword"); + } + + var_id = vars.addNewVariable(var_name); + + //parsing the other optional attributes + for (uint i = 2; i < attrs.size(); i++) { + auto curr = setops.find(attrs[i].getAttrName()); + if (curr == setops.end()) { + throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'byte_extract' keyword"); + } + auto set_func = curr->second; + (this->*set_func)(attrs[i], vars); + } + + if (data_type == BaseId::BIN) { + if (!bytes.isConstant()) { + throw KeywordError("Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword"); + } + int num_bytes = bytes.evalAttr(nullptr); + if (num_bytes != 1 && num_bytes != 2 && num_bytes != 4) { + throw KeywordError("Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword"); + } + if (is_little_end && num_bytes == 1) { + throw KeywordError( + "Little endian is set, " + "but the number of bytes is invalid in the 'byte_extract' keyword" + ); + } + if (align != 1) { + throw KeywordError("The 'align' is set and data type is binary in the 'byte_extract' keyword"); + } + } else { + if (is_little_end) { + throw KeywordError("Little endian is set, but the data type is not binary in the 'byte_extract' keyword"); + } + } +} + +static uint +addOffset(uint offset, int add) +{ + if (add < 0 && offset < static_cast(-add)) { + dbgWarning(D_KEYWORD) + << "The offset was set to 0 " + << "due to an attempt to jump before the beginning of the buffer in the 'jump' keyword"; + return 0; + } + return offset + add; +} + +pair +ByteExtractKeyword::getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const +{ + uint relative_offset = is_relative ? prev->getOffset(ctx) : 0; + int offset_attr = offset.evalAttr(prev); + uint start_offset = addOffset(relative_offset, offset_attr); + + if (start_offset >= buf_size) return make_pair(0, 0); + + uint length = buf_size - start_offset; + return make_pair(start_offset, length); +} + +Maybe +ByteExtractKeyword::readValue(uint start, uint length, const Buffer &buf) const +{ + if (data_type != BaseId::BIN) return readStringValue(start, length, buf); + + uint res = 0; + for (uint i = 0; i < length; i++) { + uint ch = buf[start + i]; + if (is_little_end) { + ch <<= 8*i; + res += ch; + } else { + res <<= 8; + res += ch; + } + } + return res; +} + +Maybe +ByteExtractKeyword::readStringValue(uint start, uint length, const Buffer &buf) const +{ + const u_char *data = buf.getPtr(start, length).unpack(); // start and length were checked outside of the function + string val_str(reinterpret_cast(data), length); + + uint base = static_cast(data_type); + try { + size_t idx; + auto res = stoul(val_str, &idx, base); + if (idx != val_str.length()) throw invalid_argument(""); + if (res > INT_MAX) { + throw out_of_range(""); + } + return res; + } + catch (invalid_argument &) { + return genError("Unable to convert the \"" + val_str + "\" to a number due to an invalid argument"); + } + catch (out_of_range &) { + return genError( + "Unable to convert the \"" + + val_str + + "\" to a number. The maximum is: " + + to_string(INT_MAX) + ); + } +} + +uint +ByteExtractKeyword::applyAlignment(uint value) const +{ + int reminder = value % align; + if (reminder != 0) { + value += (align - reminder); + } + return value; +} + +MatchStatus +ByteExtractKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + auto part = Singleton::Consume::by()->get(static_cast(ctx)); + + if (!part.ok()) return MatchStatus::NoMatchFinal; + + uint bytes_to_extr = bytes.evalAttr(prev); + if (bytes_to_extr == 0) { + dbgDebug(D_KEYWORD) << "Number of bytes is zero in the 'byte_extract' keyword"; + return MatchStatus::NoMatch; //the case of constant number of bytes was checked during compilation + } + + uint start_offset, length_to_end; + tie(start_offset, length_to_end) = getStartOffsetAndLength((*part).size(), prev); + + uint offset_after_extracted_bytes = applyAlignment(start_offset + bytes_to_extr); + + if (length_to_end == 0 || offset_after_extracted_bytes > (*part).size()) { + dbgDebug(D_KEYWORD) + << "Offset after the number of bytes to extract exceeds the buffer size in the 'byte_extract' keyword"; + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; + } + + auto res = readValue(start_offset, bytes_to_extr, *part); + if (!res.ok()) { + dbgDebug(D_KEYWORD) << "Trying to store an invalid value in the 'byte_extract' keyword: " + res.getErr(); + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; + } + + uint extracted_val = res.unpack(); + + if (extracted_val > INT_MAX) { + dbgDebug(D_KEYWORD) << "Value exceeds the maximum in the 'byte_extract' keyword"; + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; + } + + //add variable and move offset after number of extracted bytes + VariableRuntimeState new_var(prev, var_id, extracted_val); + OffsetRuntimeState new_offset(&new_var, ctx, offset_after_extracted_bytes); + return runNext(&new_offset); +} + +unique_ptr +genByteExtractKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/compare_keyword.cc b/components/utils/keywords/compare_keyword.cc new file mode 100644 index 0000000..b502374 --- /dev/null +++ b/components/utils/keywords/compare_keyword.cc @@ -0,0 +1,70 @@ +#include "single_keyword.h" +#include "output.h" +#include "debug.h" + +#include +#include +#include "limits.h" + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class CompareKeyword : public SingleKeyword +{ +public: + explicit CompareKeyword(const vector &attr, const VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override; + +private: + bool + isConstant() const + { + return first_val.isConstant() && second_val.isConstant(); + } + + NumericAttr first_val; + NumericAttr second_val; + ComparisonAttr comparison; +}; + +CompareKeyword::CompareKeyword(const vector &attrs, const VariablesMapping &vars) +{ + if (attrs.size() != 3) throw KeywordError("Invalid number of attributes in the 'compare' keyword"); + + auto &first_val_param = attrs[0].getParams(); + if (first_val_param.size() != 1) { + throw KeywordError("More than one element in the first value in the 'compare' keyword"); + } + first_val.setAttr("first_val", first_val_param[0], vars, "compare"); + + auto &comparison_param = attrs[1].getParams(); + if (comparison_param.size() != 1) { + throw KeywordError("More than one element in the comparison operator in the 'compare' keyword"); + } + comparison.setAttr(comparison_param[0], "compare"); + + auto &second_val_param = attrs[2].getParams(); + if (second_val_param.size() != 1) { + throw KeywordError("More than one element in the second value in the 'compare' keyword"); + } + second_val.setAttr("second_val", second_val_param[0], vars, "compare"); +} + +MatchStatus +CompareKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + int keyword_first_val = first_val.evalAttr(prev); + int keyword_second_val = second_val.evalAttr(prev); + + if (comparison(keyword_first_val, keyword_second_val)) return runNext(prev); + + // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; +} + +unique_ptr +genCompareKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/data_keyword.cc b/components/utils/keywords/data_keyword.cc new file mode 100644 index 0000000..3681f91 --- /dev/null +++ b/components/utils/keywords/data_keyword.cc @@ -0,0 +1,409 @@ +#include "single_keyword.h" +#include "output.h" +#include "debug.h" + +#include +#include + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class DataKeyword : public SingleKeyword +{ +public: + explicit DataKeyword(const vector &attr, const VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override; + +private: + void + setOffset(const KeywordAttr &attr, const VariablesMapping &vars) + { + offset.setAttr(attr, vars, "data"); + } + + void + setDepth(const KeywordAttr &attr, const VariablesMapping &vars) + { + depth.setAttr(attr, vars, "data"); + } + + void + setCaret(const KeywordAttr &attr, const VariablesMapping &) + { + is_caret.setAttr(attr, "data"); + } + + void + setRelative(const KeywordAttr &attr, const VariablesMapping &) + { + is_relative.setAttr(attr, "data"); + } + + void + setCaseInsensitive(const KeywordAttr &attr, const VariablesMapping &) + { + is_case_insensitive.setAttr(attr, "data"); + } + + void + setContext(const KeywordAttr &attr, const VariablesMapping &) + { + ctx.setAttr(attr, "data"); + } + + void parseString(const string &str); + + void + addChar(char ch) + { + pattern.push_back(static_cast(ch)); + } + + void calcTables(); + + pair getStartAndEndOffsets(uint buf_size, const I_KeywordRuntimeState *prev) const; + uint bytesMatched(const Buffer&, uint) const; + + uint + moveOnMatch() const + { + return pattern.size(); + } + + uint + moveOnNoMatch(uint offset_from_end, unsigned char first_unmatched_byte) const + { + dbgAssert(shift.size() > offset_from_end) << "Shift table of the 'data' keyword is shorter than the offset"; + + uint skip_size; + if (skip[first_unmatched_byte]>offset_from_end) { + skip_size = skip[first_unmatched_byte]-offset_from_end; + } else { + skip_size = 1; + } + + return max(shift[offset_from_end], skip_size); + } + + bool + isConstant() const + { + return !is_relative && offset.isConstant() && depth.isConstant(); + } + + vector pattern; + uint skip[256]; + vector shift; + + NumericAttr offset; + NumericAttr depth; + BoolAttr is_negative; + BoolAttr is_caret; + BoolAttr is_relative; + BoolAttr is_case_insensitive; + CtxAttr ctx; + + static const map setops; +}; + +const map DataKeyword::setops = { + { "relative", &DataKeyword::setRelative }, + { "offset", &DataKeyword::setOffset }, + { "depth", &DataKeyword::setDepth }, + { "caret", &DataKeyword::setCaret }, + { "nocase", &DataKeyword::setCaseInsensitive }, + { "part", &DataKeyword::setContext } +}; + +DataKeyword::DataKeyword(const vector &attrs, const VariablesMapping &vars) + : + offset(), + depth() +{ + auto &pattern_param = attrs[0].getParams(); + + if (pattern_param.size() != 1) throw KeywordError("More than one element in the 'data' keyword pattern"); + const string &string_pattern = pattern_param[0]; + + if (string_pattern.length() == 0) throw KeywordError("No input for the 'data' keyword"); + + uint start = 0; + if (string_pattern[0] == '!') { + is_negative.setAttr("data", "negative"); + start++; + } + if (string_pattern[start] != '"') throw KeywordError("The data pattern does not begin with '\"'"); + + uint end = string_pattern.length()-1; + if (string_pattern[end] != '"') throw KeywordError("The data pattern does not end with '\"'"); + + if (start+1 >= end) throw KeywordError("No input for the 'data' keyword"); + + parseString(string_pattern.substr(start+1, end-(start+1))); + + for (uint i = 1; isecond; + (this->*set_func)(attrs[i], vars); + } + + calcTables(); +} + +void +DataKeyword::calcTables() +{ + if (is_case_insensitive) { + for (auto &ch : pattern) { + if (isupper(ch)) { + ch = tolower(ch); + } + } + } + + // Initialize skip table - when we meet a charecter that isn't in the pattern, we skip the whole pattern + for (auto &ch_skip : skip) { + ch_skip = pattern.size(); + } + + // Go over the charecters in the pattern. + // We can skip from a charecter to the end of the pattern. + // If a charecter appear more than once, the latest occurence take precedent. + for (uint index = 0; index0) throw KeywordError("Stoping in the middle of hex string in the 'data' keyword"); + hex_mode = false; + } + break; + } + case '\\': { + if (hex_mode) throw KeywordError("Backslash in hex string in the 'data' keyword"); + after_bslash = true; + break; + } + case '"': { + throw KeywordError("Unescaped double quotation mark in the 'data' keyword"); + break; + } + default: + if (hex_mode) { + if (!isxdigit(ch)) { + if (ch != ' ') { + throw KeywordError( + "Illegal character '" + + dumpHexChar(ch) + + "' in the hex string in the 'data' keyword" + ); + } + if (hex.size()>0) { + throw KeywordError("Space separating nibbles in the hex string in the 'data' keyword"); + } + break; + } + + hex += ch; + + if (hex.size()>=2) { + addChar(stol(hex, nullptr, 16)); + hex = ""; + } + } else { + if (!isprint(ch)) { + throw KeywordError( + "Illegal character '" + + dumpHexChar(ch) + + "' in the pattern in the 'data' keyword" + ); + } + addChar(ch); + } + } + } + + if ( hex_mode || after_bslash ) { + throw KeywordError("The 'data' keyword's pattern has ended in the middle of the parsing"); + } +} + +static uint +addOffset(uint offset, int add) +{ + if (add<0 && offset(-add)) return 0; + return offset + add; +} + +pair +DataKeyword::getStartAndEndOffsets(uint buf_size, const I_KeywordRuntimeState *prev) const +{ + uint relative_offset = is_relative?prev->getOffset(ctx):0; + int offset_attr = offset.evalAttr(prev); + uint start_offset = addOffset(relative_offset, offset_attr); + + if (depth.isSet()) { + uint depth_size = addOffset(start_offset, depth.evalAttr(prev)); + buf_size = std::min(buf_size, depth_size); + } + if (is_caret) { + buf_size = std::min(buf_size, start_offset+static_cast(pattern.size())); + } + + return make_pair(start_offset, buf_size); +} + +uint +DataKeyword::bytesMatched(const Buffer &buf, uint offset) const +{ + if (is_case_insensitive) { + for (uint i = 0; i0) << "Trying to run on an uninitialized keyword data"; + + dbgDebug(D_KEYWORD) << "Searching for " << dumpHex(pattern); + + auto part = Singleton::Consume::by()->get(static_cast(ctx)); + if (!part.ok()) { + if (is_negative) return runNext(prev); + return MatchStatus::NoMatchFinal; + } + + const auto &buf = part.unpack(); + + dbgTrace(D_KEYWORD) << "Full buffer: " << dumpHex(buf); + + uint offset, max_offset; + + tie(offset, max_offset) = getStartAndEndOffsets(buf.size(), prev); + offset += pattern.size(); + + bool match_found = false; + while (offset<=max_offset) { + // Short circuit for the common, simple case where the last byte doesn't match + if (skip[buf[offset-1]]) { + offset += skip[buf[offset - 1]]; + continue; + } + + // Full search Boyer-Moore + uint match_size = bytesMatched(buf, offset); + if (match_size == pattern.size()) { + if (is_negative) { + return isConstant()?MatchStatus::NoMatchFinal:MatchStatus::NoMatch; + } + match_found = true; + OffsetRuntimeState new_offset(prev, ctx, offset); + auto next_keyword_result = runNext(&new_offset); + if (next_keyword_result!=MatchStatus::NoMatch) return next_keyword_result; + offset += moveOnMatch(); + } else { + offset += moveOnNoMatch(match_size, buf[offset-(match_size+1)]); + } + } + + // No matchs is a success for negative keywords + if (is_negative && !match_found) return runNext(prev); + + // If there were no matchs and the keyword is an effected by other keywords, then we know that the rule won't match + if (isConstant() && !match_found) return MatchStatus::NoMatchFinal; + + return MatchStatus::NoMatch; +} + +unique_ptr +genDataKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/jump_keyword.cc b/components/utils/keywords/jump_keyword.cc new file mode 100644 index 0000000..490040c --- /dev/null +++ b/components/utils/keywords/jump_keyword.cc @@ -0,0 +1,174 @@ +#include "single_keyword.h" +#include "output.h" +#include "debug.h" + +#include +#include + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class jumpKeyword : public SingleKeyword +{ +public: + explicit jumpKeyword(const vector &attr, const VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override; + +private: + enum class JumpFromId + { + RELATIVE, + FROM_BEGINNING, + FROM_END + }; + + void + setContext(const KeywordAttr &attr) + { + ctx.setAttr(attr, "byte_extract"); + } + + void + setAlign(const KeywordAttr &attr) + { + if (align != 1) throw KeywordError("Double definition of the 'align' in the 'jump' keyword"); + auto &vec = attr.getParams(); + if (vec.size() != 2) throw KeywordError("Malformed 'align' in the 'jump' keyword"); + + if (vec[1] == "2") { + align = 2; + } else if (vec[1] == "4") { + align = 4; + } else { + throw KeywordError("Unknown 'align' in the 'jump' keyword: " + vec[1]); + } + } + + bool + isConstant() const + { + return jumping_from != JumpFromId::RELATIVE && jumping_val.isConstant(); + } + + JumpFromId jumping_from; + NumericAttr jumping_val; + int align = 1; + CtxAttr ctx; + + static const map setops; + uint getStartOffset(uint buf_size, const I_KeywordRuntimeState *prev) const; + uint applyAlignment(uint value) const; + uint addOffset(uint offset, int add) const; +}; + +const map jumpKeyword::setops = { + { "part", &jumpKeyword::setContext }, + { "align", &jumpKeyword::setAlign } +}; + +jumpKeyword::jumpKeyword(const vector &attrs, const VariablesMapping &vars) +{ + + //two requied attributes - jumping value and jumping from + if (attrs.size() < 2) throw KeywordError("Invalid number of attributes in the 'jump' keyword"); + + //parisng first attribute (Required) - jumping value + auto &jumping_val_param = attrs[0].getParams(); + if (jumping_val_param.size() != 1) { + throw KeywordError("More than one element in the jumping value in the 'jump' keyword"); + } + jumping_val.setAttr("jumping value", jumping_val_param[0], vars, "jump"); + + //parisng second attribute (Required) - jumping from + auto &jumping_from_param = attrs[1].getParams(); + if (jumping_from_param.size() != 1) { + throw KeywordError("More than one element in the jumping 'from' parameter in the 'jump' keyword"); + } + + if (jumping_from_param[0] == "from_beginning") { + jumping_from = JumpFromId::FROM_BEGINNING; + } else if (jumping_from_param[0] == "from_end") { + jumping_from = JumpFromId::FROM_END; + } else if (jumping_from_param[0] == "relative") { + jumping_from = JumpFromId::RELATIVE; + } else { + throw KeywordError("Unknown jumping 'from' parameter in the 'jump' keyword: " + jumping_from_param[0]); + } + + //parisng optional attributes + for (uint i = 2; i < attrs.size(); i++) { + auto curr = setops.find(attrs[i].getAttrName()); + if (curr == setops.end()) { + throw KeywordError("Unknown attribute " + attrs[i].getAttrName() + " in the 'jump' keyword"); + } + auto set_func = curr->second; + (this->*set_func)(attrs[i]); + } +} + +uint +jumpKeyword::applyAlignment(uint value) const +{ + int reminder = value % align; + if (reminder != 0) { + value += (align - reminder); + } + return value; +} + +uint +jumpKeyword::addOffset(uint offset, int add) const +{ + if (add < 0 && offset < static_cast(-add)) { + dbgWarning(D_KEYWORD) + << "The offset was set to 0 " + << "due to an attempt to jump before the beginning of the buffer in the 'jump' keyword"; + return 0; + } + return applyAlignment(offset + add); +} + +uint +jumpKeyword::getStartOffset(uint buf_size, const I_KeywordRuntimeState *prev) const +{ + switch (jumping_from) { + case JumpFromId::FROM_BEGINNING: { + return 0; + } + case JumpFromId::FROM_END: { + return buf_size; + } + case JumpFromId::RELATIVE: { + return prev->getOffset(ctx); + } + } + dbgAssert(false) << "Invalid jumping 'from' parameter"; + return 0; +} + +MatchStatus +jumpKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + auto part = Singleton::Consume::by()->get(static_cast(ctx)); + + if (!part.ok()) return MatchStatus::NoMatchFinal; + + uint start_offset = getStartOffset((*part).size(), prev); + + uint offset_to_jump = addOffset(start_offset, jumping_val.evalAttr(prev)); + + if (offset_to_jump > (*part).size()) { + dbgDebug(D_KEYWORD) << "New offset exceeds the buffer size in the 'jump' keyword"; + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; + } + + OffsetRuntimeState new_offset(prev, ctx, offset_to_jump); + return runNext(&new_offset); +} + +unique_ptr +genJumpKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/keywords_rule.cc b/components/utils/keywords/keywords_rule.cc new file mode 100644 index 0000000..c75a891 --- /dev/null +++ b/components/utils/keywords/keywords_rule.cc @@ -0,0 +1,171 @@ +#include "keyword_comp.h" + +#include +#include "sentinel_runtime_state.h" + +using namespace std; + +static const string whitespaces = " \t"; + +static string +getSubStrNoPadding(const string &str, uint start, uint end) +{ + auto r_start = str.find_first_not_of(whitespaces, start); + auto r_end = str.find_last_not_of(whitespaces, end-1); + + if (r_end==string::npos || r_start==string::npos || r_start>r_end) { + throw KeywordError("Found an empty section in the '"+ str + "'"); + } + + return str.substr(r_start, r_end-r_start+1); +} + +static vector +split(const string &str, const string &delim, uint start = 0) +{ + vector res; + uint part_start = start; + bool escape = false; + bool in_string = false; + + for (uint index = start; index::From +{ +public: + Maybe> + genRule(const string &rule) + { + shared_ptr res; + try { + res = KeywordsRuleImpl::genRule(rule); + } catch (const KeywordError &e) { + return genError(e.getErr());; + } + return move(res); + } + +private: + class KeywordsRuleImpl : public VirtualRule + { + public: + bool isMatch() const override { return start.isMatch() == MatchStatus::Match; } + + static unique_ptr + genRule(const string &rule) + { + auto res = make_unique(); + + auto pos = rule.find_last_not_of(whitespaces); + if (pos==string::npos) { + // Empty rule + return res; + } + + if (rule[pos]!=';') throw KeywordError(rule + " - end of text pass rule"); + + VariablesMapping known_vars; + + auto key_vec = split(rule, ";"); + for (auto &keyword : key_vec) { + res->start.appendKeyword(getKeywordByName(keyword, known_vars)); + } + + return res; + } + + private: + SentinelKeyword start; + }; +}; + +KeywordComp::KeywordComp() : Component("KeywordComp"), pimpl(make_unique()) {} + +KeywordComp::~KeywordComp() {} + +string I_KeywordsRule::keywords_tag = "keywords_rule_tag"; diff --git a/components/utils/keywords/keywords_ut/CMakeLists.txt b/components/utils/keywords/keywords_ut/CMakeLists.txt new file mode 100644 index 0000000..260a6f8 --- /dev/null +++ b/components/utils/keywords/keywords_ut/CMakeLists.txt @@ -0,0 +1,5 @@ +add_unit_test( + keywords_ut + "keywords_ut.cc;single_keyword_ut.cc" + "keywords;pcre2-8;buffers;singleton;table;event_is;metric;-lboost_regex" +) diff --git a/components/utils/keywords/keywords_ut/keywords_ut.cc b/components/utils/keywords/keywords_ut/keywords_ut.cc new file mode 100644 index 0000000..70894b7 --- /dev/null +++ b/components/utils/keywords/keywords_ut/keywords_ut.cc @@ -0,0 +1,996 @@ +#include "keyword_comp.h" +#include "environment.h" +#include "mock/mock_table.h" +#include "cptest.h" +#include "mock/mock_time_get.h" +#include "mock/mock_mainloop.h" +#include "config.h" +#include "config_component.h" + +using namespace std; + +class KeywordsRuleTest : public ::testing::Test +{ +public: + void + appendBuffer(const string &id, const string &str) + { + buffers[id] += Buffer(str); + } + + string + ruleCompileFail(const string &_rule) + { + auto rule = Singleton::Consume::from(comp)->genRule(_rule); + EXPECT_FALSE(rule.ok()) << "Compile supposed to fail"; + return rule.getErr(); + } + + bool + ruleRun(const string &_rule, const string &default_ctx = "default") + { + auto rule = Singleton::Consume::from(comp)->genRule(_rule); + EXPECT_TRUE(rule.ok()) << "Compile not supposed to fail: " << rule.getErr(); + ScopedContext ctx; + ctx.registerValue(I_KeywordsRule::getKeywordsRuleTag(), default_ctx); + for (auto &value : buffers) { + ctx.registerValue(value.first, value.second); + } + return (*rule)->isMatch(); + } + +private: + KeywordComp comp; + ::testing::NiceMock mock_mainloop; + ::testing::NiceMock mock_timer; + Environment env; + map buffers; +}; + +TEST_F(KeywordsRuleTest, data_basic_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + + EXPECT_TRUE(ruleRun("data: \"234\" , part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("data: \"234\";", "HTTP_RESPONSE_BODY")); + EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, data_relative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("data: \"567\", part HTTP_RESPONSE_BODY; data: \"234\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE( + ruleRun("data: \"567\", part HTTP_RESPONSE_BODY; data: \"234\", part HTTP_RESPONSE_BODY, relative;") + ); + EXPECT_TRUE(ruleRun("data: \"234\", part HTTP_RESPONSE_BODY; data: \"567\", part HTTP_RESPONSE_BODY, relative;")); +} + +TEST_F(KeywordsRuleTest, data_depth_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("data: \"345\", depth 5, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"345\", depth 4, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, data_nocase_test) { + appendBuffer("HTTP_RESPONSE_BODY", "abcdefg"); + + EXPECT_TRUE(ruleRun("data: \"cde\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"CDE\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("data: \"CDE\", nocase, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, data_offset_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("data: \"345\", offset 2, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"345\", offset 3, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, data_caret_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY, caret;")); + EXPECT_TRUE(ruleRun("data: \"345\", caret, part HTTP_RESPONSE_BODY, offset 2;")); +} + +TEST_F(KeywordsRuleTest, data_negative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_FALSE(ruleRun("data: !\"345\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("data: !\"365\", part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, data_part_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + appendBuffer("HTTP_REQUEST_BODY", "abcdefg"); + + EXPECT_TRUE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY; data: \"cde\", part HTTP_REQUEST_BODY;")); + EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_RESPONSE_BODY; data: \"cde\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"345\", part HTTP_REQUEST_BODY; data: \"cde\", part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, pcre_basic_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/5..7/\", part HTTP_RESPONSE_BODY;")); +} + + +TEST_F(KeywordsRuleTest, pcre_relative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/R\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE( + ruleRun("pcre: \"/5.7/\", part HTTP_RESPONSE_BODY; pcre: \"/2.4/\", relative, part HTTP_RESPONSE_BODY;") + ); + EXPECT_TRUE(ruleRun("pcre: \"/2.4/\", part HTTP_RESPONSE_BODY; pcre: \"/5.7/R\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE( + ruleRun("pcre: \"/2.4/\", part HTTP_RESPONSE_BODY; pcre: \"/5.7/\", relative, part HTTP_RESPONSE_BODY;") + ); +} + +TEST_F(KeywordsRuleTest, pcre_depth_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", depth 5, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", depth 4, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, pcre_nocase_test) { + appendBuffer("HTTP_RESPONSE_BODY", "abcdefg"); + + EXPECT_TRUE(ruleRun("pcre: \"/c.e/\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/C.E/\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("pcre: \"/C.E/i\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("pcre: \"/C.E/\", nocase, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, pcre_offset_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", offset 2, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", offset 300, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, pcre_part_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + appendBuffer("HTTP_REQUEST_BODY", "abcdefg"); + + EXPECT_TRUE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY; pcre: \"/c.e/\", part HTTP_REQUEST_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", part HTTP_RESPONSE_BODY; pcre: \"/c.e/\", part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("pcre: \"/3.5/\", part HTTP_REQUEST_BODY; pcre: \"/c.e/\", part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, pcre_negative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_FALSE(ruleRun("pcre: !\"/3.5/\", part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("pcre: !\"/3..5/\", part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, compare_comparison_test) { + EXPECT_TRUE(ruleRun("compare: 0, =, 0;")); + EXPECT_TRUE(ruleRun("compare: -1, =, -1;")); + EXPECT_FALSE(ruleRun("compare: 0, =, 1;")); + EXPECT_FALSE(ruleRun("compare: -1, =, -2;")); + EXPECT_FALSE(ruleRun("compare: 1, =, -1;")); + EXPECT_FALSE(ruleRun("compare: -1, =, 1;")); + EXPECT_TRUE(ruleRun("compare: 2, !=, 3;")); + EXPECT_TRUE(ruleRun("compare: 2, <=, 3;")); + EXPECT_TRUE(ruleRun("compare: 2, <, 3;")); + EXPECT_FALSE(ruleRun("compare: 2, >, 3;")); + EXPECT_FALSE(ruleRun("compare: 2, >=, 3;")); + EXPECT_TRUE(ruleRun("compare: -2, !=, -3;")); + EXPECT_TRUE(ruleRun("compare: -2, >=, -3;")); + EXPECT_TRUE(ruleRun("compare: -2, >, -3;")); + EXPECT_FALSE(ruleRun("compare: -2, <, -3;")); + EXPECT_FALSE(ruleRun("compare: -2, <=, -3;")); + EXPECT_TRUE(ruleRun("compare: -2, !=, 3;")); + EXPECT_TRUE(ruleRun("compare: -2, <=, 3;")); + EXPECT_TRUE(ruleRun("compare: -2, <, 3;")); + EXPECT_FALSE(ruleRun("compare: -2, >, 3;")); + EXPECT_FALSE(ruleRun("compare: -2, >=, 3;")); + EXPECT_TRUE(ruleRun("compare: 2, !=, -3;")); + EXPECT_TRUE(ruleRun("compare: 2, >=, -3;")); + EXPECT_TRUE(ruleRun("compare: 2, >, -3;")); + EXPECT_FALSE(ruleRun("compare: 2, <, -3;")); + EXPECT_FALSE(ruleRun("compare: 2, <=, -3;")); +} + +TEST_F(KeywordsRuleTest, compare_compile_fail_test) { + EXPECT_EQ(ruleCompileFail("compare: 0;"), "Invalid number of attributes in the 'compare' keyword"); + EXPECT_EQ(ruleCompileFail("compare: 0, =;"), "Invalid number of attributes in the 'compare' keyword"); + EXPECT_EQ(ruleCompileFail("compare: 0, =, 0, 0;"), "Invalid number of attributes in the 'compare' keyword"); + EXPECT_EQ( + ruleCompileFail("compare: 0 1, =, 0;"), + "More than one element in the first value in the 'compare' keyword" + ); + EXPECT_EQ( + ruleCompileFail("compare: 0, = =, 0;"), + "More than one element in the comparison operator in the 'compare' keyword" + ); + EXPECT_EQ( + ruleCompileFail("compare: 0, =, 0 1;"), + "More than one element in the second value in the 'compare' keyword" + ); + EXPECT_EQ( + ruleCompileFail("compare: 0, ==, 0;"), + "Unknown comparison operator in the 'compare' keyword: Could not find the operator: ==" + ); +} + +TEST_F(KeywordsRuleTest, length_basic_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + appendBuffer("HTTP_REQUEST_BODY", ""); + + EXPECT_TRUE( + ruleRun( + "length: length_var, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 9;" + ) + ); + EXPECT_TRUE( + ruleRun( + "length: length_var, part HTTP_REQUEST_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_FALSE( + ruleRun( + "length: length_var, part HTTP_REQUEST_BODY;" + "compare: length_var, =, 1;" + ) + ); +} + +TEST_F(KeywordsRuleTest, length_part_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("length: length_var, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("length: length_var, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, length_relative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + + EXPECT_TRUE( + ruleRun( + "data: \"234\", part HTTP_RESPONSE_BODY;" + "length: relative_length_var, part HTTP_RESPONSE_BODY, relative;" + "compare: relative_length_var, =, 5;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"234\", part HTTP_RESPONSE_BODY;" + "length: relative_length_var, part HTTP_RESPONSE_BODY;" + "compare: relative_length_var, =, 5;" + ) + ); + EXPECT_TRUE( + ruleRun( + "data: \"89\", part HTTP_RESPONSE_BODY;" + "length: zero_length_var, part HTTP_RESPONSE_BODY, relative;" + "compare: zero_length_var, =, 0;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"89\", part HTTP_RESPONSE_BODY;" + "length: zero_length_var, part HTTP_RESPONSE_BODY;" + "compare: zero_length_var, =, 0;" + ) + ); +} + +TEST_F(KeywordsRuleTest, length_compare_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123"); + EXPECT_FALSE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;")); + + appendBuffer("HTTP_RESPONSE_BODY", "456"); + EXPECT_TRUE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;")); + + appendBuffer("HTTP_RESPONSE_BODY", "789"); + EXPECT_TRUE(ruleRun("length: 6, min, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("length: 6, exact, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("length: 6, max, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, length_compile_fail_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + + EXPECT_EQ( + ruleCompileFail("length: two_elem 2, part HTTP_RESPONSE_BODY;"), + "More than one element in the variable name in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length: relative, part HTTP_RESPONSE_BODY;"), + "The 'relative' cannot be the variable name in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length: part, part HTTP_RESPONSE_BODY;"), + "The 'part' cannot be the variable name in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length: -minus, part HTTP_RESPONSE_BODY;"), + "Malformed variable name in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length: 1digit, part HTTP_RESPONSE_BODY;"), + "Malformed variable name in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length: bad_attr, partt HTTP_RESPONSE_BODY;"), + "Unknown attribute 'partt' in the 'length' keyword" + ); + EXPECT_EQ( + ruleCompileFail("length:;"), + "Invalid number of attributes in the 'length' keyword" + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_dec_string_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234"); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, dec_var, string dec, part HTTP_RESPONSE_BODY;" + "data: \"234\", offset dec_var, part HTTP_RESPONSE_BODY;" + ) + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 1, dec_var, string dec, part HTTP_RESPONSE_BODY;" + "data: \"123\", offset dec_var, part HTTP_RESPONSE_BODY;" + ) + ); + + appendBuffer("HTTP_REQUEST_BODY", "A"); + + EXPECT_FALSE(ruleRun("byte_extract: 1, bad_dec_var, string dec, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, byte_extract_hex_string_test) { + appendBuffer("HTTP_RESPONSE_BODY", "A123"); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, hex_var, string hex, part HTTP_RESPONSE_BODY;" + "compare: hex_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 2, hex_var, string hex, part HTTP_RESPONSE_BODY;" + "compare: hex_var, =, 161;" + ) + ); + + appendBuffer("HTTP_REQUEST_BODY", "10G"); + + EXPECT_FALSE( + ruleRun( + "byte_extract: 2, hex_var, string hex, part HTTP_REQUEST_BODY;" + "compare: hex_var, =, 10;" + ) + ); + EXPECT_FALSE(ruleRun("byte_extract: 3, bad_hex_var, string oct, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, byte_extract_oct_string_test) { + appendBuffer("HTTP_RESPONSE_BODY", "13ABC"); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 2, oct_var, string oct, part HTTP_RESPONSE_BODY;" + "compare: oct_var, =, 11;" + ) + ); + + appendBuffer("HTTP_REQUEST_BODY", "118"); + + EXPECT_FALSE( + ruleRun( + "byte_extract: 2, oct_var, string oct, part HTTP_REQUEST_BODY;" + "compare: oct_var, =, 13;" + ) + ); + EXPECT_FALSE(ruleRun("byte_extract: 3, bad_oct_var, string oct, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, byte_extract_binary_data_test) { + string one_byte_binary_data = {10}; + appendBuffer("HTTP_RESPONSE_BODY", one_byte_binary_data); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, binary_data_var, part HTTP_RESPONSE_BODY;" + "compare: binary_data_var, =, 10;" + ) + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 1, dec_data_var, offset 2, string dec, part HTTP_RESPONSE_BODY;" + "compare: dec_data_var, =, 10;" + ) + ); + + string two_bytes_binary_data = {1, 0, 0}; + appendBuffer("HTTP_REQUEST_BODY", two_bytes_binary_data); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 2, binary_data_var, part HTTP_REQUEST_BODY;" + "compare: binary_data_var , =, 256;" + ) + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 3, not1/2/4, part HTTP_REQUEST_BODY;"), + "Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail( + "byte_extract: 1, no_constant, part HTTP_REQUEST_BODY;" + "byte_extract: no_constant, var, part HTTP_REQUEST_BODY;" + ), + "Data type is binary, but the 'bytes' is not constant in the 'byte_extract' keyword" + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_bad_num_of_bytes_test) { + appendBuffer("HTTP_RESPONSE_BODY", "0"); + + EXPECT_EQ( + ruleCompileFail("byte_extract: 0, zero_bytes_var, string dec, part HTTP_RESPONSE_BODY;"), + "Number of bytes is zero in the 'byte_extract' keyword" + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 1, one_byte_var, string dec, part HTTP_RESPONSE_BODY;" + "byte_extract: one_byte_var, zero_bytes_var, string dec, part HTTP_RESPONSE_BODY;" + ) + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_part_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123"); + EXPECT_TRUE(ruleRun("byte_extract: 1, part_var, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("byte_extract: 1, part_var, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, byte_extract_offset_test) { + appendBuffer("HTTP_REQUEST_BODY", "1A23456789hello"); + + EXPECT_FALSE( + ruleRun( + "byte_extract: 1, hex_var, offset 1, string hex, part HTTP_REQUEST_BODY; " + "data: \"9hell\", offset hex_var, part HTTP_REQUEST_BODY;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, hex_var, offset 1, string hex, part HTTP_REQUEST_BODY;" + "data: \"hell\", offset hex_var, part HTTP_REQUEST_BODY;" + ) + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 1, dec_var, offset -1, string dec, part HTTP_REQUEST_BODY;" + "data: \"1A2\", offset dec_var, part HTTP_REQUEST_BODY;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, dec_var, offset -1, string dec, part HTTP_REQUEST_BODY;" + "data: \"A2\", offset dec_var, part HTTP_REQUEST_BODY;" + ) + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_relative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + + EXPECT_TRUE( + ruleRun( + "data: \"12\", part HTTP_RESPONSE_BODY;" + "byte_extract: 1, relative_var, relative, string dec, part HTTP_RESPONSE_BODY;" + "compare: relative_var, =, 3;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"12\", part HTTP_RESPONSE_BODY;" + "byte_extract: 1, non_relative_var, string dec, part HTTP_RESPONSE_BODY;" + "compare: non_relative_var, =, 3;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"89\", part HTTP_RESPONSE_BODY;" + "byte_extract: 1, relative_var, string dec, relative, part HTTP_RESPONSE_BODY;" + ) + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_endianness_test) { + string little_end_test_str = {8, 0, 0}; + appendBuffer("HTTP_RESPONSE_BODY", little_end_test_str); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 2, lit_end_var, little_endian, part HTTP_RESPONSE_BODY;" + "compare: lit_end_var, =, 8;" + ) + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 2, big_end_var, part HTTP_RESPONSE_BODY;" + "compare: big_end_var, =, 8;" + ) + ); + + little_end_test_str[1] = 0; + little_end_test_str[2] = 1; + appendBuffer("HTTP_REQUEST_BODY", little_end_test_str); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 2, lit_end_with_offset_var," + "offset 1, little_endian, part HTTP_REQUEST_BODY;" + "compare: lit_end_with_offset_var, =, 256;" + ) + ); + EXPECT_FALSE( + ruleRun( + "byte_extract: 2, big_end_with_offset_var, offset 1, part HTTP_REQUEST_BODY;" + "compare: big_end_with_offset_var, =, 256;" + ) + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, var, little_endian, part HTTP_REQUEST_BODY;"), + "Little endian is set, but the number of bytes is invalid in the 'byte_extract' keyword" + ); + + EXPECT_EQ( + ruleCompileFail("byte_extract: 2, no_binary, little_endian, string dec, part HTTP_REQUEST_BODY;"), + "Little endian is set, but the data type is not binary in the 'byte_extract' keyword" + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_align_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234"); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, align2_var, align 2, string dec, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 2;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, align4_var, align 4, string dec, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, align2_var, offset 3, align 2, string dec, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, align4_var, offset 3, align 4, string dec, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + + appendBuffer("HTTP_REQUEST_BODY", "123"); + + EXPECT_TRUE( + ruleRun( + "byte_extract: 1, align2_var, offset 1, align 2, string dec, part HTTP_REQUEST_BODY;" + "length: length_var, relative, part HTTP_REQUEST_BODY;" + "compare: length_var, =, 1;" + ) + ); + EXPECT_FALSE(ruleRun("byte_extract: 1, align4_var, align 4, string dec, part HTTP_REQUEST_BODY;")); + EXPECT_FALSE(ruleRun("byte_extract: 1, align2_var, offset 2, align 2, string dec, part HTTP_REQUEST_BODY;")); + + string binary_data_str = { 1 }; + appendBuffer("HTTP_REQUEST_BODY", binary_data_str); + + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, align_binary_var, align 2, part HTTP_REQUEST_BODY;"), + "The 'align' is set and data type is binary in the 'byte_extract' keyword" + ); +} + +TEST_F(KeywordsRuleTest, byte_extract_overflow_test) { + string overflow_dec_data_str = to_string((uint)INT_MAX + 1); + appendBuffer("HTTP_RESPONSE_BODY", overflow_dec_data_str); + + EXPECT_FALSE( + ruleRun( + "byte_extract: " + to_string(overflow_dec_data_str.length()) + "," + "overflow_var, string dec, part HTTP_RESPONSE_BODY;" + ) + ); + + string max_value_dec_data_str = to_string(INT_MAX); + appendBuffer("HTTP_REQUEST_BODY", max_value_dec_data_str); + + EXPECT_TRUE( + ruleRun( + "byte_extract: " + to_string(max_value_dec_data_str.length()) + "," + "max_var, string dec, part HTTP_REQUEST_BODY;" + "compare: max_var, =, " + max_value_dec_data_str + ";" + ) + ); + + string overflow_binary_data_str = { 0x7f, 0x7f, 0x7f, 0x7f, 0 }; + appendBuffer("HTTP_REQUEST_HEADERS", overflow_binary_data_str); + + EXPECT_FALSE(ruleRun("byte_extract: 5 ,overflow_num_var, string dec, part HTTP_REQUEST_HEADERS;")); +} + +TEST_F(KeywordsRuleTest, byte_extract_compile_fail_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1 2, dec_var, string dec, part HTTP_RESPONSE_BODY;"), + "More than one element in the 'bytes' in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, dec_var 1, string dec, part HTTP_RESPONSE_BODY;"), + "More than one element in the variable name in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, align, string dec, part HTTP_RESPONSE_BODY;"), + "'align' cannot be the variable name in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, -1, string dec, part HTTP_RESPONSE_BODY;"), + "Malformed variable name in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_data_type, string dechex, part HTTP_RESPONSE_BODY;"), + "Unknown data type in the 'byte_extract' keyword: dechex" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, 1var, string dec, part HTTP_RESPONSE_BODY;"), + "Malformed variable name in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_align, align 3, part HTTP_RESPONSE_BODY;"), + "Unknown 'align' in the 'byte_extract' keyword: 3" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_constant, offset 0x;"), + "Malformed constant '0x' in the 'offset' in the 'byte_extract' keyword" + ); + EXPECT_EQ(ruleCompileFail("byte_extract: 1;"), "Invalid number of attributes in the 'byte_extract' keyword"); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_attr, offset;"), + "Malformed offset' in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_attr, string hex dec;"), + "Malformed data type in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_attr, ofset 5;"), + "Unknown attribute 'ofset' in the 'byte_extract' keyword" + ); + EXPECT_EQ( + ruleCompileFail("byte_extract: 1, bad_align, align 2 4;"), + "Malformed 'align' in the 'byte_extract' keyword" + ); +} + +TEST_F(KeywordsRuleTest, jump_from_beginning_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 9;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 9;" + ) + ); + EXPECT_FALSE( + ruleRun( + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: -1, from_beginning, part HTTP_RESPONSE_BODY;" + "length: length_var, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 10, from_beginning, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_FALSE(ruleRun("jump: 11, from_beginning, part HTTP_RESPONSE_BODY;")); +} + +TEST_F(KeywordsRuleTest, jump_relative_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE( + ruleRun( + "data: \"1\", part HTTP_RESPONSE_BODY;" + "jump: 1, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 8;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"1\", part HTTP_RESPONSE_BODY;" + "jump: 1, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 9;" + ) + ); + EXPECT_TRUE( + ruleRun( + "data: \"1\", part HTTP_RESPONSE_BODY;" + "jump: -2, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "data: \"1\", part HTTP_RESPONSE_BODY;" + "jump: 9, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_FALSE( + ruleRun( + "data: \"1\", part HTTP_RESPONSE_BODY;" + "jump: 10, relative, part HTTP_RESPONSE_BODY;" + ) + ); +} + +TEST_F(KeywordsRuleTest, jump_from_end_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_FALSE(ruleRun("jump: 1, from_end, part HTTP_RESPONSE_BODY;")); + EXPECT_TRUE( + ruleRun( + "jump: -1, from_end, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 1;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: -10, from_end, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: -11, from_end, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); +} + +TEST_F(KeywordsRuleTest, combined_jumps_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: -1, from_end, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 1;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: -1, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: -1, from_end, part HTTP_RESPONSE_BODY;" + "jump: 1, relative, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 0;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: -1, from_end, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 1;" + ) + ); +} + + +TEST_F(KeywordsRuleTest, jump_alignment_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, align 2, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 8;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 1, from_beginning, align 4, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 6;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: 2, relative, align 2, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 4;" + ) + ); + EXPECT_FALSE( + ruleRun( + "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: 2, relative, align 2, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 7;" + ) + ); + EXPECT_FALSE( + ruleRun( + "jump: 3, from_beginning, part HTTP_RESPONSE_BODY;" + "jump: 2, relative, align 4, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 3;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 2, from_beginning, align 2, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 8;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 4, from_beginning, align 4, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 6;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 0, from_beginning, align 2, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); + EXPECT_TRUE( + ruleRun( + "jump: 0, from_beginning, align 4, part HTTP_RESPONSE_BODY;" + "length: length_var, relative, part HTTP_RESPONSE_BODY;" + "compare: length_var, =, 10;" + ) + ); +} + +TEST_F(KeywordsRuleTest, jump_part_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_TRUE(ruleRun("jump: 1, from_beginning, part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("jump: 1, from_beginning, part HTTP_REQUEST_BODY;")); +} + +TEST_F(KeywordsRuleTest, jump_compile_fail_test) { + appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); + + EXPECT_EQ(ruleCompileFail("jump: 1;"), "Invalid number of attributes in the 'jump' keyword"); + EXPECT_EQ( + ruleCompileFail("jump: 2 1, from_beginning;"), + "More than one element in the jumping value in the 'jump' keyword" + ); + EXPECT_EQ( + ruleCompileFail("jump: 2, from_relative;"), + "Unknown jumping 'from' parameter in the 'jump' keyword: from_relative" + ); + EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 3;"), "Unknown 'align' in the 'jump' keyword: 3"); + EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 1;"), "Unknown 'align' in the 'jump' keyword: 1"); + EXPECT_EQ(ruleCompileFail("jump: 2, relative, align2 2;"), "Unknown attribute align2 in the 'jump' keyword"); + EXPECT_EQ(ruleCompileFail("jump: 2, relative, align 2 4;"), "Malformed 'align' in the 'jump' keyword"); + EXPECT_EQ( + ruleCompileFail("jump: 2, from_beginning relative;"), + "More than one element in the jumping 'from' parameter in the 'jump' keyword" + ); +} + +TEST_F(KeywordsRuleTest, stateop) +{ + using testing::_; + + ConfigComponent conf; + testing::StrictMock table; + + std::unique_ptr opq; + TableOpaqueBase *opq_ptr; + bool has_stage = false; + EXPECT_CALL(table, createStateRValueRemoved(_, _)) + .WillOnce(testing::DoAll( + testing::Invoke( + [&] (const type_index &, std::unique_ptr &other) + { + opq = std::move(other); + opq_ptr = opq.get(); + has_stage = true; + } + ), + testing::Return(true) + )); + EXPECT_CALL(table, getState(_)).WillRepeatedly(testing::ReturnPointee(&opq_ptr)); + EXPECT_CALL(table, hasState(_)).WillRepeatedly(testing::ReturnPointee(&has_stage)); + + EXPECT_FALSE(ruleRun("stateop: state sss, isset;")); + + EXPECT_TRUE(ruleRun("stateop: state sss, unset;")); + EXPECT_FALSE(ruleRun("stateop: state sss, isset;")); + + EXPECT_TRUE(ruleRun("stateop: state sss, set;")); + EXPECT_TRUE(ruleRun("stateop: state sss, isset;")); + EXPECT_FALSE(ruleRun("stateop: state dd, isset;")); + + EXPECT_TRUE(ruleRun("stateop: state sss, unset;")); + EXPECT_FALSE(ruleRun("stateop: state sss, isset;")); +} + +TEST_F(KeywordsRuleTest, no_match) +{ + EXPECT_FALSE(ruleRun("no_match;")); +} diff --git a/components/utils/keywords/keywords_ut/single_keyword_ut.cc b/components/utils/keywords/keywords_ut/single_keyword_ut.cc new file mode 100644 index 0000000..db3d03d --- /dev/null +++ b/components/utils/keywords/keywords_ut/single_keyword_ut.cc @@ -0,0 +1,171 @@ +#include "../sentinel_runtime_state.h" +#include "../single_keyword.h" +#include "cptest.h" +#include + +using namespace std; +using namespace testing; + +#define FIRST_VARIABLE_ID 1 +#define FIRST_VARIABLE_VAL 2u +#define SECOND_VARIABLE_ID 3 +#define SECOND_VARIABLE_VAL 4u +#define THIRD_VARIABLE_ID 5 +#define THIRD_VARIABLE_VAL 6u + +#define FIRST_OFFSET 4u +#define SECOND_OFFSET 5u +#define THIRD_OFFSET 6u + +const static unsigned int zero = 0; + +class I_KeywordRuntimeStateTest : public Test +{ +public: + // Constructs SentinelKeyword as head because it is the only I_KeywordRuntimeState implementation + // that does not hold I_KeywordRuntimeState *prev + I_KeywordRuntimeStateTest() : list_head(&sentinel) {} + + uint + getOffset(const string &id) + { + return list_head->getOffset(id); + } + + uint + getVariable(uint requested_var_id) + { + return list_head->getVariable(requested_var_id); + } + + void + addOffsetState(const string &_ctx, uint _offset) + { + offset_list.push_front(make_unique(list_head, _ctx, _offset)); + list_head = offset_list.front().get(); + } + + void + addVariableState(uint _var_id, uint _val) + { + variable_list.push_front(make_unique(list_head, _var_id, _val)); + list_head = variable_list.front().get(); + } + +private: + list> variable_list; + list> offset_list; + SentinelRuntimeState sentinel; + I_KeywordRuntimeState *list_head; +}; + +TEST_F(I_KeywordRuntimeStateTest, one_element_list_positive_test) { + EXPECT_EQ(getOffset("HTTP_METHOD"), zero); + EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero); + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero); +} + +TEST_F(I_KeywordRuntimeStateTest, one_variable_state_list_positive_test) { + addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL); + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero); + EXPECT_EQ(getOffset("HTTP_METHOD"), zero); + EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL); +} + +TEST_F(I_KeywordRuntimeStateTest, one_offset_state_list_positive_test) { + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero); + addOffsetState("HTTP_REQUEST_HEADERS", FIRST_OFFSET); + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), FIRST_OFFSET); +} + +TEST_F(I_KeywordRuntimeStateTest, one_element_list_negative_test) { + cptestPrepareToDie(); + EXPECT_DEATH(getVariable(FIRST_OFFSET), ""); + EXPECT_DEATH(getVariable(SECOND_OFFSET), ""); + EXPECT_DEATH(getVariable(THIRD_OFFSET), ""); +} + +TEST_F(I_KeywordRuntimeStateTest, variable_runtime_state_list_positive_test) { + // Notice that variables ids and values are different + addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL); + addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL); + addVariableState(THIRD_VARIABLE_ID, THIRD_VARIABLE_VAL); + + EXPECT_EQ(getOffset("HTTP_METHOD"), zero); + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero); + EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero); + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero); + EXPECT_EQ(getOffset("HTTP_REQUEST_BODY"), zero); + + EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL); + EXPECT_EQ(getVariable(SECOND_VARIABLE_ID), SECOND_VARIABLE_VAL); + EXPECT_EQ(getVariable(THIRD_VARIABLE_ID), THIRD_VARIABLE_VAL); +} + +TEST_F(I_KeywordRuntimeStateTest, OffsetRuntimeState_list_negative_test) { + addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET); + addOffsetState("HTTP_REQ_COOKIE", SECOND_OFFSET); + addOffsetState("HTTP_METHOD", THIRD_OFFSET); + + cptestPrepareToDie(); + EXPECT_DEATH(getVariable(FIRST_OFFSET), ""); + EXPECT_DEATH(getVariable(SECOND_OFFSET), ""); + EXPECT_DEATH(getVariable(THIRD_OFFSET), ""); +} + +TEST_F(I_KeywordRuntimeStateTest, offset_runtime_state_list_positive_test) { + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), zero); + EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), zero); + + addOffsetState("HTTP_REQUEST_HEADERS", FIRST_OFFSET); + addOffsetState("HTTP_REQ_COOKIE", SECOND_OFFSET); + + EXPECT_EQ(getOffset("HTTP_REQUEST_HEADERS"), FIRST_OFFSET); + EXPECT_EQ(getOffset("HTTP_REQ_COOKIE"), SECOND_OFFSET); +} + +TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_positive_test) { + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), zero); + EXPECT_EQ(getOffset("HTTP_METHOD"), zero); + + addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET); + addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL); + addOffsetState("HTTP_METHOD", THIRD_OFFSET); + + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), FIRST_OFFSET); + EXPECT_EQ(getOffset("HTTP_METHOD"), THIRD_OFFSET); + + EXPECT_EQ(getVariable(SECOND_VARIABLE_ID), SECOND_VARIABLE_VAL); +} + +TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_negative_test) { + addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET); + addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL); + addOffsetState("HTTP_METHOD", THIRD_OFFSET); + + cptestPrepareToDie(); + EXPECT_DEATH(getVariable(FIRST_OFFSET), ""); + EXPECT_DEATH(getVariable(THIRD_OFFSET), ""); +} + +TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_offset_shadowing_test) { + addOffsetState("HTTP_COMPLETE_URL_ENCODED", FIRST_OFFSET); + + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), FIRST_OFFSET); + + addVariableState(SECOND_VARIABLE_ID, SECOND_VARIABLE_VAL); + addOffsetState("HTTP_COMPLETE_URL_ENCODED", THIRD_OFFSET); + + EXPECT_EQ(getOffset("HTTP_COMPLETE_URL_ENCODED"), THIRD_OFFSET); +} + +TEST_F(I_KeywordRuntimeStateTest, mixed_types_list_variable_shadowing_test) { + addVariableState(FIRST_VARIABLE_ID, FIRST_VARIABLE_VAL); + + EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), FIRST_VARIABLE_VAL); + + addOffsetState("HTTP_COMPLETE_URL_ENCODED", SECOND_OFFSET); + addVariableState(FIRST_VARIABLE_ID, THIRD_VARIABLE_VAL); + + EXPECT_EQ(getVariable(FIRST_VARIABLE_ID), THIRD_VARIABLE_VAL); +} diff --git a/components/utils/keywords/length_keyword.cc b/components/utils/keywords/length_keyword.cc new file mode 100644 index 0000000..25906fa --- /dev/null +++ b/components/utils/keywords/length_keyword.cc @@ -0,0 +1,162 @@ +#include "single_keyword.h" +#include "output.h" +#include "debug.h" +#include "flags.h" + +#include +#include + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class LengthKeyword : public SingleKeyword +{ +public: + explicit LengthKeyword(const vector &attr, VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override; + +private: + enum class Mode { EXACT, MIN, MAX, COUNT }; + using ModeFlags = Flags; + + void + setRelative(const KeywordAttr &attr, const VariablesMapping &) + { + is_relative.setAttr(attr, "length"); + } + + void + setExact(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation"); + mode.setFlag(Mode::EXACT); + } + + void + setMin(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation"); + mode.setFlag(Mode::MIN); + } + + + void + setMax(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'length' keyword operation"); + mode.setFlag(Mode::MAX); + } + + + void + setContext(const KeywordAttr &attr, const VariablesMapping &) + { + ctx.setAttr(attr, "length"); + } + + bool + isConstant() const + { + return !is_relative && compare_size.isConstant(); + } + + BoolAttr is_relative; + ModeFlags mode; + CtxAttr ctx; + uint var_id; + NumericAttr compare_size; + + static const map setops; +}; + +const map LengthKeyword::setops = { + { "relative", &LengthKeyword::setRelative }, + { "exact", &LengthKeyword::setExact }, + { "min", &LengthKeyword::setMin }, + { "max", &LengthKeyword::setMax }, + { "part", &LengthKeyword::setContext } +}; + +LengthKeyword::LengthKeyword(const vector &attrs, VariablesMapping &vars) +{ + if (attrs.size() == 0) throw KeywordError("Invalid number of attributes in the 'length' keyword"); + + //parisng first attribute (Required) - variable name + auto &var_name_param = attrs[0].getParams(); + + if (var_name_param.size() != 1) { + throw KeywordError("More than one element in the variable name in the 'length' keyword"); + } + + const string &string_var_name = var_name_param[0]; + + if (string_var_name == "relative") { + throw KeywordError("The 'relative' cannot be the variable name in the 'length' keyword"); + } + if (string_var_name == "part") { + throw KeywordError("The 'part' cannot be the variable name in the 'length' keyword"); + } + if (string_var_name == "exact") { + throw KeywordError("The 'exact' cannot be the variable name in the 'length' keyword"); + } + if (string_var_name == "min") { + throw KeywordError("The 'min' cannot be the variable name in the 'length' keyword"); + } + if (string_var_name == "max") { + throw KeywordError("The 'max' cannot be the variable name in the 'length' keyword"); + } + + //parsing the other optional attributes + for (uint i = 1; i < attrs.size(); i++) { + auto curr = setops.find(attrs[i].getAttrName()); + if (curr == setops.end()) { + throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'length' keyword"); + } + auto set_func = curr->second; + (this->*set_func)(attrs[i], vars); + } + + if (mode.empty()) { + if (isdigit(string_var_name[0]) || string_var_name[0] == '-') { + throw KeywordError("Malformed variable name in the 'length' keyword"); + } + + var_id = vars.addNewVariable(string_var_name); + } else { + compare_size.setAttr("length value", string_var_name, vars, "length", 10, true); + } +} + +MatchStatus +LengthKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + auto part = Singleton::Consume::by()->get(static_cast(ctx)); + + if (!part.ok()) return MatchStatus::NoMatchFinal; + + uint offset = is_relative ? prev->getOffset(ctx) : 0; + uint size = (*part).size(); + + if (offset <= size) { + if (mode.isSet(Mode::EXACT)) { + if (size - offset == static_cast(compare_size.evalAttr(prev))) return runNext(prev); + } else if (mode.isSet(Mode::MIN)) { + if (size - offset >= static_cast(compare_size.evalAttr(prev))) return runNext(prev); + } else if (mode.isSet(Mode::MAX)) { + if (size - offset <= static_cast(compare_size.evalAttr(prev))) return runNext(prev); + } else { + VariableRuntimeState new_length_var(prev, var_id, size-offset); + return runNext(&new_length_var); + } + } + + // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match + return isConstant() ? MatchStatus::NoMatchFinal : MatchStatus::NoMatch; +} + +unique_ptr +genLengthKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/no_match_keyword.cc b/components/utils/keywords/no_match_keyword.cc new file mode 100644 index 0000000..b3ea4d3 --- /dev/null +++ b/components/utils/keywords/no_match_keyword.cc @@ -0,0 +1,29 @@ +#include "single_keyword.h" +#include "table_opaque.h" +#include "debug.h" + +#include +#include + +#include "cereal/types/set.hpp" + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class NoMatchKeyword : public SingleKeyword +{ +public: + explicit NoMatchKeyword(const vector &attr, VariablesMapping &) + { + if (!attr.empty()) throw KeywordError("The 'no_match' keyword doesn't take attributes"); + } + + MatchStatus isMatch(const I_KeywordRuntimeState *) const override { return MatchStatus::NoMatchFinal; } +}; + +unique_ptr +genNoMatchKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/pcre_keyword.cc b/components/utils/keywords/pcre_keyword.cc new file mode 100644 index 0000000..c9ba9e3 --- /dev/null +++ b/components/utils/keywords/pcre_keyword.cc @@ -0,0 +1,372 @@ +#include "single_keyword.h" + +#include + +#define PCRE2_CODE_UNIT_WIDTH 8 +#include + +#include "output.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class PCREKeyword : public SingleKeyword +{ +public: + explicit PCREKeyword(const vector &attr, const VariablesMapping &known_vars); + MatchStatus isMatch(const I_KeywordRuntimeState *prev) const override; + +private: + void + setOffset(const KeywordAttr &attr, const VariablesMapping &vars) + { + offset.setAttr(attr, vars, "pcre"); + } + + void + setDepth(const KeywordAttr &attr, const VariablesMapping &vars) + { + depth.setAttr(attr, vars, "pcre"); + } + + void + setRelative(const KeywordAttr &attr, const VariablesMapping &) + { + is_relative.setAttr(attr, "pcre"); + } + + void + setCaseInsensitive(const KeywordAttr &attr, const VariablesMapping &) + { + is_case_insensitive.setAttr(attr, "pcre"); + } + + void + setContext(const KeywordAttr &attr, const VariablesMapping &) + { + ctx.setAttr(attr, "pcre"); + } + + string parseString(const string &str); + pair findExprInStr(const string &str, size_t start, size_t end); + void parseOptions(const string &str); + void compilePCRE(const string &str); + + pair getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const; + + bool + isConstant() const + { + return !is_relative && offset.isConstant() && depth.isConstant(); + } + + class PCREDelete + { + public: + void + operator()(pcre2_code *ptr) + { + pcre2_code_free(ptr); + } + }; + unique_ptr pcre_machine; + + class PCREResultDelete + { + public: + void + operator()(pcre2_match_data *ptr) + { + pcre2_match_data_free(ptr); + } + }; + unique_ptr pcre_result; + + NumericAttr offset; + NumericAttr depth; + BoolAttr is_negative; + BoolAttr is_relative; + + BoolAttr is_case_insensitive; + BoolAttr is_multiline; + BoolAttr is_dotall; + BoolAttr is_extended; + BoolAttr is_dollar_endonly; + BoolAttr is_anchor; + BoolAttr is_ungreedy; + + CtxAttr ctx; + + string pcre_expr; + + static const map setops; +}; + +const map PCREKeyword::setops = { + { "relative", &PCREKeyword::setRelative }, + { "offset", &PCREKeyword::setOffset }, + { "depth", &PCREKeyword::setDepth }, + { "nocase", &PCREKeyword::setCaseInsensitive }, + { "part", &PCREKeyword::setContext }, +}; + +PCREKeyword::PCREKeyword(const vector &attrs, const VariablesMapping &known_vars) + : + offset(), + depth() +{ + auto &expr_param = attrs[0].getParams(); + if (expr_param.size() != 1) throw KeywordError("More than one element in the 'pcre' keyword pattern"); + auto expr = parseString(expr_param[0]); + dbgDebug(D_KEYWORD) << "Creating a new 'pcre' expression: " << expr; + + for (uint i = 1; isecond; + (this->*set_func)(attrs[i], known_vars); + } + + compilePCRE(expr); +} + +string +PCREKeyword::parseString(const string &str) +{ + size_t start_offset = 0, end_offset = str.size(); + + if (start_offset=end_offset || str[start_offset]!='"' || str[end_offset-1]!='"') { + throw KeywordError("The 'pcre' expression should be enclosed in quotation marks"); + } + start_offset++; + end_offset--; + + string expr, options; + tie(expr, options) = findExprInStr(str, start_offset, end_offset); + + parseOptions(options); + + return expr; +} + +pair +PCREKeyword::findExprInStr(const string &str, size_t start, size_t end) +{ + if (start>=end) throw KeywordError("The 'pcre' string is empty"); + + // There are two way to write the regular expression: + // Either between '/' charecters: "/regexp/" (this is the default delimiter) + // Or use 'm' to set a delimiter: "mDregexpD" (here 'D' is used as the delimiter) + // The switch will set the parameter 'start' so the 'str[start]' is the first delimiter. + switch (str[start]) { + case '/': { + break; + } + case 'm': { + start++; + if (start>=end) { + throw KeywordError("Failed to detect a delimiter in the 'pcre' keyword regular expression"); + } + break; + } + default: + KeywordError("Bad start for the 'pcre' regular expression"); + } + + size_t expr_end = str.find_last_of(str[start], end-1); + start++; + if (expr_end<=start) throw KeywordError("The 'pcre' regular expression is empty"); + + auto options_start = expr_end+1; + return make_pair(str.substr(start, expr_end-start), str.substr(options_start, end-options_start)); +} + +void +PCREKeyword::parseOptions(const string &options) +{ + for (auto ch : options) { + switch (ch) { + case 'i': { + is_case_insensitive.setAttr("pcre", "nocase"); + break; + } + case 'R': { + is_relative.setAttr("pcre", "relative"); + break; + } + case 'm': { + is_multiline.setAttr("pcre", "multiline"); + break; + } + case 's': { + is_dotall.setAttr("pcre", "dotall"); + break; + } + case 'x': { + is_extended.setAttr("pcre", "extended"); + break; + } + case 'E': { + is_dollar_endonly.setAttr("pcre", "dollar_endonly"); + break; + } + case 'A': { + is_anchor.setAttr("pcre", "anchor"); + break; + } + case 'G': { + is_ungreedy.setAttr("pcre", "ungreedy"); + break; + } + default: + throw KeywordError("Unknown option " + dumpHexChar(ch) + " in the 'pcre' keyword"); + } + } +} + +void +PCREKeyword::compilePCRE(const string &expr) +{ + uint32_t options = PCRE2_NO_AUTO_CAPTURE; + if (is_case_insensitive) options |= PCRE2_CASELESS; + if (is_multiline) options |= PCRE2_MULTILINE; + if (is_dotall) options |= PCRE2_DOTALL; + if (is_extended) options |= PCRE2_EXTENDED; + if (is_dollar_endonly) options |= PCRE2_DOLLAR_ENDONLY; + if (is_anchor) options |= PCRE2_ANCHORED; + if (is_ungreedy) options |= PCRE2_UNGREEDY; + + int error; + PCRE2_SIZE error_offset; + auto pattern = reinterpret_cast(expr.c_str()); + pcre_machine.reset(pcre2_compile(pattern, expr.size(), options, &error, &error_offset, nullptr)); + if (pcre_machine == nullptr) { + vector msg; + msg.reserve(128); + pcre2_get_error_message(error, msg.data(), msg.capacity()); + + throw KeywordError( + "Failed to compile the 'pcre' at offset " + + to_string(error_offset) + + " with error: " + + reinterpret_cast(msg.data()) + ); + } + + pcre_result.reset(pcre2_match_data_create_from_pattern(pcre_machine.get(), nullptr)); + if (pcre_result == nullptr) { + throw KeywordError("Failed to allocate PCRE results container"); + } + + pcre_expr = expr; +} + +static uint +addOffset(uint offset, int add) +{ + if (add<0 && offset(-add)) return 0; + return offset + add; +} + +pair +PCREKeyword::getStartOffsetAndLength(uint buf_size, const I_KeywordRuntimeState *prev) const +{ + uint keyword_offset = is_relative?prev->getOffset(ctx):0; + uint start_offset = addOffset(keyword_offset, offset.evalAttr(prev)); + + if (start_offset>=buf_size) return make_pair(0, 0); + + uint length = buf_size-start_offset; + + if (depth.isSet()) { + length = min(length, static_cast(depth.evalAttr(prev))); + } + + return make_pair(start_offset, length); +} + +MatchStatus +PCREKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + dbgAssert(pcre_machine!=nullptr) << "Trying to run on an uninitialized keyword 'pcre'"; + + auto part = Singleton::Consume::by()->get(static_cast(ctx)); + + if (!part.ok()) { + if (is_negative) { + return runNext(prev); + } + return MatchStatus::NoMatchFinal; + } + + uint offset, length; + tie(offset, length) = getStartOffsetAndLength((*part).size(), prev); + auto buf = (*part).getPtr(offset, length); + + if (!buf.ok()) { + dbgTrace(D_KEYWORD) << "Could not get the buffer for the 'pcre' keyword"; + return MatchStatus::NoMatchFinal; + } + const unsigned char *ptr = *buf; + + bool match_found = false; + uint buf_offset_found; + for (uint buf_pos = 0; buf_pos +genPCREKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/components/utils/keywords/sentinel_runtime_state.h b/components/utils/keywords/sentinel_runtime_state.h new file mode 100644 index 0000000..e2a6808 --- /dev/null +++ b/components/utils/keywords/sentinel_runtime_state.h @@ -0,0 +1,13 @@ +#ifndef ___SENTINEL_RUNTIME_STATE_H__ +#define ___SENTINEL_RUNTIME_STATE_H__ + +#include "single_keyword.h" + +class SentinelRuntimeState : public I_KeywordRuntimeState +{ +public: + uint getOffset(const std::string &) const override; + uint getVariable(uint) const override; +}; + +#endif // ___SENTINEL_RUNTIME_STATE_H__ diff --git a/components/utils/keywords/single_keyword.cc b/components/utils/keywords/single_keyword.cc new file mode 100644 index 0000000..a8437c0 --- /dev/null +++ b/components/utils/keywords/single_keyword.cc @@ -0,0 +1,276 @@ +#include "single_keyword.h" + +#include + +using namespace std; + +void +SingleKeyword::appendKeyword(unique_ptr &&_next) +{ + if (next==nullptr) { + next = move(_next); + } else { + next->appendKeyword(move(_next)); + } +} + +MatchStatus +SingleKeyword::runNext(const I_KeywordRuntimeState *curr) const +{ + if (next==nullptr) { + return MatchStatus::Match; + } + return next->isMatch(curr); +} + +OffsetRuntimeState::OffsetRuntimeState( + const I_KeywordRuntimeState *_p, + const string &_ctx, + uint _offset) + : + prev(_p), + ctx(_ctx), + offset(_offset) +{ +} + +uint +OffsetRuntimeState::getOffset(const string &requested_ctx) const +{ + if (ctx==requested_ctx) return offset; + return prev->getOffset(requested_ctx); +} + +uint +OffsetRuntimeState::getVariable(uint requested_var_id) const +{ + return prev->getVariable(requested_var_id); +} + +VariableRuntimeState::VariableRuntimeState( + const I_KeywordRuntimeState *_p, + uint _var_id, + uint _val) + : + prev(_p), + var_id(_var_id), + value(_val) +{ +} + +uint +VariableRuntimeState::getOffset(const string &requested_ctx) const +{ + return prev->getOffset(requested_ctx); +} + +uint +VariableRuntimeState::getVariable(uint requested_var_id) const +{ + if (var_id==requested_var_id) return value; + return prev->getVariable(requested_var_id); +} + +uint +VariablesMapping::addNewVariable(const string ¶m) +{ + auto iter = mapping.find(param); + if (iter==mapping.end()) { + mapping[param] = mapping.size(); + } + return mapping[param]; +} + +Maybe +VariablesMapping::getVariableId(const string ¶m) const +{ + auto iter = mapping.find(param); + if (iter==mapping.end()) { + return genError(string("Unknown parameter ")+param); + } + return iter->second; +} + +void +NumericAttr::setAttr( + const KeywordAttr &attr, + const VariablesMapping &known_vars, + const string &keyword_name, + const uint base, + bool is_unsigned_val) +{ + auto &vec = attr.getParams(); + if (vec.size()!= 2) { + throw KeywordError("Malformed " + attr.getAttrName() + "' in the '" + keyword_name + "' keyword"); + } + + setAttr(attr.getAttrName(), vec[1], known_vars, keyword_name, base, is_unsigned_val); +} + +void +NumericAttr::setAttr( + const string &attr_name, + const string ¶m, + const VariablesMapping &known_vars, + const string &keyword_name, + const uint base, + bool is_unsigned_val) +{ + if (isSet()) { + throw KeywordError("Double definition of the '" + attr_name + "' in the '" + keyword_name + "' keyword"); + } + + if (is_unsigned_val && param[0]=='-') { + throw KeywordError( + "Negative constant '" + + param + + "' in the '" + + attr_name + + "' in the '" + + keyword_name + + "' keyword" + ); + } + + if (isdigit(param[0]) || param[0] == '-') { + status = Status::Const; + try { + size_t idx; + val = stol(param, &idx, base); + if (idx != param.length()) throw invalid_argument(""); + } + catch (...) { + throw KeywordError( + "Malformed constant '" + + param + + "' in the '" + + attr_name + + "' in the '" + + keyword_name + + "' keyword" + ); + } + } else { + status = Status::Var; + val = known_vars.getVariableId(param).unpack( + "In " + keyword_name + + " in " + attr_name + ": " + ); + } +} + +int +NumericAttr::evalAttr(const I_KeywordRuntimeState *prev) const +{ + if (status==Status::Var) { + return prev->getVariable(val); + } + return val; +} + +void +BoolAttr::setAttr(const KeywordAttr &attr, const string &keyword_name) +{ + if (attr.getParams().size()!=1) { + throw KeywordError("Malformed " + attr.getAttrName() + "' in the '" + keyword_name + "' keyword"); + } + + val = true; +} + +void +BoolAttr::setAttr(const string &keyword_name, const string &attr_name) +{ + if (val) throw KeywordError("Double definition of the '" + attr_name + "' in the '" + keyword_name + "' keyword"); + + val = true; +} + +void +CtxAttr::setAttr(const KeywordAttr &attr, const string &keyword_name) +{ + if (is_set) throw KeywordError("Double definition of the 'part' in the '" + keyword_name + "' keyword"); + is_set = true; + auto vec = attr.getParams(); + if (vec.size()!=2) throw KeywordError("Malformed 'part' in the '" + keyword_name + "' keyword"); + ctx = vec[1]; +} + +const map ComparisonAttr::name_to_operator { + { "=", CompId::EQUAL }, + { "!=", CompId::NOT_EQUAL }, + { "<", CompId::LESS_THAN }, + { ">", CompId::GREATER_THAN }, + { "<=", CompId::LESS_THAN_OR_EQUAL }, + { ">=", CompId::GREATER_THAN_OR_EQUAL } +}; + +Maybe +ComparisonAttr::getComparisonByName(const string &name) +{ + auto iter = name_to_operator.find(name); + if (iter == name_to_operator.end()) { + return genError("Could not find the operator: " + name); + } + return iter->second; +} + +void +ComparisonAttr::setAttr(const string ¶m, const string &keyword_name) +{ + if (isSet()) { + throw KeywordError("Double definition of the comparison opearator in the '" + keyword_name + "' keyword"); + } + is_set = true; + comp_val = getComparisonByName(param).unpack( + "Unknown comparison operator in the '" + keyword_name + "' keyword: " + ); +} + +bool +ComparisonAttr::operator()(int first_val, int second_val) const +{ + switch (comp_val) { + case ComparisonAttr::CompId::EQUAL: { + return first_val == second_val; + } + case ComparisonAttr::CompId::NOT_EQUAL: { + return first_val != second_val; + } + case ComparisonAttr::CompId::LESS_THAN: { + return first_val < second_val; + } + case ComparisonAttr::CompId::GREATER_THAN: { + return first_val > second_val; + } + case ComparisonAttr::CompId::LESS_THAN_OR_EQUAL: { + return first_val <= second_val; + } + case ComparisonAttr::CompId::GREATER_THAN_OR_EQUAL: { + return first_val >= second_val; + } + } + dbgAssert(false) << "ComparisonAttr::operator found an invalid comparison operator"; + return false; +} + +using InitFunc = unique_ptr(*)(const vector &, VariablesMapping &); + +const map initializers = { + {"data", genDataKeyword }, + {"pcre", genPCREKeyword }, + {"length", genLengthKeyword }, + {"byte_extract", genByteExtractKeyword }, + {"compare", genCompareKeyword }, + {"stateop", genStateopKeyword }, + {"no_match", genNoMatchKeyword }, + {"jump", genJumpKeyword } +}; + +unique_ptr +getKeywordByName(const KeywordParsed &keyword, VariablesMapping &known_vars) +{ + auto iter = initializers.find(keyword.getName()); + if (iter==initializers.end()) throw KeywordError(keyword.getName() + " - unknown keyword type"); + return iter->second(keyword.getAttr(), known_vars); +} diff --git a/components/utils/keywords/single_keyword.h b/components/utils/keywords/single_keyword.h new file mode 100644 index 0000000..25766a0 --- /dev/null +++ b/components/utils/keywords/single_keyword.h @@ -0,0 +1,300 @@ +#ifndef ___SINGLE_KEYWORD_H__ +#define ___SINGLE_KEYWORD_H__ + +#include +#include +#include +#include + +#include "buffer.h" +#include "keyword_comp.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_KEYWORD); + +enum class MatchStatus { Match, NoMatch, NoMatchFinal }; + +class KeywordError +{ +public: + KeywordError(const std::string &str) : err(str) + { + } + + const std::string & + getErr() const + { + return err; + } + +private: + std::string err; +}; + +class KeywordAttr +{ +public: + KeywordAttr(const std::string &str); + + const std::string& + getAttrName() const + { + return params[0]; + } + + const std::vector & + getParams() const + { + return params; + } + +private: + std::vector params; +}; + +class KeywordParsed +{ +public: + KeywordParsed(const std::string &keyword); + + const std::string & + getName() const + { + return name; + } + + const std::vector & + getAttr() const + { + return attr; + } + +private: + std::string name; + std::vector attr; +}; + +class I_KeywordRuntimeState +{ +public: + virtual uint getOffset(const std::string &ctx) const = 0; + virtual uint getVariable(uint requested_var_id) const = 0; +protected: + virtual ~I_KeywordRuntimeState() + { + } +}; + +class OffsetRuntimeState : public I_KeywordRuntimeState +{ +public: + OffsetRuntimeState(const I_KeywordRuntimeState *prev, const std::string &ctx, uint offset); + virtual ~OffsetRuntimeState() + { + } + uint getOffset(const std::string &requested_ctx_id) const override; + uint getVariable(uint requested_var_id) const override; + +private: + const I_KeywordRuntimeState *prev; + std::string ctx; + uint offset; +}; + +class VariableRuntimeState : public I_KeywordRuntimeState +{ +public: + VariableRuntimeState(const I_KeywordRuntimeState *prev, uint var_id, uint val); + virtual ~VariableRuntimeState() + { + } + + uint getOffset(const std::string &requested_ctx_id) const override; + uint getVariable(uint requested_var_id) const override; + +private: + const I_KeywordRuntimeState *prev; + uint var_id; + uint value; +}; + +class VariablesMapping +{ +public: + uint addNewVariable(const std::string &name); + Maybe getVariableId(const std::string &name) const; + +private: + std::map mapping; +}; + +class NumericAttr +{ + enum class Status { Unset, Const, Var }; +public: + void setAttr( + const KeywordAttr &attr, + const VariablesMapping &known_vars, + const std::string &keyword_name, + const uint base = 10, + bool is_unsigned_val = false); + + void setAttr( + const std::string &attr_name, + const std::string ¶m, + const VariablesMapping &known_vars, + const std::string &keyword_name, + const uint base = 10, + bool is_unsigned_val = false); + + int evalAttr(const I_KeywordRuntimeState *prev) const; + + bool + isConstant() const + { + return status!=Status::Var; + } + + bool + isSet() const + { + return status!=Status::Unset; + } + +private: + Status status = Status::Unset; + int val = 0; +}; + +class BoolAttr +{ +public: + void setAttr(const KeywordAttr &attr, const std::string &keyword_name); + void setAttr(const std::string &keyword_name, const std::string &attr_name); + + operator bool() const + { + return val; + } + +private: + bool val = false; +}; + +class CtxAttr +{ +public: + void setAttr(const KeywordAttr &attr, const std::string &keyword_name); + + operator std::string() const + { + if (!is_set) { + auto env = Singleton::Consume::by(); + auto default_ctx = env->get(I_KeywordsRule::getKeywordsRuleTag()); + if (default_ctx.ok()) return *default_ctx; + dbgError(D_KEYWORD) << "Running keyword rule without specific context and without default"; + return "Missing Default Context"; + } + return ctx; + } + +private: + std::string ctx; + bool is_set = false; +}; + +class ComparisonAttr +{ +public: + enum class CompId + { + EQUAL, + NOT_EQUAL, + LESS_THAN, + GREATER_THAN, + LESS_THAN_OR_EQUAL, + GREATER_THAN_OR_EQUAL + }; + + void setAttr(const std::string ¶m, const std::string &keyword_name); + bool operator()(int first_val, int second_val) const; + + bool + isSet() const + { + return is_set; + } + +private: + Maybe getComparisonByName(const std::string &name); + + const static std::map name_to_operator; + bool is_set = false; + CompId comp_val; +}; + +class SingleKeyword +{ +public: + SingleKeyword() + { + } + + virtual ~SingleKeyword() + { + } + + MatchStatus runNext(const I_KeywordRuntimeState *curr) const; + void appendKeyword(std::unique_ptr &&_next); + virtual MatchStatus isMatch(const I_KeywordRuntimeState *prev) const = 0; + +private: + std::unique_ptr next; +}; + +std::unique_ptr genDataKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genPCREKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genLengthKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genByteExtractKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genCompareKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genStateopKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genNoMatchKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr genJumpKeyword( + const std::vector &attr, + VariablesMapping &vars +); + +std::unique_ptr getKeywordByName( + const KeywordParsed &parsed_data, + VariablesMapping &vars +); + +#endif // ___SINGLE_KEYWORD_H__ diff --git a/components/utils/keywords/stateop_keyword.cc b/components/utils/keywords/stateop_keyword.cc new file mode 100644 index 0000000..c6f5f78 --- /dev/null +++ b/components/utils/keywords/stateop_keyword.cc @@ -0,0 +1,145 @@ +#include "single_keyword.h" +#include "table_opaque.h" +#include "debug.h" +#include "flags.h" + +#include +#include + +#include "cereal/types/set.hpp" + +using namespace std; + +USE_DEBUG_FLAG(D_KEYWORD); + +class StateopKeyword : public SingleKeyword +{ +public: + explicit StateopKeyword(const vector &attr, VariablesMapping &vars); + MatchStatus isMatch(const I_KeywordRuntimeState* prev) const override; + +private: + enum class Operation { ISSET, SET, UNSET, COUNT }; + using OpFlags = Flags; + + void + setState(const KeywordAttr &attr, const VariablesMapping &) + { + auto &var_name_param = attr.getParams(); + + if (var_name_param.size() != 2) { + throw KeywordError("More than one element in the state name in the 'stateop' keyword"); + } + + var_name = var_name_param[1]; + } + + void + setTesting(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation"); + mode.setFlag(Operation::ISSET); + } + + void + setSetting(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation"); + mode.setFlag(Operation::SET); + } + + void + setUnsetting(const KeywordAttr &, const VariablesMapping &) + { + if (!mode.empty()) throw KeywordError("Redefining 'stateop' keyword operation"); + mode.setFlag(Operation::UNSET); + } + + string var_name; + OpFlags mode; + + static const map setops; +}; + +const map StateopKeyword::setops = { + { "isset", &StateopKeyword::setTesting }, + { "set", &StateopKeyword::setSetting }, + { "unset", &StateopKeyword::setUnsetting }, + { "state", &StateopKeyword::setState } +}; + +StateopKeyword::StateopKeyword(const vector &attrs, VariablesMapping &vars) +{ + if (attrs.size() != 2) throw KeywordError("Invalid number of attributes in the 'stateop' keyword"); + + for (uint i = 0; i < attrs.size(); i++) { + auto curr = setops.find(attrs[i].getAttrName()); + if (curr == setops.end()) { + throw KeywordError("Unknown attribute '" + attrs[i].getAttrName() + "' in the 'stateop' keyword"); + } + auto set_func = curr->second; + (this->*set_func)(attrs[i], vars); + } + + if (var_name == "" || mode.empty()) { + throw KeywordError("Bad 'stateop' attribute configuration"); + } +} + +class KeywordStateop : public TableOpaqueSerialize +{ +public: + KeywordStateop() : TableOpaqueSerialize(this) {} + + bool hasVariable(const string &state) { return states.find(state) != states.end(); } + void addVariable(const string &state) { states.insert(state); } + void removeVariable(const string &state) { states.erase(state); } + + // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists + template + void + serialize(T &ar, uint32_t) + { + ar(states); + } + + static std::string name() { return "KeywordStateop"; } + static std::unique_ptr prototype() { return std::make_unique(); } + static uint currVer() { return 0; } + static uint minVer() { return 0; } + // LCOV_EXCL_STOP + +private: + set states; +}; + +MatchStatus +StateopKeyword::isMatch(const I_KeywordRuntimeState *prev) const +{ + auto table = Singleton::Consume::by(); + + if (mode.isSet(Operation::ISSET)) { + if (!table->hasState()) return MatchStatus::NoMatchFinal; + auto &state = table->getState(); + if (state.hasVariable(var_name)) return runNext(prev); + else return MatchStatus::NoMatchFinal; + } else if (mode.isSet(Operation::SET)) { + if (!table->hasState()) table->createState(); + table->getState().addVariable(var_name); + return runNext(prev); + } else if (mode.isSet(Operation::UNSET)) { + if (table->hasState()) table->getState().removeVariable(var_name); + return runNext(prev); + } else { + dbgAssert(false) << "Impossible 'stateop' keyword without operation"; + } + + // If there was no matches and the keyword is effected by other keywords, then we know that the rule won't match + return MatchStatus::NoMatchFinal; +} + +unique_ptr +genStateopKeyword(const vector &attr, VariablesMapping &known_vars) +{ + return make_unique(attr, known_vars); +} diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 2ddc0af..9488d32 100755 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -68,6 +68,19 @@ makeDir(const string &path, mode_t permission) return true; } +/// @brief Get basename of a path +/// @param path path to a file +/// @return base file name +string +getFileName(const string &path) +{ + dbgFlow(D_INFRA_UTILS) << "Trying to extract file name from path: " << path; + size_t pos = path.rfind("/"); + if (pos != string::npos) return path.substr(pos+1, path.length() - pos); + + return path; +} + bool makeDirRecursive(const string &path, mode_t permission) { diff --git a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc index 648f2f1..5476f13 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc +++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc @@ -77,6 +77,7 @@ TEST_F(AgentCoreUtilUT, directoryTest) EXPECT_TRUE(NGEN::Filesystem::deleteDirectory("/tmp/1", true)); EXPECT_FALSE(NGEN::Filesystem::exists("/tmp/1")); } + TEST_F(AgentCoreUtilUT, printTest) { EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(0), "0 Bytes"); @@ -96,3 +97,9 @@ TEST_F(AgentCoreUtilUT, printTest) EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(1000*gigabyte), "1000.00 GB"); EXPECT_EQ(NGEN::Filesystem::convertToHumanReadable(1024*gigabyte), "1024.00 GB"); } + + +TEST_F(AgentCoreUtilUT, fileBasenameTest) +{ + EXPECT_EQ(NGEN::Filesystem::getFileName("/test/base/file/name"), "name"); +} diff --git a/core/config/config.cc b/core/config/config.cc index 44432f0..82b8caf 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -150,6 +150,8 @@ private: bool commitFailure(const string &error); bool reloadConfigurationImpl(const string &version, bool is_async); void reloadConfigurationContinuesWrapper(const string &version, uint id); + vector fillMultiTenantConfigFiles(const map> &tenants); + vector fillMultiTenantExpectedConfigFiles(const map> &tenants); string getActiveTenant() const @@ -274,7 +276,7 @@ private: map> expected_configuration_files; set config_file_paths; - I_TenantManager *tenant_mananger = nullptr; + I_TenantManager *tenant_manager = nullptr; vector configuration_prepare_cbs; vector configuration_commit_cbs; @@ -322,7 +324,7 @@ void ConfigComponent::Impl::init() { reloadFileSystemPaths(); - tenant_mananger = Singleton::Consume::by(); + tenant_manager = Singleton::Consume::by(); if (!Singleton::exists()) return; auto mainloop = Singleton::Consume::by(); @@ -338,7 +340,7 @@ ConfigComponent::Impl::init() mainloop->addRecurringRoutine( I_MainLoop::RoutineType::System, - tenant_mananger->getTimeoutVal(), + tenant_manager->getTimeoutVal(), [this] () { clearOldTenants(); }, "Config comp old tenant cleanup" ); @@ -681,7 +683,7 @@ bool ConfigComponent::Impl::areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const { return (tenant_profile.getTenantId() == default_tenant_id && tenant_profile.getProfileId() == default_profile_id) - || tenant_mananger->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getProfileId()); + || tenant_manager->areTenantAndProfileActive(tenant_profile.getTenantId(), tenant_profile.getProfileId()); } void @@ -817,6 +819,45 @@ ConfigComponent::Impl::commitFailure(const string &error) return false; } +vector +ConfigComponent::Impl::fillMultiTenantConfigFiles(const map> &active_tenants) +{ + vector files; + for (const auto &tenant_profiles : active_tenants) { + const string &tenant = tenant_profiles.first; + const set &profile_ids = tenant_profiles.second; + for (const auto &profile_id : profile_ids) { + string settings_path = + config_directory_path + "tenant_" + tenant + "_profile_" + profile_id + "_settings.json"; + files.push_back(settings_path); + } + } + return files; +} + +vector +ConfigComponent::Impl::fillMultiTenantExpectedConfigFiles(const map> &active_tenants) +{ + vector files; + for (const auto &config_file : expected_configuration_files) { + for (const auto &type : config_file.second) { + if (type == ConfigFileType::RawData) continue; + auto global_path = getPolicyConfigPath(config_file.first, type); + auto it = find(files.begin(), files.end(), global_path); + if (it == files.end()) files.push_back(global_path); + for (const pair> &tenant_profiles : active_tenants) { + const string &tenant = tenant_profiles.first; + const set &profile_ids = tenant_profiles.second; + for (const auto &profile_id : profile_ids) { + auto tenant_path = getPolicyConfigPath(config_file.first, type, tenant, profile_id); + files.push_back(tenant_path); + } + } + } + } + return files; +} + bool ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_async) { @@ -831,38 +872,20 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as files.emplace(fullpath, make_shared(fullpath)); } - const auto &active_tenants = tenant_mananger ? tenant_mananger->fetchAllActiveTenants() : vector(); + map> active_tenants = + tenant_manager ? tenant_manager->fetchActiveTenantsAndProfiles() : map>(); dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size(); - for (const auto &config_file : expected_configuration_files) { - for (const auto &type : config_file.second) { - if (type == ConfigFileType::RawData) continue; - auto global_path = getPolicyConfigPath(config_file.first, type); - if (files.find(global_path) == files.end()) { - files.emplace(global_path, make_shared(global_path)); - } - - for (auto &tenant : active_tenants) { - const vector &profile_ids = - tenant_mananger ? tenant_mananger->fetchProfileIds(tenant) : vector(); - for (auto &profile_id : profile_ids) { - auto tenant_path = getPolicyConfigPath(config_file.first, type, tenant, profile_id); - files.emplace(tenant_path, make_shared(tenant_path)); - } - } - } + const vector &config_files = fillMultiTenantConfigFiles(active_tenants); + const vector &expected_config_files = fillMultiTenantExpectedConfigFiles(active_tenants); + for (const string &file : config_files) { + dbgTrace(D_CONFIG) << "Inserting " << file << " to the list of files to be handled"; + files.emplace(file, make_shared(file)); } - - for (const string &tenant : active_tenants) { - const vector &profile_ids = - tenant_mananger ? tenant_mananger->fetchProfileIds(tenant) : vector(); - for (auto &profile_id : profile_ids) { - string settings_path = - config_directory_path + "tenant_" + tenant + "_profile_"+ profile_id + "_settings.json"; - dbgTrace(D_CONFIG) << "Inserting a settings path: " << settings_path; - files.emplace(settings_path, make_shared(settings_path)); - } + for (const string &file : expected_config_files) { + dbgTrace(D_CONFIG) << "Inserting " << file << " to the list of files to be handled"; + files.emplace(file, make_shared(file)); } vector> archives; @@ -883,6 +906,7 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as void ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version, uint id) { + dbgFlow(D_CONFIG) << "Running reloadConfigurationContinuesWrapper. Version: " << version << ", Id: " << id; auto mainloop = Singleton::Consume::by(); LoadNewConfigurationStatus in_progress(id, false, false); diff --git a/core/include/services_sdk/interfaces/i_socket_is.h b/core/include/services_sdk/interfaces/i_socket_is.h index 5683d7e..202bc90 100755 --- a/core/include/services_sdk/interfaces/i_socket_is.h +++ b/core/include/services_sdk/interfaces/i_socket_is.h @@ -22,7 +22,7 @@ class I_Socket { public: - enum class SocketType { UNIX, TCP, UDP }; + enum class SocketType { UNIX, UNIXDG, TCP, UDP }; using socketFd = int; virtual Maybe diff --git a/core/include/services_sdk/interfaces/i_tenant_manager.h b/core/include/services_sdk/interfaces/i_tenant_manager.h index 326fd7b..167eb92 100644 --- a/core/include/services_sdk/interfaces/i_tenant_manager.h +++ b/core/include/services_sdk/interfaces/i_tenant_manager.h @@ -16,6 +16,8 @@ #include #include +#include +#include #include #include @@ -27,13 +29,14 @@ public: virtual void uponNewTenants(const newTenantCB &cb) = 0; virtual bool areTenantAndProfileActive(const std::string &tenant_id, const std::string &profile_id) const = 0; - virtual std::vector fetchActiveTenants() const = 0; - virtual std::vector fetchAllActiveTenants() const = 0; - virtual std::vector getInstances( + virtual std::set fetchAllActiveTenants() const = 0; + virtual std::set fetchActiveTenants() const = 0; + virtual std::set getInstances( const std::string &tenant_id, const std::string &profile_id ) const = 0; - virtual std::vector fetchProfileIds(const std::string &tenant_id) const = 0; + virtual std::map> fetchActiveTenantsAndProfiles() const = 0; + virtual std::set fetchProfileIds(const std::string &tenant_id) const = 0; virtual void deactivateTenant(const std::string &tenant_id, const std::string &profile_id) = 0; @@ -41,7 +44,7 @@ public: virtual std::chrono::microseconds getTimeoutVal() const = 0; - virtual std::vector getProfileId( + virtual std::set getProfileIdsForRegionAccount( const std::string &tenant_id, const std::string ®ion, const std::string &account_id = "" diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h index 5ef8911..e3659d3 100755 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h @@ -81,6 +81,7 @@ IntelligenceQuery::load(cereal::JSONInputArchive &ar) unsigned int valid_idx = 0; const auto &valid_response = bulk_response.getValid(); const auto &errors = bulk_response.getErrors(); + responses.clear(); responses.reserve(requests.size()); dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items"; for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) { diff --git a/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h b/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h index eb31390..ab9bfc1 100644 --- a/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h +++ b/core/include/services_sdk/interfaces/mock/mock_tenant_manager.h @@ -15,19 +15,20 @@ class MockTenantManager : public Singleton::Provide::From()); - MOCK_CONST_METHOD0(fetchAllActiveTenants, std::vector()); - MOCK_CONST_METHOD1(fetchProfileIds, std::vector(const std::string &)); + MOCK_CONST_METHOD0(fetchActiveTenantsAndProfiles, std::map>()); + MOCK_CONST_METHOD0(fetchActiveTenants, std::set()); + MOCK_CONST_METHOD0(fetchAllActiveTenants, std::set()); + MOCK_CONST_METHOD1(fetchProfileIds, std::set(const std::string &)); MOCK_CONST_METHOD2( getInstances, - std::vector(const std::string &, const std::string &) + std::set(const std::string &, const std::string &) ); MOCK_CONST_METHOD2(areTenantAndProfileActive, bool(const std::string &, const std::string &)); MOCK_METHOD2(addActiveTenantAndProfile, void(const std::string &, const std::string &)); MOCK_METHOD2(deactivateTenant, void(const std::string &, const std::string &)); MOCK_CONST_METHOD3( - getProfileId, - std::vector(const std::string &, const std::string &, const std::string &) + getProfileIdsForRegionAccount, + std::set(const std::string &, const std::string &, const std::string &) ); MOCK_CONST_METHOD0(getTimeoutVal, std::chrono::microseconds()); diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 446bbd4..4ee860e 100755 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -137,6 +137,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_NGINX_MESSAGE_READER, D_REVERSE_PROXY) DEFINE_FLAG(D_ERROR_REPORTER, D_REVERSE_PROXY) DEFINE_FLAG(D_UPSTREAM_KEEPALIVE, D_REVERSE_PROXY) + DEFINE_FLAG(D_FORWARD_PROXY, D_REVERSE_PROXY) DEFINE_FLAG(D_IDA, D_COMPONENT) @@ -166,6 +167,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_URL_FILTERING, D_COMPONENT) DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT) + DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT) DEFINE_FLAG(D_FLOW, D_ALL) DEFINE_FLAG(D_DROP, D_FLOW) diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 5ef1d81..1e21246 100755 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -62,6 +62,7 @@ enum class Tags { DEPLOYMENT_EMBEDDED, DEPLOYMENT_K8S, LAYER_7_ACCESS_CONTROL, + HORIZON_TELEMETRY_METRICS, COUNT }; @@ -76,6 +77,7 @@ enum class AudienceTeam SIGNATURE_DEVELOPERS, FILE_UPLOAD, IDENTITY_AWARENESS, + HORIZON_TELEMETRY, NONE, COUNT @@ -147,7 +149,8 @@ enum class IssuingEngine { IOT_NEXT, SDWAN, FILE_UPLOAD, - IDA_NEXT + IDA_NEXT, + HORIZON_TELEMETRY_METRICS }; } // namespace ReportIS diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index c13c070..19b0e67 100755 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -42,6 +42,8 @@ bool deleteFile(const std::string &path); std::string convertToHumanReadable(uint64_t size_in_bytes); +std::string getFileName(const std::string &path); + }// namespace Filesystem namespace Regex diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 81badab..a480f82 100755 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -104,7 +104,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"Kong Server", ReportIS::Tags::WEB_SERVER_KONG}, {"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED}, {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S}, - {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL} + {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL}, + {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS} }; auto report_is_tag = strings_to_tags.find(tag); @@ -264,6 +265,7 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::SDWAN: return "sdwanGwSharing"; case IssuingEngine::FILE_UPLOAD: return "fileUpload"; case IssuingEngine::IDA_NEXT: return "quantumMetaNotifyIdn"; + case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; } dbgAssert(false) << "Reached impossible engine value of: " << static_cast(issuing_engine); @@ -302,7 +304,8 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "Kong Server", "Embedded Deployment", "Kubernetes Deployment", - "Layer 7 Access Control" + "Layer 7 Access Control", + "Horizon Telemetry Metrics" }; EnumArray TagAndEnumManagement::audience_team_translation { @@ -312,5 +315,6 @@ EnumArray TagAndEnumManagement::audience_team_translation "Agent Intelligence", "cpviewMonitoring", "Signature Developers", - "Identity Awareness" + "Identity Awareness", + "unifiedMonitoring" }; diff --git a/core/socket_is/socket_is.cc b/core/socket_is/socket_is.cc index 71bee0b..93f86bf 100755 --- a/core/socket_is/socket_is.cc +++ b/core/socket_is/socket_is.cc @@ -489,6 +489,109 @@ private: struct sockaddr_un server; }; +class UnixDGSocket : public SocketInternal +{ +public: + static Maybe> + connectSock(bool _is_blocking, bool _is_server, const string &_address) + { + unique_ptr unix_socket(make_unique(_is_blocking, _is_server)); + if (unix_socket->getSocket() <= 0) return genError("Failed to create socket"); + + unix_socket->server.sun_family = AF_UNIX; + strncpy(unix_socket->server.sun_path, _address.c_str(), sizeof(unix_socket->server.sun_path) - 1); + + if (!unix_socket->isServerSock()) { + if (connect( + unix_socket->getSocket(), + reinterpret_cast(&unix_socket->server), + sizeof(struct sockaddr_un) + ) == -1 + ) { + return genError("Failed to connect socket"); + } + return move(unix_socket); + } + + static const int on = 1; + if (setsockopt(unix_socket->getSocket(), SOL_SOCKET, SO_REUSEADDR, (char *)&on, sizeof(on)) < 0) { + dbgWarning(D_SOCKET) << "Failed to set the socket descriptor as reusable"; + return genError("Failed to set the socket descriptor as reusable"); + } + + const int priority = 6; + if (setsockopt(unix_socket->getSocket(), SOL_SOCKET, SO_PRIORITY, (char *)&priority, sizeof(priority)) < 0) { + dbgWarning(D_SOCKET) << "Failed to set the socket priority to highest"; + return genError("Failed to set the socket priority to highest"); + } + + if (ioctl(unix_socket->getSocket(), FIONBIO, (char *)&on) < 0) { + dbgWarning(D_SOCKET) << "Failed to set the socket as non-blocking"; + return genError("Failed to set the socket as non-blocking"); + } + + unlink(unix_socket->server.sun_path); + if (bind( + unix_socket->getSocket(), + reinterpret_cast(&unix_socket->server), + sizeof(struct sockaddr_un) + ) == -1) { + dbgWarning(D_SOCKET) << "Failed to bind the socket: " << strerror(errno); + return genError("Failed to bind the socket"); + } + + chmod(unix_socket->server.sun_path, 0666); + + return move(unix_socket); + } + + void cleanServer() override + { + unlink(server.sun_path); + } + + Maybe> + receiveDataBlocking(uint data_size) override + { + return receiveDGData(data_size, MSG_DONTWAIT); + } + + Maybe> + receiveDataNonBlocking(uint data_size) override + { + return receiveDGData(data_size, 0); + } + + Maybe> + receiveDGData(uint data_size, int flag) + { + if (data_size == 0) data_size = udp_max_packet_size; + dbgDebug(D_SOCKET) << "data_size: " << data_size; + vector param_to_read(data_size, 0); + int res = recv(socket_int, param_to_read.data(), data_size, flag); + + if (res == -1) { + string error_message = strerror(errno); + dbgWarning(D_SOCKET) << "Failed to read data, Error: " + error_message; + return genError( + "Failed to read data, Error: " + error_message + ); + } + param_to_read.resize(res); + return param_to_read; + } + + UnixDGSocket(bool _is_blocking, bool _is_server_socket) + : + SocketInternal(_is_blocking, _is_server_socket) + { + socket_int = socket(AF_UNIX, SOCK_DGRAM, 0); + } + +private: + struct sockaddr_un server; +}; + class SocketIS::Impl : Singleton::Provide::From @@ -527,6 +630,11 @@ SocketIS::Impl::genSocket( if (!unix_sock.ok()) return unix_sock.passErr(); new_sock = unix_sock.unpackMove(); socketTypeName = "UNIX"; + } else if (type == SocketType::UNIXDG) { + Maybe> unix_dg_sock = UnixDGSocket::connectSock(is_blocking, is_server, address); + if (!unix_dg_sock.ok()) return unix_dg_sock.passErr(); + new_sock = unix_dg_sock.unpackMove(); + socketTypeName = "UNIXDG"; } else if (type == SocketType::TCP) { Maybe> tcp_sock = TCPSocket::connectSock(is_blocking, is_server, address); if (!tcp_sock.ok()) return tcp_sock.passErr(); diff --git a/core/tenant_manager/tenant_manager.cc b/core/tenant_manager/tenant_manager.cc index bda3fa7..fd01494 100644 --- a/core/tenant_manager/tenant_manager.cc +++ b/core/tenant_manager/tenant_manager.cc @@ -78,10 +78,11 @@ public: void uponNewTenants(const newTenantCB &cb) override; bool areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const override; - vector fetchAllActiveTenants() const override; - vector fetchActiveTenants() const override; - vector getInstances(const string &tenant_id, const string &profile_id) const override; - vector fetchProfileIds(const string &tenant_id) const override; + map> fetchActiveTenantsAndProfiles() const override; + set fetchAllActiveTenants() const override; + set fetchActiveTenants() const override; + set getInstances(const string &tenant_id, const string &profile_id) const override; + set fetchProfileIds(const string &tenant_id) const override; void addActiveTenantAndProfile(const string &tenant_id, const string &profile_id) override; @@ -89,7 +90,11 @@ public: chrono::microseconds getTimeoutVal() const override; - vector getProfileId(const string &tenant_id, const string ®ion, const string &account) const override; + set getProfileIdsForRegionAccount( + const string &tenant_id, + const string ®ion, + const string &account + ) const override; void addInstance(const string &tenant_id, const string &profile_id, const string &instace_id) @@ -111,9 +116,9 @@ public: private: void runUponNewTenants(const vector &new_tenants); void sendTenantAndProfile(const string &tenant_id, const string &profile_id); - vector getAllTenants() const; - vector fetchAllProfileIds(const string &tenant_id) const; - vector getProfileIds(const string &profile_id) const; + set getAllTenants() const; + set fetchAllProfileIds(const string &tenant_id) const; + set getProfileIds(const string &tenant_id) const; bool sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port); TemporaryCache active_tenants; @@ -169,7 +174,7 @@ public: active_tenants = Singleton::Consume::from()->fetchAllActiveTenants(); } - S2C_PARAM(std::vector, active_tenants); + S2C_PARAM(set, active_tenants); }; class GetActiveTenants : public ClientRest @@ -179,7 +184,7 @@ public: Maybe genJson() const { return string("{}"); }; - S2C_PARAM(vector, active_tenants); + S2C_PARAM(set, active_tenants); }; class FetchProfileIds : public ServerRest @@ -191,7 +196,7 @@ public: profile_ids = Singleton::Consume::from()->fetchProfileIds(tenant_id); } - S2C_PARAM(vector, profile_ids); + S2C_PARAM(set, profile_ids); C2S_PARAM(string, tenant_id); }; @@ -200,7 +205,7 @@ class GetProfileIds : public ClientRest public: GetProfileIds(const string &_tenant_id) : profile_ids(), tenant_id(_tenant_id) {}; - S2C_PARAM(vector, profile_ids); + S2C_PARAM(set, profile_ids); C2S_PARAM(string, tenant_id); }; @@ -318,7 +323,7 @@ TenantManager::Impl::sendTenantAndProfile(const string &tenant_id, const string } } -vector +set TenantManager::Impl::getAllTenants() const { dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active tenants"; @@ -348,7 +353,7 @@ TenantManager::Impl::getAllTenants() const return active_tenant.active_tenants.get(); } -vector +set TenantManager::Impl::getProfileIds(const string &_tenant_id) const { dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a client. Requesting the active profiles"; @@ -379,22 +384,25 @@ TenantManager::Impl::getProfileIds(const string &_tenant_id) const } -vector -TenantManager::Impl::getProfileId(const string &tenant_id, const string ®ion, const string &account_id = "") const +set +TenantManager::Impl::getProfileIdsForRegionAccount( + const string &tenant_id, + const string ®ion, + const string &account_id = "") const { if (region.empty()) { dbgWarning(D_TENANT_MANAGER) << "Can't find the profile ID. Region is empty"; - return vector(); + return set(); } - vector profile_ids = fetchProfileIds(tenant_id); + set profile_ids = fetchProfileIds(tenant_id); dbgTrace(D_TENANT_MANAGER) << "Fetched " << profile_ids.size() << " profiles"; auto i_env = Singleton::Consume::by(); auto unset_tenant_on_exit = make_scope_exit([&]() { i_env->unsetActiveTenantAndProfile(); }); - vector profiles_to_return; + set profiles_to_return; for (const string &profile_id : profile_ids) { string account_dbg = account_id.empty() ? "" : (" in the account " + account_id); dbgDebug(D_TENANT_MANAGER) @@ -413,20 +421,20 @@ TenantManager::Impl::getProfileId(const string &tenant_id, const string ®ion, auto account_region_set = maybe_account_region_set.unpack().getAccoutRegionPairs(); if (account_region_set.empty()) { dbgTrace(D_TENANT_MANAGER) << "Old profile with new hook. Resolving to profile ID: " << profile_id; - profiles_to_return.push_back(profile_id); + profiles_to_return.insert(profile_id); return profiles_to_return; } for (const AccountRegionPair &account : account_region_set) { if (region == account.getRegion() && (account_id.empty() || account_id == account.getAccountID())) { dbgTrace(D_TENANT_MANAGER) << "Found a corresponding profile ID: " << profile_id; - profiles_to_return.push_back(profile_id); + profiles_to_return.insert(profile_id); } } } else { auto maybe_region = getSetting("region"); if (maybe_region.ok() && region == maybe_region.unpack()) { dbgDebug(D_TENANT_MANAGER) << "The region corresponds to profile ID " << profile_id; - profiles_to_return.push_back(profile_id); + profiles_to_return.insert(profile_id); return profiles_to_return; } else { if (maybe_region.ok()) { @@ -448,7 +456,7 @@ TenantManager::Impl::getProfileId(const string &tenant_id, const string ®ion, } dbgWarning(D_TENANT_MANAGER) << "Found no corresponding profile ID"; - return vector(); + return set(); } void @@ -490,58 +498,69 @@ TenantManager::Impl::deactivateTenant(const string &tenant_id, const string &pro active_tenants.deleteEntry(TenantProfilePair(tenant_id, profile_id)); } -vector +map> +TenantManager::Impl::fetchActiveTenantsAndProfiles() const +{ + dbgFlow(D_TENANT_MANAGER) << "Fetching active teants and profiles map"; + map> active_tenants_and_profiles; + set tenants = fetchAllActiveTenants(); + for (const string &tenant : tenants) { + active_tenants_and_profiles[tenant] = fetchProfileIds(tenant); + } + + return active_tenants_and_profiles; +} + +set TenantManager::Impl::fetchAllActiveTenants() const { dbgFlow(D_TENANT_MANAGER) << "Fetching all active tenants"; return (type == TenantManagerType::CLIENT) ? getAllTenants() : fetchActiveTenants(); } -vector +set TenantManager::Impl::fetchActiveTenants() const { dbgFlow(D_TENANT_MANAGER) << "Tenant Manager is a server. Fetching active tenants"; - vector tenants; - tenants.reserve(active_tenants.size()); - for (auto iter = begin(active_tenants); iter != end(active_tenants); iter++) { - dbgDebug(D_TENANT_MANAGER) << "Found a tenant to return. Tenant ID: " << iter->first.getTenantId(); - tenants.push_back(iter->first.getTenantId()); + set tenants; + for (const auto &iter : active_tenants) { + dbgDebug(D_TENANT_MANAGER) << "Found a tenant to return. Tenant ID: " << iter.first.getTenantId(); + tenants.insert(iter.first.getTenantId()); } return tenants; } -vector +set TenantManager::Impl::getInstances(const string &tenant_id, const string &profile_id) const { - vector instances; + set instances; auto tenant_profile_pair = TenantProfilePair(tenant_id, profile_id); auto tenant_instance_cache = mapper.find(tenant_profile_pair); if (tenant_instance_cache == mapper.end()) return instances; - instances.reserve(tenant_instance_cache->second.size()); for (auto iter = begin(tenant_instance_cache->second); iter != end(tenant_instance_cache->second); iter++) { - instances.push_back(iter->first); + instances.insert(iter->first); } return instances; } -vector +set TenantManager::Impl::fetchAllProfileIds(const string &tenant_id) const { - vector tenant_profile_ids; + set tenant_profile_ids; for (auto iter = begin(active_tenants); iter != end(active_tenants); iter++) { if (iter->first.getTenantId() == tenant_id) { dbgTrace(D_TENANT_MANAGER) << "Returning a fetched profile ID: " << iter->first.getProfileId(); - tenant_profile_ids.push_back(iter->first.getProfileId()); + tenant_profile_ids.insert(iter->first.getProfileId()); } } return tenant_profile_ids; } -vector +set TenantManager::Impl::fetchProfileIds(const string &tenant_id) const { dbgFlow(D_TENANT_MANAGER) << "Fetching all profile IDs for tenant " << tenant_id; diff --git a/events/include/new_table_entry.h b/events/include/new_table_entry.h new file mode 100644 index 0000000..71b2699 --- /dev/null +++ b/events/include/new_table_entry.h @@ -0,0 +1,10 @@ +#ifndef __NEW_TABLE_ENTRY_H__ +#define __NEW_TABLE_ENTRY_H__ + +#include "event.h" + +class NewTableEntry : public Event +{ +}; + +#endif // __NEW_TABLE_ENTRY_H__ diff --git a/events/include/parsed_context.h b/events/include/parsed_context.h new file mode 100644 index 0000000..f64032c --- /dev/null +++ b/events/include/parsed_context.h @@ -0,0 +1,25 @@ +#ifndef __PARSED_CONTEXT_H__ +#define __PARSED_CONTEXT_H__ + +#include + +#include "event.h" +#include "buffer.h" + +enum class ParsedContextReply { ACCEPT, DROP }; + +class ParsedContext : public Event +{ +public: + ParsedContext(const Buffer &_buf, const std::string &_name, uint _id) : buf(_buf), name(_name), id(_id) {} + const Buffer & getBuffer() const { return buf; } + const std::string & getName() const { return name; } + uint getId() const { return id; } + +private: + Buffer buf; + std::string name; + uint id; +}; + +#endif // __PARSED_CONTEXT_H__ diff --git a/nodes/http_transaction_handler/CMakeLists.txt b/nodes/http_transaction_handler/CMakeLists.txt index 12b60bc..dca96e2 100755 --- a/nodes/http_transaction_handler/CMakeLists.txt +++ b/nodes/http_transaction_handler/CMakeLists.txt @@ -32,6 +32,8 @@ target_link_libraries(cp-nano-http-transaction-handler waap waap_clib reputation + ips + keywords -Wl,--end-group ) diff --git a/nodes/http_transaction_handler/main.cc b/nodes/http_transaction_handler/main.cc index 68f40c0..bd51931 100755 --- a/nodes/http_transaction_handler/main.cc +++ b/nodes/http_transaction_handler/main.cc @@ -17,6 +17,8 @@ #include "gradual_deployment.h" #include "http_manager.h" #include "waap.h" +#include "ips_comp.h" +#include "keyword_comp.h" int main(int argc, char **argv) @@ -26,7 +28,9 @@ main(int argc, char **argv) NginxAttachment, GradualDeployment, HttpManager, - WaapComponent + WaapComponent, + IPSComp, + KeywordComp > comps; comps.registerGlobalValue("Is Rest primary routine", true); diff --git a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc index f55761f..b118499 100755 --- a/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc +++ b/nodes/orchestration/package/cpnano_debug/cpnano_debug.cc @@ -95,6 +95,7 @@ enum class Service { HELLO_WORLD, IDA, IOT_ACCESS_CONTROL, + HORIZON_TELEMETRY, COUNT }; @@ -171,6 +172,7 @@ getServiceString(const Service service) case (Service::HELLO_WORLD): return "hello-world"; case (Service::IDA): return "identity-awareness"; case (Service::IOT_ACCESS_CONTROL): return "iot-access-control"; + case (Service::HORIZON_TELEMETRY): return "horizon-telemetry"; default: cerr << "Internal Error: the provided service (" @@ -359,6 +361,11 @@ getServiceConfig (const Service service) filesystem_path + "/conf/cp-nano-iot-access-control-debug-conf.json", log_files_path + "/nano_agent/cp-nano-iot-access-control.dbg" ); + case (Service::HORIZON_TELEMETRY): + return ServiceConfig( + filesystem_path + "/conf/cp-nano-horizon-telemetry-debug-conf.json", + log_files_path + "/nano_agent/cp-nano-horizon-telemetry.dbg" + ); default: cerr << "Internal Error: the provided service (" @@ -1287,6 +1294,8 @@ extractServices(const vector &args) services.push_back(Service::IDA); } else if (getServiceString(Service::IOT_ACCESS_CONTROL).find(maybe_service) == 0) { services.push_back(Service::IOT_ACCESS_CONTROL); + } else if (getServiceString(Service::HORIZON_TELEMETRY).find(maybe_service) == 0) { + services.push_back(Service::HORIZON_TELEMETRY); } else { break; }