From c2ea2cda6d720282af3c95fab305ce81b6d122c3 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Mon, 14 Oct 2024 14:51:28 +0000 Subject: [PATCH] sync code --- components/include/env_details.h | 3 + .../include/generic_rulebase/match_query.h | 2 +- components/include/ip_utilities.h | 3 +- components/include/waap.h | 4 +- .../access_control_practice.cc | 6 +- .../appsec_practice_section.cc | 95 +-- .../include/appsec_practice_section.h | 50 +- .../include/k8s_policy_utils.h | 5 + .../include/local_policy_common.h | 17 + .../include/new_appsec_policy_crd_parser.h | 2 +- .../include/new_practice.h | 50 +- .../include/rules_config_section.h | 2 + .../local_policy_mgmt_gen/k8s_policy_utils.cc | 87 ++- .../local_policy_mgmt_gen/new_practice.cc | 259 ++++++--- .../policy_maker_utils.cc | 47 +- .../rules_config_section.cc | 16 + .../details_resolver_impl.h | 7 +- .../orchestration/env_details/env_details.cc | 13 + .../orchestration/orchestration_comp.cc | 9 +- .../waap/include/i_transaction.h | 2 + .../security_apps/waap/waap_clib/CidrMatch.cc | 96 ++++ .../security_apps/waap/waap_clib/CidrMatch.h | 4 + .../waap/waap_clib/DeepParser.cc | 7 +- .../waap/waap_clib/ParserBinaryFile.cc | 3 +- .../waap/waap_clib/ParserBinaryFile.h | 2 +- .../security_apps/waap/waap_clib/ParserPDF.cc | 26 +- .../security_apps/waap/waap_clib/ParserPDF.h | 4 +- .../waap/waap_clib/ScanResult.cc | 4 + .../security_apps/waap/waap_clib/ScanResult.h | 3 + .../waap/waap_clib/ScoreBuilder.cc | 75 ++- .../waap/waap_clib/ScoreBuilder.h | 40 +- .../waap/waap_clib/Serializator.cc | 3 + .../waap/waap_clib/WaapOverride.cc | 5 +- .../waap/waap_clib/WaapOverride.h | 105 +++- .../waap/waap_clib/WaapOverrideFunctor.cc | 165 ++++-- .../waap/waap_clib/WaapOverrideFunctor.h | 12 +- .../waap/waap_clib/WaapScanner.cc | 49 +- .../waap/waap_clib/WaapScanner.h | 3 + .../waap/waap_clib/WaapScores.cc | 79 ++- .../security_apps/waap/waap_clib/WaapScores.h | 23 +- .../waap/waap_clib/WaapTrigger.h | 2 + .../waap/waap_clib/Waf2Engine.cc | 28 +- .../security_apps/waap/waap_clib/Waf2Engine.h | 9 +- .../waap/waap_clib/Waf2EngineGetters.cc | 19 + .../security_apps/waap/waap_clib/Waf2Util.cc | 39 +- .../security_apps/waap/waap_clib/Waf2Util.h | 3 +- .../security_apps/waap/waap_component_impl.cc | 3 +- .../security_apps/waap/waap_component_impl.h | 2 + .../utils/generic_rulebase/match_query.cc | 42 +- components/utils/ip_utilities/ip_utilities.cc | 18 +- components/utils/keywords/keywords_rule.cc | 24 +- .../utils/keywords/keywords_ut/keywords_ut.cc | 7 + core/agent_details/agent_details.cc | 6 + core/config/config.cc | 11 +- core/debug_is/debug.cc | 18 +- core/debug_is/debug_is_ut/debug_ut.cc | 2 +- core/environment/environment.cc | 13 +- core/environment/environment_ut/tracing_ut.cc | 2 +- .../services_sdk/interfaces/i_agent_details.h | 1 + .../services_sdk/interfaces/i_env_details.h | 1 + .../interfaces/messaging/messaging_metadata.h | 13 + .../interfaces/mock/mock_agent_details.h | 1 + .../services_sdk/resources/agent_details.h | 1 + .../component_is/components_list_impl.h | 8 + .../services_sdk/resources/debug_flags.h | 2 + .../services_sdk/resources/generic_metric.h | 4 + .../services_sdk/resources/log_generator.h | 2 + .../services_sdk/resources/metric/average.h | 6 + .../services_sdk/resources/metric/counter.h | 6 + .../resources/metric/last_reported_value.h | 6 + .../services_sdk/resources/metric/max.h | 6 + .../resources/metric/metric_calc.h | 115 +++- .../resources/metric/metric_map.h | 69 ++- .../services_sdk/resources/metric/min.h | 6 + .../resources/metric/no_reset_counter.h | 6 + .../resources/metric/top_values.h | 6 + .../intelligence_comp_v2.cc | 1 + core/logging/log_generator.cc | 9 + core/messaging/connection/connection.cc | 11 +- .../messaging_buffer_comp.cc | 5 +- .../messaging_buffer_comp_ut.cc | 2 +- core/metric/generic_metric.cc | 170 ++++++ core/metric/metric_ut/metric_ut.cc | 539 +++++++++++++++++- core/rest/rest_conn.cc | 15 +- core/rest/rest_conn.h | 3 +- core/rest/rest_server.cc | 109 +++- core/rest/rest_ut/rest_config_ut.cc | 224 +++++++- core/rest/rest_ut/rest_schema_ut.cc | 2 +- .../orchestration/package/open-appsec-ctl.sh | 8 +- 89 files changed, 2545 insertions(+), 447 deletions(-) diff --git a/components/include/env_details.h b/components/include/env_details.h index aeaa1b3..0ae9f2f 100644 --- a/components/include/env_details.h +++ b/components/include/env_details.h @@ -29,12 +29,15 @@ public: virtual EnvType getEnvType() override; virtual std::string getToken() override; + virtual std::string getNameSpace() override; private: std::string retrieveToken(); + std::string retrieveNamespace(); std::string readFileContent(const std::string &file_path); std::string token; + std::string agent_namespace; EnvType env_type; }; diff --git a/components/include/generic_rulebase/match_query.h b/components/include/generic_rulebase/match_query.h index 0a660a7..ebf9824 100755 --- a/components/include/generic_rulebase/match_query.h +++ b/components/include/generic_rulebase/match_query.h @@ -91,7 +91,7 @@ private: bool matchAttributesString(const std::set &values) const; bool matchAttributesIp(const std::set &values) const; bool isRegEx() const; - bool isIP() const; + void sortAndMergeIpRangesValues(); MatchType type; Operators operator_type; diff --git a/components/include/ip_utilities.h b/components/include/ip_utilities.h index 9968778..ba3c76f 100755 --- a/components/include/ip_utilities.h +++ b/components/include/ip_utilities.h @@ -28,8 +28,9 @@ // LCOV_EXCL_START Reason: temporary until we add relevant UT until 07/10 bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr); - bool operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr); +bool operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr); +bool operator<(const IPRange &range1, const IPRange &range2); // LCOV_EXCL_STOP Maybe> extractAddressAndMaskSize(const std::string &cidr); diff --git a/components/include/waap.h b/components/include/waap.h index 4df31ba..02f3983 100755 --- a/components/include/waap.h +++ b/components/include/waap.h @@ -33,6 +33,7 @@ class I_WaapAssetStatesManager; class I_Messaging; class I_AgentDetails; class I_Encryptor; +class I_WaapModelResultLogger; const std::string WAAP_APPLICATION_NAME = "waap application"; @@ -50,7 +51,8 @@ class WaapComponent Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: WaapComponent(); diff --git a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc index 77a3eaf..98fed11 100755 --- a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc @@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec"; parseAppsecJSONKey("name", practice_name, archive_in); - parseAppsecJSONKey("practiceMode", mode, archive_in); + parseAppsecJSONKey("practiceMode", mode, archive_in, "inherited"); + if (valid_modes.count(mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode; + throw PolicyGenException("AppSec Access control practice mode invalid: " + mode); + } parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); parseMandatoryAppsecJSONKey("rateLimit", rate_limit, archive_in); } diff --git a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc index b59ab4c..947b0eb 100755 --- a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc @@ -438,19 +438,30 @@ WebAppSection::WebAppSection( csrf_protection_mode("Disabled"), open_redirect_mode("Disabled"), error_disclosure_mode("Disabled"), + schema_validation_mode("Disabled"), + schema_validation_enforce_level("fullSchema"), practice_advanced_config(parsed_appsec_spec), anti_bots(parsed_appsec_spec.getAntiBot()), trusted_sources({ parsed_trusted_sources }) { + auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence(); + if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) { + dbgWarning(D_LOCAL_POLICY) + << "web attack mitigation severity invalid: " + << mitigation_sevirity; + throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity); + } else { + web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity); + } web_attack_mitigation = web_attack_mitigation_mode != "Disabled"; web_attack_mitigation_severity = web_attack_mitigation_mode != "Prevent" ? "Transparent" : - parsed_appsec_spec.getWebAttacks().getMinimumConfidence(); + web_attack_mitigation_severity; web_attack_mitigation_action = web_attack_mitigation_mode != "Prevent" ? "Transparent" : - web_attack_mitigation_severity == "critical" ? "low" : - web_attack_mitigation_severity == "high" ? "balanced" : - web_attack_mitigation_severity == "medium" ? "high" : + web_attack_mitigation_severity == "Critical" ? "Low" : + web_attack_mitigation_severity == "High" ? "Balanced" : + web_attack_mitigation_severity == "Medium" ? "High" : "Error"; triggers.push_back(TriggersInWaapSection(parsed_log_trigger)); @@ -479,6 +490,9 @@ WebAppSection::WebAppSection( const string &_web_attack_mitigation_severity, const string &_web_attack_mitigation_mode, const string &_bot_protection, + const string &_schema_validation_mode, + const string &_schema_validation_enforce_level, + const vector &_schema_validation_oas, const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, @@ -493,19 +507,29 @@ WebAppSection::WebAppSection( practice_id(_practice_id), practice_name(_practice_name), context(_context), - web_attack_mitigation_severity(_web_attack_mitigation_severity), web_attack_mitigation_mode(_web_attack_mitigation_mode), bot_protection(_bot_protection), + schema_validation_mode(_schema_validation_mode), + schema_validation_enforce_level(_schema_validation_enforce_level), + schema_validation_oas(_schema_validation_oas), practice_advanced_config(_practice_advanced_config), anti_bots(_anti_bots), trusted_sources({ parsed_trusted_sources }) { + if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) { + dbgWarning(D_LOCAL_POLICY) + << "web attack mitigation severity invalid: " + << _web_attack_mitigation_severity; + throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity); + } else { + web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity); + } web_attack_mitigation = web_attack_mitigation_mode != "Disabled"; web_attack_mitigation_action = web_attack_mitigation_mode != "Prevent" ? "Transparent" : - web_attack_mitigation_severity == "critical" ? "low" : - web_attack_mitigation_severity == "high" ? "balanced" : - web_attack_mitigation_severity == "medium" ? "high" : + web_attack_mitigation_severity == "Critical" ? "Low" : + web_attack_mitigation_severity == "High" ? "Balanced" : + web_attack_mitigation_severity == "Medium" ? "High" : "Error"; csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode); @@ -516,6 +540,7 @@ WebAppSection::WebAppSection( for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) { overrides.push_back(AppSecOverride(source_ident)); } + } // LCOV_EXCL_STOP @@ -523,35 +548,35 @@ WebAppSection::WebAppSection( void WebAppSection::save(cereal::JSONOutputArchive &out_ar) const { - string disabled_str = "Disabled"; vector empty_list; out_ar( - cereal::make_nvp("context", context), - cereal::make_nvp("webAttackMitigation", web_attack_mitigation), - cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity), - cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action), - cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode), - cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config), - cereal::make_nvp("csrfProtection", csrf_protection_mode), - cereal::make_nvp("openRedirect", open_redirect_mode), - cereal::make_nvp("errorDisclosure", error_disclosure_mode), - cereal::make_nvp("practiceId", practice_id), - cereal::make_nvp("practiceName", practice_name), - cereal::make_nvp("assetId", asset_id), - cereal::make_nvp("assetName", asset_name), - cereal::make_nvp("ruleId", rule_id), - cereal::make_nvp("ruleName", rule_name), - cereal::make_nvp("schemaValidation", false), - cereal::make_nvp("schemaValidation_v2", disabled_str), - cereal::make_nvp("oas", empty_list), - cereal::make_nvp("triggers", triggers), - cereal::make_nvp("applicationUrls", application_urls), - cereal::make_nvp("overrides", overrides), - cereal::make_nvp("trustedSources", trusted_sources), - cereal::make_nvp("waapParameters", empty_list), - cereal::make_nvp("botProtection", false), - cereal::make_nvp("antiBot", anti_bots), - cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect")) + cereal::make_nvp("context", context), + cereal::make_nvp("webAttackMitigation", web_attack_mitigation), + cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity), + cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action), + cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode), + cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config), + cereal::make_nvp("csrfProtection", csrf_protection_mode), + cereal::make_nvp("openRedirect", open_redirect_mode), + cereal::make_nvp("errorDisclosure", error_disclosure_mode), + cereal::make_nvp("practiceId", practice_id), + cereal::make_nvp("practiceName", practice_name), + cereal::make_nvp("assetId", asset_id), + cereal::make_nvp("assetName", asset_name), + cereal::make_nvp("ruleId", rule_id), + cereal::make_nvp("ruleName", rule_name), + cereal::make_nvp("schemaValidation", schema_validation_mode == "Prevent"), + cereal::make_nvp("schemaValidation_v2", schema_validation_mode), + cereal::make_nvp("oas", schema_validation_oas), + cereal::make_nvp("schemaValidationEnforceLevel", schema_validation_enforce_level), + cereal::make_nvp("triggers", triggers), + cereal::make_nvp("applicationUrls", application_urls), + cereal::make_nvp("overrides", overrides), + cereal::make_nvp("trustedSources", trusted_sources), + cereal::make_nvp("waapParameters", empty_list), + cereal::make_nvp("botProtection", false), + cereal::make_nvp("antiBot", anti_bots), + cereal::make_nvp("botProtection_v2", bot_protection != "" ? bot_protection : string("Detect")) ); } diff --git a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h index b03b262..f45b0eb 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h @@ -291,6 +291,9 @@ public: const std::string &_web_attack_mitigation_severity, const std::string &_web_attack_mitigation_mode, const std::string &_bot_protection, + const std::string &schema_validation_mode, + const std::string &schema_validation_enforce_level, + const std::vector &schema_validation_oas, const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, @@ -302,27 +305,30 @@ public: bool operator< (const WebAppSection &other) const; private: - std::string application_urls; - std::string asset_id; - std::string asset_name; - std::string rule_id; - std::string rule_name; - std::string practice_id; - std::string practice_name; - std::string context; - std::string web_attack_mitigation_action; - std::string web_attack_mitigation_severity; - std::string web_attack_mitigation_mode; - std::string csrf_protection_mode; - std::string open_redirect_mode; - std::string error_disclosure_mode; - std::string bot_protection; - bool web_attack_mitigation; - std::vector triggers; - PracticeAdvancedConfig practice_advanced_config; - AppsecPracticeAntiBotSection anti_bots; - std::vector trusted_sources; - std::vector overrides; + bool web_attack_mitigation; + std::string application_urls; + std::string asset_id; + std::string asset_name; + std::string rule_id; + std::string rule_name; + std::string practice_id; + std::string practice_name; + std::string context; + std::string web_attack_mitigation_action; + std::string web_attack_mitigation_severity; + std::string web_attack_mitigation_mode; + std::string csrf_protection_mode; + std::string open_redirect_mode; + std::string error_disclosure_mode; + std::string bot_protection; + std::string schema_validation_mode; + std::string schema_validation_enforce_level; + std::vector schema_validation_oas; + PracticeAdvancedConfig practice_advanced_config; + AppsecPracticeAntiBotSection anti_bots; + std::vector overrides; + std::vector trusted_sources; + std::vector triggers; }; class WebAPISection @@ -410,7 +416,7 @@ class ParsedRule { public: ParsedRule() {} - ParsedRule(const std::string &_host) : host(_host) {} + ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {} void load(cereal::JSONInputArchive &archive_in); const std::vector & getExceptions() const; diff --git a/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h b/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h index 32a579f..5284942 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h +++ b/components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h @@ -24,6 +24,7 @@ #include "maybe_res.h" #include "i_orchestration_tools.h" #include "i_shell_cmd.h" +#include "i_encryptor.h" #include "i_messaging.h" #include "i_env_details.h" #include "i_agent_details.h" @@ -40,6 +41,7 @@ class K8sPolicyUtils Singleton::Consume, Singleton::Consume, Singleton::Consume, + Singleton::Consume, Singleton::Consume { public: @@ -80,6 +82,8 @@ private: void createSnortFile(std::vector &practices) const; + void createSchemaValidationOas(std::vector &practices) const; + template std::vector extractV1Beta2ElementsFromCluster( const std::string &crd_plural, @@ -112,6 +116,7 @@ private: I_Messaging* messaging = nullptr; EnvType env_type; std::string token; + std::string agent_ns; }; #endif // __K8S_POLICY_UTILS_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h index 612687e..02c4e6b 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h +++ b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h @@ -49,6 +49,13 @@ static const std::unordered_map string_to_trigger_type { "WebUserResponse", TriggerType::WebUserResponse } }; +static const std::unordered_map key_to_mitigation_severity = { + { "high", "High"}, + { "medium", "Medium"}, + { "critical", "Critical"}, + { "Transparent", "Transparent"} +}; + static const std::unordered_map key_to_practices_val = { { "prevent-learn", "Prevent"}, { "detect-learn", "Learn"}, @@ -57,6 +64,14 @@ static const std::unordered_map key_to_practices_val = { "inactive", "Inactive"} }; +static const std::unordered_map key_to_practices_mode_val = { + { "prevent-learn", "Prevent"}, + { "detect-learn", "Detect"}, + { "prevent", "Prevent"}, + { "detect", "Detect"}, + { "inactive", "Disabled"} +}; + static const std::unordered_map key_to_practices_val2 = { { "prevent-learn", "Prevent"}, { "detect-learn", "Learn"}, @@ -66,6 +81,8 @@ static const std::unordered_map key_to_practices_val2 }; static const std::string default_appsec_url = "http://*:*"; +static const std::string default_appsec_name = "Any"; + class PolicyGenException : public std::exception { diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h b/components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h index 6d868f9..424e1fb 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h @@ -31,7 +31,7 @@ class NewParsedRule { public: NewParsedRule() {} - NewParsedRule(const std::string &_host) : host(_host) {} + NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {} void load(cereal::JSONInputArchive &archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h index 84ca770..c143172 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h @@ -23,6 +23,8 @@ #include "config.h" #include "debug.h" #include "local_policy_common.h" +#include "i_orchestration_tools.h" +#include "i_encryptor.h" bool isModeInherited(const std::string &mode); @@ -88,6 +90,8 @@ public: void save(cereal::JSONOutputArchive &out_ar) const; + bool operator<(const IpsProtectionsSection &other) const; + private: std::string context; std::string name; @@ -105,7 +109,7 @@ public: // LCOV_EXCL_START Reason: no test exist IPSSection() {}; - IPSSection(const std::vector &_ips) : ips(_ips) {}; + IPSSection(const std::vector &_ips); // LCOV_EXCL_STOP void save(cereal::JSONOutputArchive &out_ar) const; @@ -138,6 +142,12 @@ public: const std::string & getMode(const std::string &default_mode = "inactive") const; private: + + const std::string & getRulesMode( + const std::string &mode, + const std::string &default_mode = "inactive" + ) const; + std::string override_mode; std::string max_performance_impact; std::string min_severity_level; @@ -487,15 +497,16 @@ private: SnortSection snort; }; -class NewSnortSignaturesAndOpenSchemaAPI +class NewSnortSignatures { public: - NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {}; + NewSnortSignatures() : is_temporary(false) {}; void load(cereal::JSONInputArchive &archive_in); void addFile(const std::string &file_name); const std::string & getOverrideMode(const std::string &default_mode = "inactive") const; + const std::string & getEnforceLevel() const; const std::vector & getConfigMap() const; const std::vector & getFiles() const; bool isTemporary() const; @@ -503,17 +514,40 @@ public: private: std::string override_mode; + std::string enforcement_level; std::vector config_map; std::vector files; bool is_temporary; }; +class NewOpenApiSchema : Singleton::Consume, Singleton::Consume +{ +public: + NewOpenApiSchema() {}; + + void load(cereal::JSONInputArchive &archive_in); + + void addOas(const std::string &file); + const std::string & getOverrideMode(const std::string &default_mode = "inactive") const; + const std::string & getEnforceLevel() const; + const std::vector & getConfigMap() const; + const std::vector & getFiles() const; + const std::vector & getOas() const; + +private: + std::string override_mode; + std::string enforcement_level; + std::vector config_map; + std::vector files; + std::vector oas; +}; + class NewAppSecPracticeAntiBot { public: const std::vector & getIjectedUris() const; const std::vector & getValidatedUris() const; - const std::string & getMode() const; + const std::string & getMode(const std::string &default_mode = "inactive") const; void load(cereal::JSONInputArchive &archive_in); void save(cereal::JSONOutputArchive &out_ar) const; @@ -569,8 +603,8 @@ class NewAppSecPracticeSpec public: void load(cereal::JSONInputArchive &archive_in); - NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures(); - const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const; + NewSnortSignatures & getSnortSignatures(); + NewOpenApiSchema & getOpenSchemaValidation(); const NewAppSecPracticeWebAttacks & getWebAttacks() const; const NewAppSecPracticeAntiBot & getAntiBot() const; const NewIntrusionPrevention & getIntrusionPrevention() const; @@ -583,8 +617,8 @@ public: private: NewFileSecurity file_security; NewIntrusionPrevention intrusion_prevention; - NewSnortSignaturesAndOpenSchemaAPI openapi_schema_validation; - NewSnortSignaturesAndOpenSchemaAPI snort_signatures; + NewOpenApiSchema openapi_schema_validation; + NewSnortSignatures snort_signatures; NewAppSecPracticeWebAttacks web_attacks; NewAppSecPracticeAntiBot anti_bot; std::string appsec_class_name; diff --git a/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h b/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h index 98bac38..62c0851 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h @@ -123,6 +123,7 @@ public: ); const std::string & getIdentifier() const; + const std::string & getIdentifierValue() const; void save(cereal::JSONOutputArchive &out_ar) const; @@ -145,6 +146,7 @@ public: ); const std::string & getIdentifier() const; + const std::string & getIdentifierValue() const; void save(cereal::JSONOutputArchive &out_ar) const; diff --git a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc index 8a4db38..01ee9a2 100644 --- a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc @@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key) } } +string +getAppSecScopeType() +{ + auto env_res = getenv("CRDS_SCOPE"); + if (env_res != nullptr) return env_res; + return "cluster"; +} + void K8sPolicyUtils::init() { @@ -42,6 +50,7 @@ K8sPolicyUtils::init() env_type = env_details->getEnvType(); if (env_type == EnvType::K8S) { token = env_details->getToken(); + agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : ""; messaging = Singleton::Consume::by(); } } @@ -140,10 +149,12 @@ extractElementsFromNewRule( const NewParsedRule &rule, map> &policy_elements_names) { - policy_elements_names[AnnotationTypes::EXCEPTION].insert( - rule.getExceptions().begin(), - rule.getExceptions().end() - ); + if (rule.getExceptions().size() > 0) { + policy_elements_names[AnnotationTypes::EXCEPTION].insert( + rule.getExceptions().begin(), + rule.getExceptions().end() + ); + } policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert( rule.getPractices().begin(), rule.getPractices().end() @@ -152,14 +163,24 @@ extractElementsFromNewRule( rule.getAccessControlPractices().begin(), rule.getAccessControlPractices().end() ); - policy_elements_names[AnnotationTypes::TRIGGER].insert( - rule.getLogTriggers().begin(), - rule.getLogTriggers().end() - ); - policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse()); - policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers()); - policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources()); - policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings()); + if (rule.getLogTriggers().size() > 0) { + policy_elements_names[AnnotationTypes::TRIGGER].insert( + rule.getLogTriggers().begin(), + rule.getLogTriggers().end() + ); + } + if (rule.getCustomResponse() != "" ) { + policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse()); + } + if (rule.getSourceIdentifiers() != "" ) { + policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers()); + } + if (rule.getTrustedSources() != "" ) { + policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources()); + } + if (rule.getUpgradeSettings() != "" ) { + policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings()); + } } map> @@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster( dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural; vector elements; for (const string &element_name : elements_names) { + string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : ""; + string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : ""; dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name; auto maybe_appsec_element = getObjectFromCluster>( - "/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name + "/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name ); if (!maybe_appsec_element.ok()) { @@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector &practices) const practice.getSnortSignatures().setTemporary(true); for (const string &config_map : practice.getSnortSignatures().getConfigMap()) { + string ns = agent_ns == "" ? "default/" : agent_ns; auto maybe_configmap = getObjectFromCluster( - "/api/v1/namespaces/default/configmaps/" + config_map + "/api/v1/namespaces/" + ns + "configmaps/" + config_map ); if (!maybe_configmap.ok()) { dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster."; @@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector &practices) const } } +void +K8sPolicyUtils::createSchemaValidationOas(vector &practices) const +{ + for (NewAppSecPracticeSpec &practice : practices) { + vector res; + for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap()) + { + string ns = agent_ns == "" ? "default/" : agent_ns; + auto maybe_configmap = getObjectFromCluster( + "/api/v1/namespaces/" + ns + "configmaps/" + config_map + ); + if (!maybe_configmap.ok()) { + dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster."; + continue; + } + string file_content = maybe_configmap.unpack().getFileContent(); + string res = Singleton::Consume::by()->base64Encode(file_content); + practice.getOpenSchemaValidation().addOas(res); + } + } +} + Maybe K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds( const AppsecSpecParser &appsec_policy_spec, @@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds( } if (default_rule.getMode().empty() && !ingress_mode.empty()) { + dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode; default_rule.setMode(ingress_mode); } @@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds( ); createSnortFile(threat_prevention_practices); + createSchemaValidationOas(threat_prevention_practices); vector access_control_practices = extractV1Beta2ElementsFromCluster( @@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr(); dbgWarning(D_LOCAL_POLICY ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2"; + string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : ""; + string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : ""; auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster>( - "/apis/openappsec.io/v1beta2/policies/" + policy_name + "/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name ); + if (!maybe_v1beta2_appsec_policy_spec.ok()) { dbgWarning(D_LOCAL_POLICY) << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr(); @@ -535,10 +586,11 @@ K8sPolicyUtils::createPolicy( if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) { policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy; } + auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode(); if (item.getSpec().doesDefaultBackendExist()) { dbgTrace(D_LOCAL_POLICY) << "Inserting Any host rule to the specific asset set"; - K ingress_rule = K("*"); + K ingress_rule = K("*", default_mode); policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule); } @@ -556,12 +608,11 @@ K8sPolicyUtils::createPolicy( << "' uri: '" << uri.getPath() << "'"; - K ingress_rule = K(host); + K ingress_rule = K(host, default_mode); policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule); } } } - } std::tuple, map> diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index 6255bb0..f0179d7 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -22,6 +22,7 @@ static const set performance_impacts = {"low", "medium", "high"}; static const set severity_levels = {"low", "medium", "high", "critical"}; static const set size_unit = {"bytes", "KB", "MB", "GB"}; static const set confidences_actions = {"prevent", "detect", "inactive", "as-top-level", "inherited"}; +static const set valied_enforcement_level = {"fullSchema", "endpointOnly"}; static const set valid_modes = { "prevent", "detect", @@ -32,38 +33,38 @@ static const set valid_modes = { "inherited" }; static const set valid_confidences = {"medium", "high", "critical"}; -static const std::unordered_map key_to_performance_impact_val = { +static const unordered_map key_to_performance_impact_val = { { "low", "Low or lower"}, { "medium", "Medium or lower"}, { "high", "High or lower"} }; -static const std::unordered_map key_to_severity_level_val = { +static const unordered_map key_to_severity_level_val = { { "low", "Low or above"}, { "medium", "Medium or above"}, { "high", "High or above"}, { "critical", "Critical"} }; -static const std::unordered_map key_to_mode_val = { +static const unordered_map key_to_mode_val = { { "prevent-learn", "Prevent"}, { "detect-learn", "Detect"}, { "prevent", "Prevent"}, { "detect", "Detect"}, { "inactive", "Inactive"} }; -static const std::unordered_map anti_bot_key_to_mode_val = { +static const unordered_map anti_bot_key_to_mode_val = { { "prevent-learn", "Prevent"}, { "detect-learn", "Detect"}, { "prevent", "Prevent"}, { "detect", "Detect"}, { "inactive", "Disabled"} }; -static const std::unordered_map unit_to_int = { +static const unordered_map unit_to_int = { { "bytes", 1}, { "KB", 1024}, { "MB", 1048576}, { "GB", 1073741824} }; -static const std::string TRANSPARENT_MODE = "Transparent"; +static const string TRANSPARENT_MODE = "Transparent"; bool isModeInherited(const string &mode) @@ -71,11 +72,11 @@ isModeInherited(const string &mode) return mode == "as-top-level" || mode == "inherited"; } -const std::string & +const string & getModeWithDefault( - const std::string &mode, - const std::string &default_mode, - const std::unordered_map &key_to_val) + const string &mode, + const string &default_mode, + const unordered_map &key_to_val) { if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) { dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode; @@ -88,36 +89,35 @@ getModeWithDefault( return key_to_val.at(mode); } -const std::vector & +const vector & NewAppSecPracticeAntiBot::getIjectedUris() const { return injected_uris; } -const std::vector & +const vector & NewAppSecPracticeAntiBot::getValidatedUris() const { return validated_uris; } -const std::string & -NewAppSecPracticeAntiBot::getMode() const +const string & +NewAppSecPracticeAntiBot::getMode(const string &default_mode) const { - return override_mode; + return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val); } void NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots"; - string mode; parseAppsecJSONKey>("injectedUris", injected_uris, archive_in); parseAppsecJSONKey>("validatedUris", validated_uris, archive_in); - parseMandatoryAppsecJSONKey("overrideMode", mode, archive_in, "inactive"); - if (valid_modes.count(mode) == 0) { - dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << mode; + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); + if (valid_modes.count(override_mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode; + throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode); } - override_mode = anti_bot_key_to_mode_val.at(mode); } void @@ -242,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const } SnortProtectionsSection::SnortProtectionsSection( - const std::string &_context, - const std::string &_asset_name, - const std::string &_asset_id, - const std::string &_practice_name, - const std::string &_practice_id, - const std::string &_source_identifier, - const std::string &_mode, - const std::vector &_files) + const string &_context, + const string &_asset_name, + const string &_asset_id, + const string &_practice_name, + const string &_practice_id, + const string &_source_identifier, + const string &_mode, + const vector &_files) : context(_context), asset_name(_asset_name), @@ -278,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const } DetectionRules::DetectionRules( - const std::string &_type, - const std::string &_SSM, - const std::string &_keywords, - const std::vector &_context) + const string &_type, + const string &_SSM, + const string &_keywords, + const vector &_context) : type(_type), SSM(_SSM), @@ -314,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const ProtectionMetadata::ProtectionMetadata( bool _silent, - const std::string &_protection_name, - const std::string &_severity, - const std::string &_confidence_level, - const std::string &_performance_impact, - const std::string &_last_update, - const std::string &_maintrain_id, - const std::vector &_tags, - const std::vector &_cve_list) + const string &_protection_name, + const string &_severity, + const string &_confidence_level, + const string &_performance_impact, + const string &_last_update, + const string &_maintrain_id, + const vector &_tags, + const vector &_cve_list) : silent(_silent), protection_name(_protection_name), @@ -394,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const } ProtectionsSection::ProtectionsSection( - const std::vector &_protections, - const std::string &_name, - const std::string &_modification_time) + const vector &_protections, + const string &_name, + const string &_modification_time) : protections(_protections), name(_name), @@ -460,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const } void -NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in) +NewSnortSignatures::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice"; parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); parseAppsecJSONKey>("configmap", config_map, archive_in); parseAppsecJSONKey>("files", files, archive_in); + if (valid_modes.count(override_mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode; + throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode); + } is_temporary = false; if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode; @@ -474,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in) } void -NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name) +NewSnortSignatures::addFile(const string &file_name) { files.push_back(file_name); } const string & -NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const +NewSnortSignatures::getOverrideMode(const string &default_mode) const { - const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val); + const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val); return res; } const vector & -NewSnortSignaturesAndOpenSchemaAPI::getFiles() const +NewSnortSignatures::getFiles() const { return files; } const vector & -NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const +NewSnortSignatures::getConfigMap() const { return config_map; } bool -NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const +NewSnortSignatures::isTemporary() const { return is_temporary; } void -NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val) +NewSnortSignatures::setTemporary(bool val) { is_temporary = val; } +void +NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in) +{ + dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice"; + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); + parseAppsecJSONKey>("configmap", config_map, archive_in); + parseAppsecJSONKey>("files", files, archive_in); + parseAppsecJSONKey("enforcementLevel", enforcement_level, archive_in, "fullSchema"); + if (valied_enforcement_level.count(enforcement_level) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level; + throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level); + } + if (valid_modes.count(override_mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode; + throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode); + } + for (const string &file : files) + { + auto i_orchestration_tools = Singleton::Consume::by(); + auto file_content = i_orchestration_tools->readFile(file); + if (!file_content.ok()) { + dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file"; + continue; + } + oas.push_back(Singleton::Consume::by()->base64Encode(file_content.unpack())); + } +} + +void +NewOpenApiSchema::addOas(const string &file) +{ + oas.push_back(file); +} + +const string & +NewOpenApiSchema::getOverrideMode(const string &default_mode) const +{ + const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2); + return res; +} + +const string & +NewOpenApiSchema::getEnforceLevel() const +{ + return enforcement_level; +} + +const vector & +NewOpenApiSchema::getFiles() const +{ + return files; +} + +const vector & +NewOpenApiSchema::getConfigMap() const +{ + return config_map; +} + +const vector & +NewOpenApiSchema::getOas() const +{ + return oas; +} + void IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const { @@ -548,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection( { } -std::string & +string & IpsProtectionsSection::getMode() { return mode; @@ -570,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const ); } +bool +IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const +{ + // for sorting from the most specific to the least specific rule + if (name == default_appsec_name) return false; + if (other.name == default_appsec_name) return true; + return name.size() > other.name.size(); +} + +IPSSection::IPSSection(const vector &_ips) : ips(_ips) +{ + sort(ips.begin(), ips.end()); +} + void IPSSection::save(cereal::JSONOutputArchive &out_ar) const { @@ -648,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const vector ips_rules; IpsProtectionsRulesSection high_rule( min_cve_Year, - getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val), + getRulesMode(high_confidence_event_action, default_mode), string("High"), max_performance_impact, string(""), @@ -658,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const IpsProtectionsRulesSection med_rule( min_cve_Year, - getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val), + getRulesMode(medium_confidence_event_action, default_mode), string("Medium"), max_performance_impact, string(""), @@ -668,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const IpsProtectionsRulesSection low_rule( min_cve_Year, - getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val), + getRulesMode(low_confidence_event_action, default_mode), string("Low"), max_performance_impact, string(""), @@ -679,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const return ips_rules; } -const std::string & -NewIntrusionPrevention::getMode(const std::string &default_mode) const +const string & +NewIntrusionPrevention::getMode(const string &default_mode) const { - const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val); + const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val); return res; } +const string & +NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const +{ + if (isModeInherited(mode)) return default_mode; + + if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) { + dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid."; + return key_to_practices_mode_val.at("inactive"); + } + return key_to_practices_mode_val.at(mode); +} + FileSecurityProtectionsSection::FileSecurityProtectionsSection( uint64_t _file_size_limit, uint64_t _archive_file_size_limit, bool _allow_files_without_name, bool _required_file_size_limit, bool _required_archive_extraction, - const std::string &_context, - const std::string &_name, - const std::string &_asset_id, - const std::string &_practice_name, - const std::string &_practice_id, - const std::string &_action, - const std::string &_files_without_name_action, - const std::string &_high_confidence_action, - const std::string &_medium_confidence_action, - const std::string &_low_confidence_action, - const std::string &_severity_level, - const std::string &_file_size_limit_action, - const std::string &_multi_level_archive_action, - const std::string &_unopened_archive_action) + const string &_context, + const string &_name, + const string &_asset_id, + const string &_practice_name, + const string &_practice_id, + const string &_action, + const string &_files_without_name_action, + const string &_high_confidence_action, + const string &_medium_confidence_action, + const string &_low_confidence_action, + const string &_severity_level, + const string &_file_size_limit_action, + const string &_multi_level_archive_action, + const string &_unopened_archive_action) : file_size_limit(_file_size_limit), archive_file_size_limit(_archive_file_size_limit), @@ -831,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const return extract_archive_files; } -const std::string & +const string & NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const { return archived_files_within_archived_files; } -const std::string & +const string & NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const { return archived_files_where_content_extraction_failed; @@ -886,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const return (file_size_limit * unit_to_int.at(file_size_limit_unit)); } -const std::string & +const string & NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const { return files_exceeding_size_limit_action; @@ -1007,7 +1102,7 @@ void NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec"; - parseAppsecJSONKey( + parseAppsecJSONKey( "schemaValidation", openapi_schema_validation, archive_in @@ -1015,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); parseMandatoryAppsecJSONKey("fileSecurity", file_security, archive_in); parseMandatoryAppsecJSONKey("intrusionPrevention", intrusion_prevention, archive_in); - parseMandatoryAppsecJSONKey("snortSignatures", snort_signatures, archive_in); + parseMandatoryAppsecJSONKey("snortSignatures", snort_signatures, archive_in); parseMandatoryAppsecJSONKey("webAttacks", web_attacks, archive_in); parseAppsecJSONKey("antiBot", anti_bot, archive_in); parseAppsecJSONKey("name", practice_name, archive_in); parseAppsecJSONKey("practiceMode", mode, archive_in, "inherited"); + if (valid_modes.count(mode) == 0) { + dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode; + throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode); + } } void @@ -1028,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name) practice_name = _name; } -const NewSnortSignaturesAndOpenSchemaAPI & -NewAppSecPracticeSpec::getOpenSchemaValidation() const +NewOpenApiSchema & +NewAppSecPracticeSpec::getOpenSchemaValidation() { return openapi_schema_validation; } -NewSnortSignaturesAndOpenSchemaAPI & +NewSnortSignatures & NewAppSecPracticeSpec::getSnortSignatures() { return snort_signatures; diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index 493de1c..7dcf984 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -23,6 +23,14 @@ using namespace std; USE_DEBUG_FLAG(D_NGINX_POLICY); USE_DEBUG_FLAG(D_LOCAL_POLICY); +static const std::unordered_map key_to_source_identefier_val = { + { "sourceip", "Source IP"}, + { "cookie", "Cookie:"}, + { "headerkey", "Header:"}, + { "JWTKey", ""}, + { "x-forwarded-for", "X-Forwarded-For"} +}; + void SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const { @@ -1038,7 +1046,7 @@ PolicyMakerUtils::createIpsSections( practice_name, practice_id, source_identifier, - override_mode, + "Inactive", apssec_practice.getIntrusionPrevention().createIpsRules(override_mode) ); @@ -1048,8 +1056,7 @@ PolicyMakerUtils::createIpsSections( void PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary) { - auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name; - string in_file = is_temporary ? path + ".rule" : path; + auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name; if (snort_protections.find(path) != snort_protections.end()) { dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists"; @@ -1060,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_ << (is_temporary ? " temporary" : "") << " file " << path; auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py"; - auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err"; + auto tmp_out = "/tmp/" + file_name + ".out"; + auto tmp_err = "/tmp/" + file_name + ".err"; + auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err; auto res = Singleton::Consume::by()->getExecOutput(cmd); @@ -1069,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_ return; } - Maybe maybe_protections = openFileAsJson(path + ".out"); + Maybe maybe_protections = openFileAsJson(tmp_out); if (!maybe_protections.ok()){ dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr(); return; } auto i_orchestration_tools = Singleton::Consume::by(); - if (is_temporary) i_orchestration_tools->removeFile(in_file); - i_orchestration_tools->removeFile(path + ".out"); - i_orchestration_tools->removeFile(path + ".err"); + if (is_temporary) i_orchestration_tools->removeFile(path); + i_orchestration_tools->removeFile(tmp_out); + i_orchestration_tools->removeFile(tmp_err); snort_protections[path] = ProtectionsSection( maybe_protections.unpack().getProtections(), @@ -1208,7 +1217,8 @@ void PolicyMakerUtils::createWebAppSection( const V1beta2AppsecLinuxPolicy &policy, const RulesConfigRulebase& rule_config, - const string &practice_id, const string &full_url, + const string &practice_id, + const string &full_url, const string &default_mode, map &rule_annotations) { @@ -1225,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection( apssec_practice.getWebAttacks().getMaxObjectDepth(), apssec_practice.getWebAttacks().getMaxUrlSizeBytes() ); + WebAppSection web_app = WebAppSection( full_url == "Any" ? default_appsec_url : full_url, rule_config.getAssetId(), @@ -1236,7 +1247,10 @@ PolicyMakerUtils::createWebAppSection( rule_config.getContext(), apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode), apssec_practice.getWebAttacks().getMode(practice_mode), - apssec_practice.getAntiBot().getMode(), + apssec_practice.getAntiBot().getMode(practice_mode), + apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode), + apssec_practice.getOpenSchemaValidation().getEnforceLevel(), + apssec_practice.getOpenSchemaValidation().getOas(), practice_advance_config, apssec_practice.getAntiBot(), log_triggers[rule_annotations[AnnotationTypes::TRIGGER]], @@ -1290,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( ); rules_config[rule_config.getAssetName()] = rule_config; - string current_identifier; + string current_identifier, current_identifier_value; if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) { UsersIdentifiersRulebase user_identifiers = createUserIdentifiers( rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS], @@ -1299,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( ); users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers; current_identifier = user_identifiers.getIdentifier(); + current_identifier_value = user_identifiers.getIdentifierValue(); + } + + string ips_identifier, ips_identifier_value; + if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) { + ips_identifier = key_to_source_identefier_val.at(current_identifier); + } + if (current_identifier == "cookie" || current_identifier == "headerkey") { + ips_identifier_value = current_identifier_value; } createIpsSections( @@ -1306,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( rule_config.getAssetName(), practice_id, rule_annotations[AnnotationTypes::PRACTICE], - current_identifier, + ips_identifier + ips_identifier_value, rule_config.getContext(), policy, rule_annotations, diff --git a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc index c001620..67c938d 100755 --- a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc @@ -17,6 +17,8 @@ using namespace std; USE_DEBUG_FLAG(D_LOCAL_POLICY); +static const string empty_string=""; + AssetUrlParser AssetUrlParser::parse(const string &uri) { @@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const { return source_identifier; } + +const string & +UsersIdentifier::getIdentifierValue() const +{ + if (identifier_values.empty()) return empty_string; + return identifier_values[0]; +} // LCOV_EXCL_STOP void @@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const if (source_identifiers.empty()) return source_identifier; return source_identifiers[0].getIdentifier(); } + +const string & +UsersIdentifiersRulebase::getIdentifierValue() const +{ + if (source_identifiers.empty()) return empty_string; + return source_identifiers[0].getIdentifierValue(); +} // LCOV_EXCL_STOP void diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 3558abe..1c3a411 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -66,7 +66,12 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=;" "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');", getQUID) SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] " - "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''", + "&& python3 /opt/CPquid/Quid_Api.py -i " + "/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''", + getQUID) +SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] " + "&& python3 /opt/CPquid/Quid_Api.py -i " + "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''", getQUID) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( diff --git a/components/security_apps/orchestration/env_details/env_details.cc b/components/security_apps/orchestration/env_details/env_details.cc index a33fa56..0f65330 100644 --- a/components/security_apps/orchestration/env_details/env_details.cc +++ b/components/security_apps/orchestration/env_details/env_details.cc @@ -28,6 +28,7 @@ EnvDetails::EnvDetails() : env_type(EnvType::LINUX) auto tools = Singleton::Consume::from(); if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER; token = retrieveToken(); + agent_namespace = retrieveNamespace(); if (!token.empty()) { auto env_res = getenv("deployment_type"); env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S; @@ -46,12 +47,24 @@ EnvDetails::getToken() return token; } +string +EnvDetails::getNameSpace() +{ + return agent_namespace; +} + string EnvDetails::retrieveToken() { return readFileContent(k8s_service_account + "/token"); } +string +EnvDetails::retrieveNamespace() +{ + return readFileContent(k8s_service_account + "/namespace"); +} + string EnvDetails::readFileContent(const string &file_path) { diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 45e53cc..2290d6e 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1485,11 +1485,10 @@ private: } void - setUpgradeTime() + setDelayedUpgradeTime() { if (getConfigurationFlag("service_startup") != "true") return; - if (i_service_controller->getServiceToPortMap().empty()) return; - + if (!i_agent_details->isOpenAppsecAgent() && i_service_controller->getServiceToPortMap().empty()) return; try { string upgrade_delay_interval_str = getAttribute("no-setting", "UPGRADE_DELAY_INTERVAL_MIN"); int upgrade_delay_interval = upgrade_delay_interval_str != "" ? stoi(upgrade_delay_interval_str) : 30; @@ -1506,6 +1505,7 @@ private: void run() { + loadExistingPolicy(); sleep_interval = policy.getErrorSleepInterval(); Maybe registration_status(genError("Not running yet.")); while (!(registration_status = registerToTheFog()).ok()) { @@ -1530,7 +1530,6 @@ private: << " seconds"; Singleton::Consume::by()->yield(seconds(sleep_interval)); } - loadExistingPolicy(); failure_count = 0; Singleton::Consume::by()->yield(chrono::seconds(1)); @@ -1587,7 +1586,7 @@ private: ).notify(); } - setUpgradeTime(); + setDelayedUpgradeTime(); while (true) { Singleton::Consume::by()->startNewTrace(false); if (shouldReportAgentDetailsMetadata()) { diff --git a/components/security_apps/waap/include/i_transaction.h b/components/security_apps/waap/include/i_transaction.h index 2c727c1..a2f3da8 100755 --- a/components/security_apps/waap/include/i_transaction.h +++ b/components/security_apps/waap/include/i_transaction.h @@ -70,6 +70,7 @@ public: virtual const std::string getParam() const = 0; virtual const std::vector getKeywordMatches() const = 0; virtual const std::vector getKeywordsCombinations() const = 0; + virtual const std::vector getKeywordsAfterFilter() const = 0; virtual const std::string getContentTypeStr() const = 0; virtual Waap::Util::ContentType getContentType() const = 0; virtual const std::string getKeywordMatchesStr() const = 0; @@ -84,6 +85,7 @@ public: virtual const std::string getUriStr() const = 0; virtual const std::string& getSourceIdentifier() const = 0; virtual double getScore() const = 0; + virtual double getOtherModelScore() const = 0; virtual const std::vector getScoreArray() const = 0; virtual Waap::CSRF::State& getCsrfState() = 0; virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0; diff --git a/components/security_apps/waap/waap_clib/CidrMatch.cc b/components/security_apps/waap/waap_clib/CidrMatch.cc index 33af165..18b7cf0 100755 --- a/components/security_apps/waap/waap_clib/CidrMatch.cc +++ b/components/security_apps/waap/waap_clib/CidrMatch.cc @@ -25,6 +25,51 @@ #include "log_generator.h" #include +static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) { + in_addr maskedAddr; + if (prefixLength == 0) { + maskedAddr.s_addr = 0; + } else { + uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1)); // Create mask + maskedAddr.s_addr = addr.s_addr & mask; // Apply mask + } + return maskedAddr; +} + +// Function to apply a network mask to an IPv6 address +static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) { + in6_addr maskedAddr = addr; + int fullBytes = prefixLength / 8; + int remainingBits = prefixLength % 8; + + // Mask full bytes + for (int i = fullBytes; i < 16; ++i) { + maskedAddr.s6_addr[i] = 0; + } + + // Mask remaining bits + if (remainingBits > 0) { + uint8_t mask = ~((1 << (8 - remainingBits)) - 1); + maskedAddr.s6_addr[fullBytes] &= mask; + } + + return maskedAddr; +} + +// Helper function to convert an IPv4 address to string +static std::string ipv4ToString(const in_addr& ipv4) { + char str[INET_ADDRSTRLEN]; + inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN); + return std::string(str); +} + +// Helper function to convert an IPv6 address to string +static std::string ipv6ToString(const in6_addr& ipv6) { + char str[INET6_ADDRSTRLEN]; + inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN); + return std::string(str); +} + USE_DEBUG_FLAG(D_WAAP); namespace Waap { namespace Util { @@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const { isIPV6 == other.isIPV6; } +bool CIDRData::operator<(const CIDRData &other) const { + if (isIPV6) { + if (!other.isIPV6) return false; + return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0; + } + if (other.isIPV6) return true; + return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr); +} + bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) { if (bits == 0) { // C99 6.5.7 (3): u32 << 32 is undefined behaviour @@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr) memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr)); if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) { + cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits); cidr.isIPV6 = false; } else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) { + cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits); cidr.isIPV6 = true; } else @@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr) return true; } + bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) { CIDRData cidrData; @@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) { return cidrMatch(sourceip, cidrData); } + bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){ struct in_addr source_inaddr; struct in6_addr source_inaddr6; @@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){ dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions."; return false; } + +bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) { + if (first.isIPV6 != second.isIPV6) return false; // IPv4 and IPv6 cannot overlap + if (first.networkBits >= second.networkBits) return false; + + if (!first.isIPV6) { + // IPv4 containment check + in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits); + return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr); + } + // IPv6 containment check + in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits); + + for (int i = 0; i < 16; ++i) { + if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) { + return false; + } + } + return true; +} + +std::string cidrsToString(const std::vector& cidrs) { + std::stringstream ss; + bool is_first = true; + ss << "["; + for (const auto& cidr : cidrs) { + if (!is_first) ss << ", "; + if (cidr.isIPV6) { + ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast(cidr.networkBits); + } else { + ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast(cidr.networkBits); + } + is_first = false; + } + ss << "]"; + return ss.str(); +} + } } diff --git a/components/security_apps/waap/waap_clib/CidrMatch.h b/components/security_apps/waap/waap_clib/CidrMatch.h index 546fd12..9f265bc 100755 --- a/components/security_apps/waap/waap_clib/CidrMatch.h +++ b/components/security_apps/waap/waap_clib/CidrMatch.h @@ -18,6 +18,7 @@ #include #include #include +#include namespace Waap { namespace Util { @@ -29,11 +30,14 @@ struct CIDRData { uint8_t networkBits; bool isIPV6; bool operator==(const CIDRData &other) const; + bool operator<(const CIDRData &other) const; }; bool isCIDR(const std::string& strCIDR, CIDRData& cidr); bool cidrMatch(const std::string& sourceip, const CIDRData& cidr); bool cidrMatch(const std::string &sourceip, const std::string &target); +bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second); +std::string cidrsToString(const std::vector& cidrs); } } diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index 9a10533..f13c820 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -1308,7 +1308,7 @@ DeepParser::createInternalParser( *this, parser_depth + 1, '&', - valueStats.isUrlEncoded) + valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val)) ); } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) { dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)"; @@ -1323,7 +1323,7 @@ DeepParser::createInternalParser( *this, parser_depth + 1, '&', - valueStats.isUrlEncoded) + valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val)) ); offset = 0; return offset; @@ -1545,5 +1545,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const static const std::string PDF_header("%PDF-"); if (cur_val.size() < 10) return false; - return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos; + return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size()) + .find(PDF_header) != std::string::npos; } diff --git a/components/security_apps/waap/waap_clib/ParserBinaryFile.cc b/components/security_apps/waap/waap_clib/ParserBinaryFile.cc index 5188e78..6379785 100644 --- a/components/security_apps/waap/waap_clib/ParserBinaryFile.cc +++ b/components/security_apps/waap/waap_clib/ParserBinaryFile.cc @@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len) } } else { dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail; - c = strstr(buf + len - tail.size(), tail.c_str()); + size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0; + c = strstr(buf + tail_lookup_offset, tail.c_str()); dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c; if (c) { m_state = s_end; diff --git a/components/security_apps/waap/waap_clib/ParserBinaryFile.h b/components/security_apps/waap/waap_clib/ParserBinaryFile.h index caded39..39c0046 100644 --- a/components/security_apps/waap/waap_clib/ParserBinaryFile.h +++ b/components/security_apps/waap/waap_clib/ParserBinaryFile.h @@ -20,7 +20,7 @@ #define MIN_HEADER_LOOKUP 16 #define MAX_HEADER_LOOKUP 64 -#define MAX_TAIL_LOOKUP 5 +#define MAX_TAIL_LOOKUP 20 class ParserBinaryFile : public ParserBase { public: diff --git a/components/security_apps/waap/waap_clib/ParserPDF.cc b/components/security_apps/waap/waap_clib/ParserPDF.cc index d4fbfdd..e41e7c7 100644 --- a/components/security_apps/waap/waap_clib/ParserPDF.cc +++ b/components/security_apps/waap/waap_clib/ParserPDF.cc @@ -38,9 +38,10 @@ size_t ParserPDF::push(const char *buf, size_t len) { dbgTrace(D_WAAP_PARSER_PDF) - << "buf=" - << buf - << "len=" + << "buf='" + << std::string(buf, std::min((size_t)200, len)) + << (len > 200 ? "..." : "") + << "' len=" << len; const char *c; @@ -65,13 +66,18 @@ ParserPDF::push(const char *buf, size_t len) m_state = s_body; CP_FALL_THROUGH; case s_body: - c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL); - dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c; - if (c) { - m_state = s_end; - CP_FALL_THROUGH; - } else { - break; + { + size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0; + c = strstr(buf + tail_lookup_offset, PDF_TAIL); + dbgTrace(D_WAAP_PARSER_PDF) + << "string to search: " << std::string(buf + tail_lookup_offset) + << " c=" << c; + if (c) { + m_state = s_end; + CP_FALL_THROUGH; + } else { + break; + } } case s_end: if (m_receiver.onKey("PDF", 3) != 0) { diff --git a/components/security_apps/waap/waap_clib/ParserPDF.h b/components/security_apps/waap/waap_clib/ParserPDF.h index 685c85c..684888c 100644 --- a/components/security_apps/waap/waap_clib/ParserPDF.h +++ b/components/security_apps/waap/waap_clib/ParserPDF.h @@ -17,8 +17,8 @@ #include "ParserBase.h" #include -#define MAX_HEADER_LOOKUP 64 -#define MAX_TAIL_LOOKUP 5 +#define MAX_PDF_HEADER_LOOKUP 64 +#define MAX_PDF_TAIL_LOOKUP 20 class ParserPDF : public ParserBase { public: diff --git a/components/security_apps/waap/waap_clib/ScanResult.cc b/components/security_apps/waap/waap_clib/ScanResult.cc index 8bdec91..d83d048 100755 --- a/components/security_apps/waap/waap_clib/ScanResult.cc +++ b/components/security_apps/waap/waap_clib/ScanResult.cc @@ -23,9 +23,11 @@ unescaped_line(), param_name(), location(), score(0.0f), +other_model_score(0.0f), scoreNoFilter(0.0f), scoreArray(), keywordCombinations(), +keywordsAfterFilter(), attack_types(), m_isAttackInParam(false) { @@ -41,9 +43,11 @@ void Waf2ScanResult::clear() param_name.clear(); location.clear(); score = 0; + other_model_score = 0; scoreNoFilter = 0; scoreArray.clear(); keywordCombinations.clear(); + keywordsAfterFilter.clear(); attack_types.clear(); } diff --git a/components/security_apps/waap/waap_clib/ScanResult.h b/components/security_apps/waap/waap_clib/ScanResult.h index 13e00f4..ee4b003 100755 --- a/components/security_apps/waap/waap_clib/ScanResult.h +++ b/components/security_apps/waap/waap_clib/ScanResult.h @@ -29,9 +29,12 @@ struct Waf2ScanResult { std::string param_name; std::string location; double score; + double other_model_score; double scoreNoFilter; std::vector scoreArray; + std::vector coefArray; std::vector keywordCombinations; + std::vector keywordsAfterFilter; std::set attack_types; bool m_isAttackInParam; void clear(); // clear Waf2ScanResult diff --git a/components/security_apps/waap/waap_clib/ScoreBuilder.cc b/components/security_apps/waap/waap_clib/ScoreBuilder.cc index a2859be..3112608 100755 --- a/components/security_apps/waap/waap_clib/ScoreBuilder.cc +++ b/components/security_apps/waap/waap_clib/ScoreBuilder.cc @@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores) m_keywordsDataMap[it->first] = it->second; } } + + m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept; + m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef; + m_stats.isLinModel = baseScores.m_stats.isLinModel; } @@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base m_pWaapAssetState(pWaapAssetState) { restore(); - // merge mergeScores(baseScores); } @@ -352,16 +355,42 @@ void ScoreBuilder::snap() const std::string &poolName = pool.first; const KeywordsScorePool& keywordScorePool = pool.second; m_snapshotKwScoreMap[poolName]; + m_snapshotKwCoefMap[poolName]; + if (keywordScorePool.m_keywordsDataMap.empty()) { + m_snapshotStatsMap[poolName] = KeywordsStats(); + } else { + m_snapshotStatsMap[poolName] = keywordScorePool.m_stats; + } for (const auto &kwData : keywordScorePool.m_keywordsDataMap) { const std::string &kwName = kwData.first; - double kwScore = kwData.second.score; - m_snapshotKwScoreMap[poolName][kwName] = kwScore; + if (keywordScorePool.m_stats.isLinModel) { + m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score; + } else { + m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score; + } + m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef; } } } +KeywordsStats +ScoreBuilder::getSnapshotStats(const std::string &poolName) const +{ + std::map::const_iterator poolIt = m_snapshotStatsMap.find(poolName); + if (poolIt == m_snapshotStatsMap.end()) { + dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool"; + poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE); + if (poolIt == m_snapshotStatsMap.end()) { + dbgWarning(D_WAAP_SCORE_BUILDER) << + "base pool does not exist! This is probably a bug. Returning empty stats"; + return KeywordsStats(); + } + } + return poolIt->second; +} + double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const { @@ -369,12 +398,11 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double if (poolIt == m_snapshotKwScoreMap.end()) { dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool"; poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE); - } - - if (poolIt == m_snapshotKwScoreMap.end()) { - dbgDebug(D_WAAP_SCORE_BUILDER) << - "base pool does not exist! This is probably a bug. Returning default score " << defaultScore; - return defaultScore; + if (poolIt == m_snapshotKwScoreMap.end()) { + dbgDebug(D_WAAP_SCORE_BUILDER) << + "base pool does not exist! This is probably a bug. Returning default score " << defaultScore; + return defaultScore; + } } const KeywordScoreMap &kwScoreMap = poolIt->second; @@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double return kwScoreFound->second; } +double +ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const +{ + std::map::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName); + if (poolIt == m_snapshotKwCoefMap.end()) { + dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool"; + poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE); + } + + if (poolIt == m_snapshotKwCoefMap.end()) { + dbgWarning(D_WAAP_SCORE_BUILDER) << + "base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef; + return defaultCoef; + } + + const KeywordScoreMap &kwCoefMap = poolIt->second; + + KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword); + if (kwCoefFound == kwCoefMap.end()) { + dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef << + " (default, keyword not found in pool '" << poolName << "')"; + return defaultCoef; + } + + dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" << + poolName << "')"; + return kwCoefFound->second; +} + keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip) { return m_fpStore.ipItems[ip]; diff --git a/components/security_apps/waap/waap_clib/ScoreBuilder.h b/components/security_apps/waap/waap_clib/ScoreBuilder.h index 61d17d9..f0bdc52 100755 --- a/components/security_apps/waap/waap_clib/ScoreBuilder.h +++ b/components/security_apps/waap/waap_clib/ScoreBuilder.h @@ -26,6 +26,8 @@ #include #include "WaapDefines.h" +USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER); + struct ScoreBuilderData { std::string m_sourceIdentifier; std::string m_userAgent; @@ -52,11 +54,15 @@ enum KeywordType { }; struct KeywordData { - KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {} + KeywordData() : + truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN) + {} unsigned int truePositiveCtr; unsigned int falsePositiveCtr; double score; + double coef; + double new_score; KeywordType type; template @@ -65,20 +71,42 @@ struct KeywordData { cereal::make_nvp("true_positives", truePositiveCtr), cereal::make_nvp("score", score), cereal::make_nvp("type", type)); + try { + ar(cereal::make_nvp("coef", coef), + cereal::make_nvp("new_score", new_score)); + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + coef = 0; + new_score = 0; + } } }; struct KeywordsStats { - KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {} + KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0) + {} template void serialize(Archive& ar) { ar(cereal::make_nvp("false_positives", falsePositiveCtr), - cereal::make_nvp("true_positives", truePositiveCtr)); + cereal::make_nvp("true_positives", truePositiveCtr)); + try { + ar(cereal::make_nvp("intercept", linModelIntercept), + cereal::make_nvp("log_nnz_coef", linModelNNZCoef)); + isLinModel = true; + } catch (const cereal::Exception &e) { + ar.setNextName(nullptr); + linModelIntercept = 0; + linModelNNZCoef = 0; + isLinModel = false; + } } unsigned int truePositiveCtr; unsigned int falsePositiveCtr; + double linModelIntercept; + double linModelNNZCoef; + bool isLinModel; }; typedef std::unordered_set keywords_set; @@ -106,6 +134,7 @@ struct KeywordsScorePool { KeywordsScorePool(); + // LCOV_EXCL_START Reason: no test exist template KeywordsScorePool(_A &iarchive) { @@ -120,6 +149,7 @@ struct KeywordsScorePool { m_keywordsDataMap[key] = item.second; } } + // LCOV_EXCL_STOP template void serialize(Archive& ar) { @@ -148,6 +178,8 @@ public: void snap(); double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const; + double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const; + KeywordsStats getSnapshotStats(const std::string &poolName) const; keywords_set getIpItemKeywordsSet(std::string ip); keywords_set getUaItemKeywordsSet(std::string userAgent); @@ -200,6 +232,8 @@ protected: SerializedData m_serializedData; std::map &m_keywordsScorePools; // live data continuously updated during traffic std::map m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap() + std::map m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap() + std::map m_snapshotStatsMap; // the snapshot is updated only by a call to snap() std::list m_falsePositivesSetsIntersection; I_WaapAssetState* m_pWaapAssetState; }; diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index f499cb5..389f691 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -195,6 +195,9 @@ void SerializeToFileBase::saveData() } else { ss.str(string((const char *)res.output, res.num_output_bytes)); } + if (res.output) free(res.output); + res.output = nullptr; + res.num_output_bytes = 0; filestream << ss.str(); diff --git a/components/security_apps/waap/waap_clib/WaapOverride.cc b/components/security_apps/waap/waap_clib/WaapOverride.cc index 30e691b..9f1d670 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.cc +++ b/components/security_apps/waap/waap_clib/WaapOverride.cc @@ -12,7 +12,6 @@ // limitations under the License. #include "WaapOverride.h" -#include "Waf2Util.h" USE_DEBUG_FLAG(D_WAAP); @@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const (m_operand1 == other.m_operand1) && (m_operand2 == other.m_operand2) && (m_tag == other.m_tag) && - Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) && - m_cidr == other.m_cidr && + (m_valuesRegex == other.m_valuesRegex) && + m_ip_addr_values == other.m_ip_addr_values && m_isCidr == other.m_isCidr; } diff --git a/components/security_apps/waap/waap_clib/WaapOverride.h b/components/security_apps/waap/waap_clib/WaapOverride.h index 80b7174..024c656 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.h +++ b/components/security_apps/waap/waap_clib/WaapOverride.h @@ -21,6 +21,7 @@ #include #include "debug.h" #include "CidrMatch.h" +#include "RegexComparator.h" USE_DEBUG_FLAG(D_WAAP_OVERRIDE); @@ -52,23 +53,52 @@ public: m_isValid = false; dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag; } - // The name "value" here is misleading. The real meaning is "regex pattern string" - ar(cereal::make_nvp("value", m_value)); + + try { + ar(cereal::make_nvp("values", m_values)); + dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead."; + } catch (const cereal::Exception &e) { + // The name "value" here is misleading. The real meaning is "regex pattern string" + ar(cereal::make_nvp("value", m_value)); + m_values.insert(m_value); + } if (m_tag == "sourceip" || m_tag == "sourceidentifier") { - m_isCidr = Waap::Util::isCIDR(m_value, m_cidr); + m_isCidr = true; + m_ip_addr_values.resize(m_values.size()); + + int val_idx = 0; + for (const auto &cur_val : m_values) { + if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) { + dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val; + m_isValid = false; + break; + } + val_idx++; + } + sortAndMergeCIDRs(); + dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values); } m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody"); if (!m_isCidr) { - // regex build may throw boost::regex_error - m_valueRegex = nullptr; - try { - m_valueRegex = std::make_shared(m_value); - } - catch (const boost::regex_error &err) { - dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" << - m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'"; + for (const auto &cur_val : m_values) { + try { + m_valuesRegex.emplace(std::make_shared(cur_val)); + } + catch (const boost::regex_error &err) { + dbgDebug(D_WAAP_OVERRIDE) + << "Waap::Override::Match(): Failed to compile regex pattern '" + << cur_val + << "' on position " + << err.position() + << ". Reason: '" + << err.what() + << "'"; + m_isValid = false; + m_valuesRegex.clear(); + break; + } } } } @@ -95,13 +125,21 @@ public: template bool match(TestFunctor testFunctor) const { if (m_op == "basic" && m_isCidr) { - bool result = testFunctor(m_tag, m_cidr); - dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result; + bool result = testFunctor(m_tag, m_ip_addr_values); + dbgTrace(D_WAAP_OVERRIDE) + << "Override matching CIDR list: " + << cidrsToString(m_ip_addr_values) + << " result: " + << result; return result; } - else if (m_op == "basic" && m_valueRegex) { - bool result = testFunctor(m_tag, *m_valueRegex); - dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result; + else if (m_op == "basic" && !m_valuesRegex.empty()) { + bool result = testFunctor(m_tag, m_valuesRegex); + dbgTrace(D_WAAP_OVERRIDE) + << "Override matching regex list: " + << regexSetToString(m_valuesRegex) + << " result: " + << result; return result; } if (m_op == "and") { @@ -125,24 +163,39 @@ public: return false; } - bool isOverrideResponse() const { - return m_isOverrideResponse; - } + bool isOverrideResponse() const { return m_isOverrideResponse; } - bool isValidMatch() const{ - return m_isValid; - } + bool isValidMatch() const { return m_isValid; } private: + void sortAndMergeCIDRs() { + if (m_ip_addr_values.empty()) return; + std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end()); + + size_t mergedIndex = 0; + for (size_t i = 1; i < m_ip_addr_values.size(); ++i) { + Waap::Util::CIDRData ¤t = m_ip_addr_values[mergedIndex]; + Waap::Util::CIDRData &next = m_ip_addr_values[i]; + + if (!doesFirstCidrContainSecond(current, next)) { + ++mergedIndex; + if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next; + } + } + + m_ip_addr_values.resize(mergedIndex + 1); + } + std::string m_op; std::shared_ptr m_operand1; std::shared_ptr m_operand2; std::string m_tag; std::string m_value; - std::shared_ptr m_valueRegex; - Waap::Util::CIDRData m_cidr; - bool m_isCidr; - bool m_isOverrideResponse; + std::set m_values; + std::vector m_ip_addr_values; + std::set, Waap::Util::RegexComparator> m_valuesRegex; + bool m_isCidr; + bool m_isOverrideResponse; bool m_isValid; }; diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc index 0f2b5d6..e0c686d 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc @@ -17,100 +17,150 @@ #include "WaapOverrideFunctor.h" #include "Waf2Engine.h" #include "CidrMatch.h" +#include "RegexComparator.h" #include "agent_core_utilities.h" #include "debug.h" USE_DEBUG_FLAG(D_WAAP_OVERRIDE); +#define REGX_MATCH(FIELD_FETCH) \ + NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx) +#define W2T_REGX_MATCH(FIELD_GETTER) \ + REGX_MATCH(waf2Transaction.FIELD_GETTER()) + WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction) { } -bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) { +bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector &values) { + std::string sourceIp; if (tag == "sourceip") { - dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " << - value.cidrString; - std::string sourceIp = waf2Transaction.getRemoteAddr(); - // match sourceIp against the cidr - return Waap::Util::cidrMatch(sourceIp, value); + dbgDebug(D_WAAP_OVERRIDE) + << "Remote IP Address : " + << waf2Transaction.getRemoteAddr(); + + sourceIp = waf2Transaction.getRemoteAddr(); + } else if (tag == "sourceidentifier") { + dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr(); + sourceIp = waf2Transaction.getSourceIdentifier(); + } else { + dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag; + return false; } - else if (tag == "sourceidentifier") { - dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " << - value.cidrString; - std::string sourceIp = waf2Transaction.getSourceIdentifier(); - // match source against the cidr - return Waap::Util::cidrMatch(sourceIp, value); + + Waap::Util::CIDRData source_cidr; + if (!Waap::Util::isCIDR(sourceIp, source_cidr)) { + dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp; + return false; + } + + + int left = 0; + int right = values.size() - 1; + + while (left <= right) { + int mid = left + (right - left) / 2; + if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true; + + if (values[mid] < source_cidr) { + left = mid + 1; + } else { + right = mid - 1; + } } return false; } -bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx) +bool WaapOverrideFunctor::operator()( + const std::string &tag, + const std::set, Waap::Util::RegexComparator> &rxes) { boost::cmatch what; std::string tagLower = tag; std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower); + try { if (tagLower == "method") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx); + for (const auto &rx : rxes) { + if (W2T_REGX_MATCH(getMethod)) return true; + } + return false; } else if (tagLower == "url") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx); + for (const auto &rx : rxes) { + if (W2T_REGX_MATCH(getUriStr)) return true; + } + return false; } else if (tagLower == "hostname") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx); + for (const auto &rx : rxes) { + if (W2T_REGX_MATCH(getHost)) return true; + } + return false; } else if (tagLower == "sourceidentifier") { - return NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - waf2Transaction.getSourceIdentifier().c_str(), - what, - rx - ); + for (const auto &rx : rxes) { + if (W2T_REGX_MATCH(getSourceIdentifier)) return true; + } + return false; } else if (tagLower == "keyword") { - for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) { - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) { - return true; + for (const auto &rx : rxes) { + for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) { + if (REGX_MATCH(keywordStr)) { + return true; + } } } return false; } else if (tagLower == "paramname") { - for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) { - return true; + for (const auto &rx : rxes) { + for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { + if (REGX_MATCH(keywordInfo.getName())) { + return true; + } } - } - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) { - return true; - } - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) { - return true; + if (W2T_REGX_MATCH(getParamKey)) return true; + if (W2T_REGX_MATCH(getParam)) return true; } return false; } else if (tagLower == "paramvalue") { - for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) { - return true; + for (const auto &rx : rxes) { + for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { + if (REGX_MATCH(keywordInfo.getValue())) { + return true; + } } + if (W2T_REGX_MATCH(getSample)) return true; } - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) { - return true; - } + return false; } else if (tagLower == "paramlocation") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx); + for (const auto &rx : rxes) { + if (W2T_REGX_MATCH(getLocation)) return true; + } + return false; } else if (tagLower == "responsebody") { waf2Transaction.getResponseInspectReasons().setApplyOverride(true); if (!waf2Transaction.getResponseBody().empty()) { - boost::smatch matcher; - return NGEN::Regex::regexSearch(__FILE__, __LINE__, - waf2Transaction.getResponseBody().c_str(), matcher, rx); + for (const auto &rx : rxes) { + boost::smatch matcher; + if (NGEN::Regex::regexSearch( + __FILE__, + __LINE__, + waf2Transaction.getResponseBody().c_str(), + matcher, + *rx + )) { + return true; + } + } + return false; } else { return false; } @@ -119,11 +169,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required"; return false; } - for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { - std::string value = hdr_pair.first; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) { - return true; + for (const auto &rx : rxes) { + for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { + std::string value = hdr_pair.first; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if (REGX_MATCH(value)) { + return true; + } } } return false; @@ -132,11 +184,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required"; return false; } - for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { - std::string value = hdr_pair.second; - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) { - return true; + for (const auto &rx : rxes) { + for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { + std::string value = hdr_pair.second; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if (REGX_MATCH(value)) { + return true; + } } } return false; @@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what(); return false; } - // Unknown tag: should not occur dbgDebug(D_WAAP) << "Invalid override tag: " << tag; return false; diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.h b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.h index 13d76ef..cea7317 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.h +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.h @@ -16,6 +16,7 @@ namespace Waap { namespace Util { struct CIDRData; // forward decleration + struct RegexComparator; } } @@ -24,12 +25,15 @@ class Waf2Transaction; // Functor used to match Override rules against request data class WaapOverrideFunctor { public: - WaapOverrideFunctor(Waf2Transaction& waf2Transaction); + WaapOverrideFunctor(Waf2Transaction &waf2Transaction); - bool operator()(const std::string& tag, const Waap::Util::CIDRData& value); + bool operator()(const std::string &tag, const std::vector &values); - bool operator()(const std::string& tag, const boost::regex& rx); + bool operator()( + const std::string &tag, + const std::set, Waap::Util::RegexComparator> &rxes + ); private: - Waf2Transaction& waf2Transaction; + Waf2Transaction &waf2Transaction; }; diff --git a/components/security_apps/waap/waap_clib/WaapScanner.cc b/components/security_apps/waap/waap_clib/WaapScanner.cc index 21b8bb3..7f2a751 100755 --- a/components/security_apps/waap/waap_clib/WaapScanner.cc +++ b/components/security_apps/waap/waap_clib/WaapScanner.cc @@ -13,7 +13,9 @@ #include "WaapScanner.h" #include "WaapScores.h" +#include "Waf2Engine.h" #include "i_transaction.h" +#include "WaapModelResultLogger.h" #include #include "debug.h" #include "reputation_features_events.h" @@ -105,20 +107,59 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN } std::sort(newKeywords.begin(), newKeywords.end()); - + res.keywordsAfterFilter.clear(); + for (auto keyword : newKeywords) { + res.keywordsAfterFilter.push_back(keyword); + } res.scoreArray.clear(); + res.coefArray.clear(); res.keywordCombinations.clear(); + double res_score = getScoreFromPool(res, newKeywords, poolName); + + string other_pool_name = Waap::Scores::getOtherScorePoolName(); + Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings(); + + if (applyLearning && poolName != other_pool_name && + modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) { + double other_score = getScoreFromPool(res, newKeywords, other_pool_name); + + dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score + << ", vs. pool " << other_pool_name << ": " << other_score + << ", score difference: " << res_score - other_score + << ", sample: " << res.unescaped_line; + Singleton::Consume::by()->logModelResult( + modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score + ); + res.other_model_score = other_score; + } else { + res.other_model_score = res_score; + } + return res_score; +} + +double Waap::Scanner::getScoreFromPool( + Waf2ScanResult &res, const std::vector &newKeywords, const std::string &poolName +) +{ + KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName); + if (!newKeywords.empty()) { // Collect scores of individual keywords Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords, - res.scoreArray); + res.scoreArray, res.coefArray); // Collect keyword combinations and their scores. Append scores to scoresArray, // and also populate m_scanResultKeywordCombinations list Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords, - res.scoreArray, res.keywordCombinations); + res.scoreArray, res.coefArray, res.keywordCombinations); } + if (stats.isLinModel) { + return Waap::Scores::calcLogisticRegressionScore( + res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef + ); + } + // use base_scores calculation return Waap::Scores::calcArrayScore(res.scoreArray); } @@ -127,7 +168,7 @@ bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, { if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") { if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" || - (key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) { + (key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) { dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring."; return true; } diff --git a/components/security_apps/waap/waap_clib/WaapScanner.h b/components/security_apps/waap/waap_clib/WaapScanner.h index 4e62206..898a713 100644 --- a/components/security_apps/waap/waap_clib/WaapScanner.h +++ b/components/security_apps/waap/waap_clib/WaapScanner.h @@ -44,6 +44,9 @@ namespace Waap { static const std::string xmlEntityAttributeId; private: double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true); + double getScoreFromPool( + Waf2ScanResult &res, const std::vector &newKeywords, const std::string &poolName + ); bool shouldIgnoreOverride(const Waf2ScanResult &res); bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp); diff --git a/components/security_apps/waap/waap_clib/WaapScores.cc b/components/security_apps/waap/waap_clib/WaapScores.cc index 78c43b5..1c707e9 100644 --- a/components/security_apps/waap/waap_clib/WaapScores.cc +++ b/components/security_apps/waap/waap_clib/WaapScores.cc @@ -14,6 +14,8 @@ #include "WaapScores.h" #include #include +#include + #include "ScoreBuilder.h" #include "WaapDefines.h" #include "debug.h" @@ -24,11 +26,44 @@ namespace Waap { namespace Scores { std::string getScorePoolNameByLocation(const std::string &location) { - std::string poolName = KEYWORDS_SCORE_POOL_BASE; - if (location == "header") { - poolName = KEYWORDS_SCORE_POOL_HEADERS; + auto maybePoolName = getProfileAgentSetting("agent.waap.scorePoolName"); + std::string res = KEYWORDS_SCORE_POOL_BASE; + if (maybePoolName.ok()) { + res = maybePoolName.unpack(); } - return poolName; + else if (location == "header") { + res = KEYWORDS_SCORE_POOL_HEADERS; + } + return res; +} + +std::string getOtherScorePoolName() +{ + auto maybePoolName = getProfileAgentSetting("agent.waap.otherScorePoolName"); + if (maybePoolName.ok()) { + return maybePoolName.unpack(); + } + return KEYWORDS_SCORE_POOL_BASE; +} + +ModelLoggingSettings getModelLoggingSettings() +{ + ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF, + .logToS3 = false, + .logToStream = true}; + auto maybeLogS3 = getProfileAgentSetting("agent.waap.modelLogToS3"); + if (maybeLogS3.ok()) { + settings.logToS3 = maybeLogS3.unpack(); + } + auto maybeLogKusto = getProfileAgentSetting("agent.waap.modelLogToStream"); + if (maybeLogKusto.ok()) { + settings.logToStream = maybeLogKusto.unpack(); + } + auto maybeLogLevel = getProfileAgentSetting("agent.waap.modelLogLevel"); + if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) { + settings.logLevel = static_cast(maybeLogLevel.unpack()); + } + return settings; } void @@ -37,9 +72,16 @@ addKeywordScore( const std::string &poolName, std::string keyword, double defaultScore, - std::vector& scoresArray) + double defaultCoef, + std::vector& scoresArray, + std::vector& coefArray) { - scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName)); + double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName); + double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName); + dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef + << " keyword: '" << keyword << "' pool: " << poolName; + scoresArray.push_back(score); + coefArray.push_back(coef); } // Calculate score of individual keywords @@ -48,13 +90,14 @@ calcIndividualKeywords( const ScoreBuilder& scoreBuilder, const std::string &poolName, const std::vector& keyword_matches, - std::vector& scoresArray) + std::vector& scoresArray, + std::vector& coefArray) { std::vector keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!) std::sort(keywords.begin(), keywords.end()); for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) { - addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray); + addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray); } } @@ -65,10 +108,12 @@ calcCombinations( const std::string &poolName, const std::vector& keyword_matches, std::vector& scoresArray, + std::vector& coefArray, std::vector& keyword_combinations) { keyword_combinations.clear(); static const double max_combi_score = 1.0f; + double default_coef = 0.8f; for (size_t i = 0; i < keyword_matches.size(); ++i) { std::vector combinations; @@ -93,7 +138,7 @@ calcCombinations( } // set default combination score to be the sum of its keywords, bounded by 1 default_score = std::min(default_score, max_combi_score); - addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray); + addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray); keyword_combinations.push_back(combination); } } @@ -105,7 +150,6 @@ calcArrayScore(std::vector& scoreArray) // Calculate cumulative score from array of individual scores double score = 1.0f; for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) { - dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore; double left = 10.0f - score; double divisor = (*pScore / 3.0f + 10.0f); // note: divisor can't be empty because // *pScore is always positive and there's a +10 offset @@ -115,5 +159,20 @@ calcArrayScore(std::vector& scoreArray) return score; } +double +calcLogisticRegressionScore(std::vector &coefArray, double intercept, double nnzCoef) +{ + // Sparse logistic regression model, with boolean feature values + // Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features + // An additional feature was added for the log of the number of non-zero features, as a regularization term + double log_odds = intercept + nnzCoef * log(static_cast(coefArray.size()) + 1); + for (double &pCoef : coefArray) { + log_odds += pCoef; + } + // Apply the expit function to the log-odds to obtain the probability, + // and multiply by 10 to obtain a 'score' in the range [0, 10] + return 1.0f / (1.0f + exp(-log_odds)) * 10.0f; +} + } } diff --git a/components/security_apps/waap/waap_clib/WaapScores.h b/components/security_apps/waap/waap_clib/WaapScores.h index 71dcc81..d6258d9 100644 --- a/components/security_apps/waap/waap_clib/WaapScores.h +++ b/components/security_apps/waap/waap_clib/WaapScores.h @@ -20,7 +20,21 @@ namespace Waap { namespace Scores { +enum class ModelLogLevel { + OFF = 0, + DIFF = 1, + ALL = 2 +}; + +struct ModelLoggingSettings { + ModelLogLevel logLevel; + bool logToS3; + bool logToStream; +}; + std::string getScorePoolNameByLocation(const std::string &location); +std::string getOtherScorePoolName(); +ModelLoggingSettings getModelLoggingSettings(); void addKeywordScore( @@ -28,7 +42,9 @@ addKeywordScore( const std::string &poolName, std::string keyword, double defaultScore, - std::vector& scoresArray); + double defaultCoef, + std::vector& scoresArray, + std::vector& coefArray); // Calculate score of individual keywords void @@ -36,7 +52,8 @@ calcIndividualKeywords( const ScoreBuilder& scoreBuilder, const std::string &poolName, const std::vector& keyword_matches, - std::vector& scoresArray); + std::vector& scoresArray, + std::vector& coefArray); // Calculate keyword combinations and their scores void @@ -45,9 +62,11 @@ calcCombinations( const std::string &poolName, const std::vector& keyword_matches, std::vector& scoresArray, + std::vector& coefArray, std::vector& keyword_combinations); double calcArrayScore(std::vector& scoreArray); +double calcLogisticRegressionScore(std::vector &coefArray, double intercept, double nnzCoef=0.0); } } diff --git a/components/security_apps/waap/waap_clib/WaapTrigger.h b/components/security_apps/waap/waap_clib/WaapTrigger.h index e1493fc..72bf4cf 100755 --- a/components/security_apps/waap/waap_clib/WaapTrigger.h +++ b/components/security_apps/waap/waap_clib/WaapTrigger.h @@ -125,6 +125,7 @@ struct Log { struct Trigger { template void serialize(_A &ar) { + ar(cereal::make_nvp("id", triggerId)); ar(cereal::make_nvp("$triggerType", triggerType)); triggerType = to_lower_copy(triggerType); @@ -137,6 +138,7 @@ struct Trigger { Trigger(); bool operator==(const Trigger &other) const; + std::string triggerId; std::string triggerType; std::shared_ptr log; }; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 0a8a6a7..cadae7b 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan std::string tag = scanStage + "_param"; m_deepParser.m_key.push(tag.data(), tag.size()); size_t buff_len = uriEnd - p; - dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'"; - ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len)); + + bool should_decode = checkUrlEncoded(p, buff_len); + if (should_decode) { + std::string url_as_string(p, buff_len); + should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true); + } + dbgTrace(D_WAAP) << "should_decode % = " << should_decode; + + ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode); up.push(p, buff_len); up.finish(); m_deepParser.m_key.pop(tag.c_str()); @@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) { m_ignoreScore = ignoreScore; } +bool Waf2Transaction::get_ignoreScore() const +{ + auto maybe_override_ignore_score = getProfileAgentSetting("agent.waap.alwaysReportScore"); + if (maybe_override_ignore_score.ok()) { + return maybe_override_ignore_score.unpack() == "true"; + } + return m_ignoreScore; +} + void Waf2Transaction::decide( bool& bForceBlock, @@ -1889,6 +1905,14 @@ Waf2Transaction::sendLog() return; } + std::set triggers_set; + for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { + triggers_set.insert(trigger.triggerId); + dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId; + } + ScopedContext ctx; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); + auto maybeLogTriggerConf = getConfiguration("rulebase", "log"); switch (decision_type) { diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index 075d60f..6e1caa6 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -20,6 +20,8 @@ #include "DeepParser.h" #include "WaapAssetState.h" #include "PatternMatcher.h" +#include "generic_rulebase/rulebase_config.h" +#include "generic_rulebase/evaluators/trigger_eval.h" #include "Waf2Util.h" #include "WaapConfigApplication.h" #include "WaapConfigApi.h" @@ -39,6 +41,7 @@ #include "i_waap_telemetry.h" #include "i_deepAnalyzer.h" #include "i_time_get.h" +#include "i_waap_model_result_logger.h" #include "table_opaque.h" #include "WaapResponseInspectReasons.h" #include "WaapResponseInjectReasons.h" @@ -91,6 +94,7 @@ public: const std::vector getFilteredKeywords() const; const std::map> getFilteredVerbose() const; virtual const std::vector getKeywordsCombinations() const; + virtual const std::vector getKeywordsAfterFilter() const; const std::vector& getKeywordInfo() const; const std::vector >& getKvPairs() const; const std::string getKeywordMatchesStr() const; @@ -99,6 +103,9 @@ public: const std::string getLastScanSample() const; virtual const std::string& getLastScanParamName() const; double getScore() const; + // LCOV_EXCL_START Reason: model testing + double getOtherModelScore() const; + // LCOV_EXCL_STOP const std::vector getScoreArray() const; Waap::CSRF::State& getCsrfState(); const std::set getFoundPatterns() const; @@ -161,7 +168,7 @@ public: // decision functions void set_ignoreScore(bool ignoreScore); - bool get_ignoreScore() const { return m_ignoreScore; } + bool get_ignoreScore() const; void decide( bool& bForceBlock, bool& bForceException, diff --git a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc index 0b338cf..73ecfbb 100755 --- a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc +++ b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc @@ -189,6 +189,15 @@ const std::vector Waf2Transaction::getKeywordsCombinations() const } return std::vector(); } +const std::vector Waf2Transaction::getKeywordsAfterFilter() const +{ + if (m_scanResult) + { + return m_scanResult->keywordsAfterFilter; + } + return std::vector(); +} + const std::vector& Waf2Transaction::getKeywordInfo() const { return m_deepParser.m_keywordInfo; @@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const } return 0; } +// LCOV_EXCL_START Reason: model testing +double Waf2Transaction::getOtherModelScore() const +{ + if (m_scanResult) { + return m_scanResult->other_model_score; + } + return 0; +} +// LCOV_EXCL_STOP const std::vector Waf2Transaction::getScoreArray() const { if (m_scanResult) { @@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog( waap_log << LogField("waapUriFalsePositiveScore", (int)( autonomousSecurityDecision->getFpMitigationScore() * 100)); waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100)); + waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100)); waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100)); waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel()); } diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index d924b57..d360bfa 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -508,6 +508,8 @@ const char* g_htmlTags[] = { }; static const string b64_prefix("base64,"); +// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded +static const double base64_entropy_threshold = 4.01; const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]); @@ -966,7 +968,8 @@ base64_decode_status decodeBase64Chunk( string::const_iterator it, string::const_iterator end, string& decoded, - bool clear_on_error) + bool clear_on_error, + bool called_with_prefix) { decoded.clear(); uint32_t acc = 0; @@ -974,6 +977,7 @@ base64_decode_status decodeBase64Chunk( int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them. uint32_t nonPrintableCharsCount = 0; uint32_t spacer_count = 0; + uint32_t length = end - it; dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'"; string::const_iterator begin = it; @@ -986,6 +990,8 @@ base64_decode_status decodeBase64Chunk( return B64_DECODE_INVALID; } + std::unordered_map frequency; + while (it != end) { unsigned char c = *it; @@ -1008,6 +1014,7 @@ base64_decode_status decodeBase64Chunk( // allow for more terminator characters it++; + frequency[c]++; continue; } @@ -1032,6 +1039,7 @@ base64_decode_status decodeBase64Chunk( // Start tracking terminator characters terminatorCharsSeen++; it++; + frequency[c]++; continue; } else { @@ -1062,6 +1070,7 @@ base64_decode_status decodeBase64Chunk( } it++; + frequency[c]++; } // end of encoded sequence decoded. @@ -1078,6 +1087,27 @@ base64_decode_status decodeBase64Chunk( << "; decoded='" << decoded << "'"; + // Check if entropy is correlates with b64 threshold (initially > 4.5) + if (!called_with_prefix) { + double entropy = 0; + double p = 0; + for (const auto& pair : frequency) { + p = pair.second / length; + entropy -= p * std::log2(p); + } + dbgTrace(D_WAAP_BASE64) << " ===b64Test===: base entropy = " << entropy << "length = " << length; + // Add short payload factor + if (length < 16) + entropy = entropy * 16 / length; + // Enforce tailoring '=' characters + entropy+=terminatorCharsSeen; + + dbgTrace(D_WAAP_BASE64) << " ===b64Test===: corrected entropy = " << entropy << "length = " << length; + if (entropy <= base64_entropy_threshold) { + return B64_DECODE_INVALID; + } + } + // Return success only if decoded.size>=5 and there are less than 10% of non-printable // characters in output. if (decoded.size() >= 5) { @@ -1323,10 +1353,11 @@ processDecodedChunk( string::const_iterator start, string::const_iterator end, string &value, - BinaryFileType &binaryFileType + BinaryFileType &binaryFileType, + bool called_with_prefix = false ) { - base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false); + base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix); dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal << " value.size()=" << value.size(); if (retVal != B64_DECODE_INVALID && !value.empty()) { @@ -1349,7 +1380,7 @@ bool isBase64PrefixProcessingOK ( if (detectBase64Chunk(s, start, end)) { dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected"; if ((start != s.end()) && (end == s.end())) { - retVal = processDecodedChunk(s, start, end, value, binaryFileType); + retVal = processDecodedChunk(s, start, end, value, binaryFileType, true); } } else if (start != s.end()) { dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected." diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index d95a8ee..48542dd 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -871,7 +871,8 @@ decodeBase64Chunk( std::string::const_iterator it, std::string::const_iterator end, std::string &decoded, - bool clear_on_error = true); + bool clear_on_error = true, + bool called_with_prefix = false); bool b64DecodeChunk( diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 5549977..72db6ca 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -50,7 +50,8 @@ WaapComponent::Impl::Impl() : drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP), waapStateTable(NULL), transactionsCount(0), - deepAnalyzer() + deepAnalyzer(), + waapModelResultLogger() { } diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index 91ab2c3..33c8551 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -19,6 +19,7 @@ #include "table_opaque.h" #include "i_transaction.h" #include "waap_clib/DeepAnalyzer.h" +#include "waap_clib/WaapModelResultLogger.h" #include "waap_clib/WaapAssetState.h" #include "waap_clib/WaapAssetStatesManager.h" #include "reputation_features_agg.h" @@ -80,6 +81,7 @@ private: uint64_t transactionsCount; // instance of singleton classes DeepAnalyzer deepAnalyzer; + WaapModelResultLogger waapModelResultLogger; WaapAssetStatesManager waapAssetStatesManager; std::unordered_set m_seen_assets_id; }; diff --git a/components/utils/generic_rulebase/match_query.cc b/components/utils/generic_rulebase/match_query.cc index 4703c2d..ab37c09 100644 --- a/components/utils/generic_rulebase/match_query.cc +++ b/components/utils/generic_rulebase/match_query.cc @@ -157,6 +157,9 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in) dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what(); } } + if (isKeyTypeIp()) { + sortAndMergeIpRangesValues(); + } first_value = *(value.begin()); } break; @@ -301,7 +304,7 @@ MatchQuery::matchAttributes( bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn; bool match = false; - if (isIP()) { + if (isKeyTypeIp()) { match = matchAttributesIp(values); } else if (isRegEx()) { match = matchAttributesRegEx(values, matched_override_keywords); @@ -352,10 +355,18 @@ MatchQuery::matchAttributesString(const set &values) const bool MatchQuery::matchAttributesIp(const set &values) const { - for (const IPRange &rule_ip_range : ip_addr_value) { - for (const string &requested_value : values) { - IpAddress ip_addr = IPUtilities::createIpFromString(requested_value); - if (IPUtilities::isIpAddrInRange(rule_ip_range, ip_addr)) return true; + for (const string &requested_value : values) { + int left = 0; + int right = ip_addr_value.size() - 1; + IpAddress ip_addr = IPUtilities::createIpFromString(requested_value); + while (left <= right) { + int mid = left + (right - left) / 2; + if (IPUtilities::isIpAddrInRange(ip_addr_value[mid], ip_addr)) return true; + if (ip_addr_value[mid].start < ip_addr) { + left = mid + 1; + } else { + right = mid - 1; + } } } return false; @@ -367,8 +378,23 @@ MatchQuery::isRegEx() const return key != "protectionName"; } -bool -MatchQuery::isIP() const +void +MatchQuery::sortAndMergeIpRangesValues() { - return key == "sourceIP" || key == "destinationIP"; + if (ip_addr_value.empty()) return; + + sort(ip_addr_value.begin(), ip_addr_value.end()); + size_t mergedIndex = 0; + for (size_t i = 1; i < ip_addr_value.size(); ++i) { + if (ip_addr_value[i].start <= ip_addr_value[mergedIndex].end) { + if (ip_addr_value[mergedIndex].end <= ip_addr_value[i].end) { + ip_addr_value[mergedIndex].end = ip_addr_value[i].end; + } + } else { + ++mergedIndex; + ip_addr_value[mergedIndex] = ip_addr_value[i]; + } + } + + ip_addr_value.resize(mergedIndex + 1); } diff --git a/components/utils/ip_utilities/ip_utilities.cc b/components/utils/ip_utilities/ip_utilities.cc index 2b7d012..9f77754 100644 --- a/components/utils/ip_utilities/ip_utilities.cc +++ b/components/utils/ip_utilities/ip_utilities.cc @@ -22,7 +22,10 @@ bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr) { if (this_ip_addr.ip_type < other_ip_addr.ip_type) return true; - if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr < other_ip_addr.addr4_t.s_addr; + if (this_ip_addr.ip_type > other_ip_addr.ip_type) return false; + if (this_ip_addr.ip_type == IP_VERSION_4) { + return ntohl(this_ip_addr.addr4_t.s_addr) < ntohl(other_ip_addr.addr4_t.s_addr); + } return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) < 0; } @@ -33,6 +36,19 @@ operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr) if (this_ip_addr.ip_type == IP_VERSION_4) return this_ip_addr.addr4_t.s_addr == other_ip_addr.addr4_t.s_addr; return memcmp(&this_ip_addr.addr6_t, &other_ip_addr.addr6_t, sizeof(struct in6_addr)) == 0; } + +bool +operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr) +{ + if (this_ip_addr < other_ip_addr || this_ip_addr == other_ip_addr) return true; + return false; +} + +bool +operator<(const IPRange &range1, const IPRange &range2) +{ + return range1.start < range2.start || (range1.start == range2.start && range1.end < range2.end); +} // LCOV_EXCL_STOP Maybe> diff --git a/components/utils/keywords/keywords_rule.cc b/components/utils/keywords/keywords_rule.cc index 4e7bbc1..d8b3f53 100644 --- a/components/utils/keywords/keywords_rule.cc +++ b/components/utils/keywords/keywords_rule.cc @@ -5,6 +5,8 @@ using namespace std; +USE_DEBUG_FLAG(D_KEYWORD); + static const string whitespaces = " \t"; static string @@ -14,6 +16,13 @@ getSubStrNoPadding(const string &str, uint start, uint end) auto r_end = str.find_last_not_of(whitespaces, end-1); if (r_end==string::npos || r_start==string::npos || r_start>r_end) { + dbgWarning(D_KEYWORD) + << "Can't extract substring from '" + << str + << "', padded start: " + << r_start + << ", padded end: " + << r_end; throw KeywordError("Found an empty section in the '"+ str + "'"); } @@ -45,13 +54,24 @@ split(const string &str, const string &delim, uint start = 0) } default: if (!in_string && delim.find(str[index])!=string::npos) { - res.push_back(getSubStrNoPadding(str, part_start, index)); + if (part_start == index) { + dbgTrace(D_KEYWORD) << "Encountered consecutive delimiter in: " << str; + } else { + res.push_back(getSubStrNoPadding(str, part_start, index)); + } part_start = index+1; } } } - if (escape||in_string) throw KeywordError("Split has ended in the middle of the parsing"); + if (escape||in_string) { + dbgWarning(D_KEYWORD) + << "Failed to split " + << str + << ". Split ended in " + << (escape ? "escape" : "middle of string"); + throw KeywordError("Split has ended in the middle of the parsing"); + } if (str.find_first_not_of(whitespaces, part_start)!=string::npos) { res.push_back(getSubStrNoPadding(str, part_start, str.size())); diff --git a/components/utils/keywords/keywords_ut/keywords_ut.cc b/components/utils/keywords/keywords_ut/keywords_ut.cc index 70894b7..731703d 100644 --- a/components/utils/keywords/keywords_ut/keywords_ut.cc +++ b/components/utils/keywords/keywords_ut/keywords_ut.cc @@ -55,6 +55,13 @@ TEST_F(KeywordsRuleTest, data_basic_test) { EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;")); } +TEST_F(KeywordsRuleTest, consecutive_delimiter_test) { + appendBuffer("HTTP_RESPONSE_BODY", "123456789"); + + EXPECT_TRUE(ruleRun("data: \"234\" , part HTTP_RESPONSE_BODY;")); + EXPECT_FALSE(ruleRun("data: \"75\", part HTTP_RESPONSE_BODY;")); +} + TEST_F(KeywordsRuleTest, data_relative_test) { appendBuffer("HTTP_RESPONSE_BODY", "1234567890"); diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index fdbeabe..8d28ed2 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -304,6 +304,12 @@ AgentDetails::getOrchestrationMode() const return orchestration_mode; } +bool +AgentDetails::isOpenAppsecAgent() const +{ + return (orchestration_mode == OrchestrationMode::HYBRID) || (tenant_id.rfind("org_", 0) == 0); +} + string AgentDetails::getAccessToken() const { diff --git a/core/config/config.cc b/core/config/config.cc index e30842d..b77f6d4 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -315,22 +315,13 @@ void ConfigComponent::Impl::preload() { I_Environment *environment = Singleton::Consume::by(); - auto executable = environment->get("Executable Name"); + auto executable = environment->get("Base Executable Name"); if (!executable.ok() || *executable == "") { dbgWarning(D_CONFIG) << "Could not load nano service's settings since \"Executable Name\" in not found in the environment"; return; } - executable_name = *executable; - auto file_path_end = executable_name.find_last_of("/"); - if (file_path_end != string::npos) { - executable_name = executable_name.substr(file_path_end + 1); - } - auto file_sufix_start = executable_name.find_first_of("."); - if (file_sufix_start != string::npos) { - executable_name = executable_name.substr(0, file_sufix_start); - } config_file_paths.insert(executable_name + "-conf.json"); config_file_paths.insert(executable_name + "-debug-conf.json"); diff --git a/core/debug_is/debug.cc b/core/debug_is/debug.cc index 89ad565..00dacd5 100644 --- a/core/debug_is/debug.cc +++ b/core/debug_is/debug.cc @@ -766,22 +766,8 @@ Debug::findDebugFilePrefix(const string &file_name) string Debug::getExecutableName() { - auto executable = env->get("Executable Name"); - if (!executable.ok() || *executable == "") { - return ""; - } - - string executable_name = *executable; - auto file_path_end = executable_name.find_last_of("/"); - if (file_path_end != string::npos) { - executable_name = executable_name.substr(file_path_end + 1); - } - auto file_sufix_start = executable_name.find_first_of("."); - if (file_sufix_start != string::npos) { - executable_name = executable_name.substr(0, file_sufix_start); - } - - return executable_name; + auto executable = env->get("Base Executable Name"); + return executable.ok() ? *executable : ""; } void diff --git a/core/debug_is/debug_is_ut/debug_ut.cc b/core/debug_is/debug_is_ut/debug_ut.cc index 66ea543..106a028 100644 --- a/core/debug_is/debug_is_ut/debug_ut.cc +++ b/core/debug_is/debug_is_ut/debug_ut.cc @@ -836,7 +836,7 @@ TEST_F(DebugConfigTest, testSetConfig) EXPECT_CALL(mock_rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true)); env.preload(); - Singleton::Consume::from(env)->registerValue("Executable Name", "debug-ut"); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "debug-ut"); env.init(); Debug::init(); diff --git a/core/environment/environment.cc b/core/environment/environment.cc index c65fcca..0a29ef5 100644 --- a/core/environment/environment.cc +++ b/core/environment/environment.cc @@ -265,18 +265,9 @@ Environment::Impl::getCurrentHeadersMap() tracing_headers["X-Span-Id"] = span_id; } - auto exec_name = get("Executable Name"); + auto exec_name = get("Base Executable Name"); if (exec_name.ok() && *exec_name != "") { - string executable_name = *exec_name; - auto file_path_end = executable_name.find_last_of("/"); - if (file_path_end != string::npos) { - executable_name = executable_name.substr(file_path_end + 1); - } - auto file_sufix_start = executable_name.find_first_of("."); - if (file_sufix_start != string::npos) { - executable_name = executable_name.substr(0, file_sufix_start); - } - tracing_headers["X-Calling-Service"] = executable_name; + tracing_headers["X-Calling-Service"] = *exec_name; } return tracing_headers; diff --git a/core/environment/environment_ut/tracing_ut.cc b/core/environment/environment_ut/tracing_ut.cc index ebf0a1d..f065e10 100644 --- a/core/environment/environment_ut/tracing_ut.cc +++ b/core/environment/environment_ut/tracing_ut.cc @@ -346,7 +346,7 @@ TEST_F(TracingCompRoutinesTest, 2SpansDifFlow) { I_MainLoop::Routine routine = [&] () { string service_name = "test-service-name"; - i_env->registerValue("Executable Name", service_name); + i_env->registerValue("Base Executable Name", service_name); i_env->startNewTrace(true, "a687b388-1108-4083-9852-07c33b1074e9"); trace_id = i_env->getCurrentTrace(); diff --git a/core/include/services_sdk/interfaces/i_agent_details.h b/core/include/services_sdk/interfaces/i_agent_details.h index d5bae32..eb66943 100644 --- a/core/include/services_sdk/interfaces/i_agent_details.h +++ b/core/include/services_sdk/interfaces/i_agent_details.h @@ -46,6 +46,7 @@ public: virtual OrchestrationMode getOrchestrationMode() const = 0; virtual std::string getAccessToken() const = 0; virtual void loadAccessToken() = 0; + virtual bool isOpenAppsecAgent() const = 0; // OpenSSL virtual void setOpenSSLDir(const std::string &openssl_dir) = 0; diff --git a/core/include/services_sdk/interfaces/i_env_details.h b/core/include/services_sdk/interfaces/i_env_details.h index f1b5d04..e85595e 100644 --- a/core/include/services_sdk/interfaces/i_env_details.h +++ b/core/include/services_sdk/interfaces/i_env_details.h @@ -24,6 +24,7 @@ class I_EnvDetails public: virtual EnvType getEnvType() = 0; virtual std::string getToken() = 0; + virtual std::string getNameSpace() = 0; protected: virtual ~I_EnvDetails() {} diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h index 5e0407c..6cf7ed9 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -215,6 +215,18 @@ public: return is_to_fog; } + void + setSniHostName(const std::string &_host_name) + { + sni_host_name = _host_name; + } + + Maybe + getSniHostName() const + { + return sni_host_name; + } + template void serialize(Archive &ar) @@ -237,6 +249,7 @@ public: private: std::string host_name = ""; + Maybe sni_host_name = genError("SNI host name not set"); std::string ca_path = ""; std::string client_cert_path = ""; std::string client_key_path = ""; diff --git a/core/include/services_sdk/interfaces/mock/mock_agent_details.h b/core/include/services_sdk/interfaces/mock/mock_agent_details.h index 37c171b..e6b35e6 100644 --- a/core/include/services_sdk/interfaces/mock/mock_agent_details.h +++ b/core/include/services_sdk/interfaces/mock/mock_agent_details.h @@ -28,6 +28,7 @@ public: MOCK_CONST_METHOD0(getAgentId, std::string()); MOCK_METHOD0(loadAccessToken, void()); MOCK_CONST_METHOD0(getAccessToken, std::string()); + MOCK_CONST_METHOD0(isOpenAppsecAgent, bool()); // OpenSSL MOCK_METHOD1(setOpenSSLDir, void(const std::string&)); diff --git a/core/include/services_sdk/resources/agent_details.h b/core/include/services_sdk/resources/agent_details.h index 5d026e3..c428c91 100644 --- a/core/include/services_sdk/resources/agent_details.h +++ b/core/include/services_sdk/resources/agent_details.h @@ -73,6 +73,7 @@ public: Maybe getOpenSSLDir() const; std::string getClusterId() const; OrchestrationMode getOrchestrationMode() const; + bool isOpenAppsecAgent() const; std::string getAccessToken() const; void loadAccessToken(); diff --git a/core/include/services_sdk/resources/component_is/components_list_impl.h b/core/include/services_sdk/resources/component_is/components_list_impl.h index e9c6c48..c5b4342 100644 --- a/core/include/services_sdk/resources/component_is/components_list_impl.h +++ b/core/include/services_sdk/resources/component_is/components_list_impl.h @@ -165,6 +165,14 @@ public: } registerGlobalValue("Executable Name", arg_vec.front()); + + auto file_path_end = arg_vec.front().find_last_of("/"); + auto executable_name = arg_vec.front().substr(file_path_end + 1); + auto file_sufix_start = executable_name.find_first_of("."); + if (file_sufix_start != std::string::npos) { + executable_name = executable_name.substr(0, file_sufix_start); + } + registerGlobalValue("Base Executable Name", executable_name); } void diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 196b7d9..0e0ca5c 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -76,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP) DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP) DEFINE_FLAG(D_WAAP_SCANNER, D_WAAP) + DEFINE_FLAG(D_WAAP_MODEL_LOGGER, D_WAAP) DEFINE_FLAG(D_WAAP_DEEP_PARSER, D_WAAP) DEFINE_FLAG(D_WAAP_BASE64, D_WAAP) DEFINE_FLAG(D_WAAP_JSON, D_WAAP) @@ -190,6 +191,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_L7_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT) + DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT) DEFINE_FLAG(D_FLOW, D_ALL) DEFINE_FLAG(D_DROP, D_FLOW) diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h index 69771e9..4414de3 100644 --- a/core/include/services_sdk/resources/generic_metric.h +++ b/core/include/services_sdk/resources/generic_metric.h @@ -24,6 +24,7 @@ #include "i_mainloop.h" #include "i_time_get.h" #include "i_agent_details.h" +#include "i_instance_awareness.h" #include "i_environment.h" #include "i_messaging.h" #include "i_rest_api.h" @@ -52,6 +53,7 @@ class GenericMetric Singleton::Consume, Singleton::Consume, Singleton::Consume, + Singleton::Consume, Singleton::Consume, Singleton::Consume, Singleton::Consume, @@ -111,7 +113,9 @@ private: void handleMetricStreamSending(); void generateLog(); + void generatePrometheus(); void generateDebug(); + void generateAiopsLog(); I_MainLoop *i_mainloop; I_TimeGet *i_time; diff --git a/core/include/services_sdk/resources/log_generator.h b/core/include/services_sdk/resources/log_generator.h index dfbd76a..8d42618 100644 --- a/core/include/services_sdk/resources/log_generator.h +++ b/core/include/services_sdk/resources/log_generator.h @@ -102,6 +102,8 @@ public: std::string getLogInsteadOfSending(); + void addMarkerSuffix(const std::string &suffix); + private: std::chrono::microseconds getCurrentTime() const; void loadBaseLogFields(); diff --git a/core/include/services_sdk/resources/metric/average.h b/core/include/services_sdk/resources/metric/average.h index a904106..1e61f10 100644 --- a/core/include/services_sdk/resources/metric/average.h +++ b/core/include/services_sdk/resources/metric/average.h @@ -54,6 +54,12 @@ public: return (count > 0) ? double(sum)/count : 0; } + float + getValue() const override + { + return static_cast(getAverage()); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/counter.h b/core/include/services_sdk/resources/metric/counter.h index c4c56a1..24255e8 100644 --- a/core/include/services_sdk/resources/metric/counter.h +++ b/core/include/services_sdk/resources/metric/counter.h @@ -44,6 +44,12 @@ public: return counter; } + float + getValue() const override + { + return static_cast(counter); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/last_reported_value.h b/core/include/services_sdk/resources/metric/last_reported_value.h index 2e53ffb..c88db4b 100644 --- a/core/include/services_sdk/resources/metric/last_reported_value.h +++ b/core/include/services_sdk/resources/metric/last_reported_value.h @@ -44,6 +44,12 @@ public: return last_reported; } + float + getValue() const override + { + return static_cast(last_reported); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/max.h b/core/include/services_sdk/resources/metric/max.h index 277c73b..81f0e77 100644 --- a/core/include/services_sdk/resources/metric/max.h +++ b/core/include/services_sdk/resources/metric/max.h @@ -56,6 +56,12 @@ public: return max; } + float + getValue() const override + { + return static_cast(max); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/metric_calc.h b/core/include/services_sdk/resources/metric/metric_calc.h index 8cd6b42..836f6dd 100644 --- a/core/include/services_sdk/resources/metric/metric_calc.h +++ b/core/include/services_sdk/resources/metric/metric_calc.h @@ -18,21 +18,129 @@ #error metric/metric_calc.h should not be included directly #endif // __GENERIC_METRIC_H_ +#include #include #include "report/report.h" +#include "customized_cereal_map.h" class GenericMetric; enum class MetricType { GAUGE, COUNTER }; +struct PrometheusData +{ + std::string name; + std::string type; + std::string desc; + std::string label; + std::string value; +}; + +class AiopsMetricData +{ +public: + AiopsMetricData( + const std::string &_name, + const std::string &_type, + const std::string &_units, + const std::string &_description, + std::map _resource_attributes, + float _value) + : + name(_name), + type(_type), + units(_units), + description(_description), + resource_attributes(_resource_attributes), + value(_value) + { + timestamp = Singleton::Consume::by()->getWalltimeStr(); + asset_id = Singleton::Consume::by()->getAgentId(); + } + + void + serialize(cereal::JSONOutputArchive &ar) const + { + ar(cereal::make_nvp("Timestamp", timestamp)); + ar(cereal::make_nvp("MetricName", name)); + ar(cereal::make_nvp("MetricType", type)); + ar(cereal::make_nvp("MetricUnit", units)); + ar(cereal::make_nvp("MetricDescription", description)); + ar(cereal::make_nvp("MetricValue", value)); + ar(cereal::make_nvp("ResourceAttributes", resource_attributes)); + ar(cereal::make_nvp("MetricAttributes", metric_attributes)); + ar(cereal::make_nvp("AssetID", asset_id)); + } + + std::string + toString() const + { + std::stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + serialize(ar); + } + return ss.str(); + } + + void + addMetricAttribute(const std::string &label, const std::string &value) + { + metric_attributes[label] = value; + } + +private: + std::string timestamp = ""; + std::string asset_id = ""; + std::string name; + std::string type; + std::string units; + std::string description; + std::map resource_attributes; + std::map metric_attributes; + float value = 0; +}; + +class AiopsMetricList +{ +public: + void + addMetrics(const std::vector &_metrics) + { + metrics.insert(metrics.end(), _metrics.begin(), _metrics.end()); + } + + void + serialize(cereal::JSONOutputArchive &ar) const + { + ar(cereal::make_nvp("Metrics", metrics)); + } + +// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage + std::string + toString() const + { + std::stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + serialize(ar); + } + return ss.str(); + } +// LCOV_EXCL_STOP + +private: + std::vector metrics; +}; + class MetricCalc { public: template MetricCalc(GenericMetric *metric, const std::string &calc_title, const Args & ... args) { - setMetadata("BaseName", calc_title); + setMetricName(calc_title); addMetric(metric); parseMetadata(args ...); } @@ -47,7 +155,11 @@ public: std::string getMetircDescription() const { return getMetadata("Description"); } std::string getMetadata(const std::string &metadata) const; virtual MetricType getMetricType() const { return MetricType::GAUGE; } + virtual std::vector getPrometheusMetrics() const; + virtual float getValue() const = 0; + virtual std::vector getAiopsMetrics() const; + void setMetricName(const std::string &name) { setMetadata("BaseName", name); } void setMetricDotName(const std::string &name) { setMetadata("DotName", name); } void setMetircUnits(const std::string &units) { setMetadata("Units", units); } void setMetircDescription(const std::string &description) { setMetadata("Description", description); } @@ -55,6 +167,7 @@ public: protected: void addMetric(GenericMetric *metric); + std::map getBasicLabels() const; template void diff --git a/core/include/services_sdk/resources/metric/metric_map.h b/core/include/services_sdk/resources/metric/metric_map.h index 845e54f..e23982c 100644 --- a/core/include/services_sdk/resources/metric/metric_map.h +++ b/core/include/services_sdk/resources/metric/metric_map.h @@ -54,6 +54,37 @@ class MetricMap : public MetricCalc return first->second.getMetricType(); } + std::vector + getPrometheusMetrics(const std::string &label, const std::string &name) const + { + std::vector res; + + for (auto &metric : inner_map) { + auto sub_res = metric.second.getPrometheusMetrics(); + for (auto &sub_metric : sub_res) { + sub_metric.label += "," + label + "=\"" + metric.first + "\""; + sub_metric.name = name; + } + res.insert(res.end(), sub_res.begin(), sub_res.end()); + } + + return res; + } + + std::vector + getAiopsMetrics(const std::string &label) const + { + std::vector aiops_metrics; + for (auto &metric : inner_map) { + auto metric_data = metric.second.getAiopsMetrics(); + for (auto &sub_metric : metric_data) { + sub_metric.addMetricAttribute(label, metric.first); + } + aiops_metrics.insert(aiops_metrics.end(), metric_data.begin(), metric_data.end()); + } + return aiops_metrics; + } + typename std::map::const_iterator begin() const { return inner_map.begin(); } typename std::map::const_iterator end() const { return inner_map.end(); } @@ -63,9 +94,17 @@ class MetricMap : public MetricCalc public: template - MetricMap(GenericMetric *metric, const std::string &title, const Args & ... args) + MetricMap( + const Metric &sub_metric, + GenericMetric *metric, + const std::string &l, + const std::string &title, + const Args & ... args + ) : - MetricCalc(metric, title, args ...) + MetricCalc(metric, title, args ...), + base_metric(sub_metric), + label(l) { } @@ -75,6 +114,14 @@ public: if (getMetricType() == MetricType::GAUGE) metric_map.clear(); } +// LCOV_EXCL_START Reason: Covered by printPromeathusMultiMap unit-test, but misdetected by the coverage + float + getValue() const override + { + return std::nanf(""); + } +// LCOV_EXCL_STOP + void save(cereal::JSONOutputArchive &ar) const override { @@ -89,7 +136,9 @@ public: { std::stringstream string_key; string_key << key; - auto metric = metric_map.emplace(string_key.str(), Metric(nullptr, string_key.str())).first; + auto new_metric = base_metric; + new_metric.setMetricName(string_key.str()); + auto metric = metric_map.emplace(string_key.str(), std::move(new_metric)).first; metric->second.report(new_values...); } @@ -105,8 +154,22 @@ public: return field; } + std::vector + getPrometheusMetrics() const override + { + return metric_map.getPrometheusMetrics(label, getMetricName()); + } + + std::vector + getAiopsMetrics() const + { + return metric_map.getAiopsMetrics(label); + } + private: InnerMap metric_map; + Metric base_metric; + std::string label; }; } // namespace MetricCalculations diff --git a/core/include/services_sdk/resources/metric/min.h b/core/include/services_sdk/resources/metric/min.h index 5a2a160..77f3b85 100644 --- a/core/include/services_sdk/resources/metric/min.h +++ b/core/include/services_sdk/resources/metric/min.h @@ -56,6 +56,12 @@ public: return min; } + float + getValue() const override + { + return static_cast(min); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/no_reset_counter.h b/core/include/services_sdk/resources/metric/no_reset_counter.h index e1538de..5d50cd6 100644 --- a/core/include/services_sdk/resources/metric/no_reset_counter.h +++ b/core/include/services_sdk/resources/metric/no_reset_counter.h @@ -41,6 +41,12 @@ public: return counter; } + float + getValue() const override + { + return static_cast(counter); + } + void save(cereal::JSONOutputArchive &ar) const override { diff --git a/core/include/services_sdk/resources/metric/top_values.h b/core/include/services_sdk/resources/metric/top_values.h index b33d02f..31ce84f 100644 --- a/core/include/services_sdk/resources/metric/top_values.h +++ b/core/include/services_sdk/resources/metric/top_values.h @@ -59,6 +59,12 @@ public: values.clear(); } + float + getValue() const override + { + return std::nanf(""); + } + std::vector getTopValues() const { diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 35bb241..9e9aadc 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -485,6 +485,7 @@ private: MessageMetadata req_md(server, *port); req_md.insertHeaders(getHTTPHeaders()); req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); return sendIntelligenceRequestImpl(rest_req, req_md); } diff --git a/core/logging/log_generator.cc b/core/logging/log_generator.cc index b79111d..1c80091 100644 --- a/core/logging/log_generator.cc +++ b/core/logging/log_generator.cc @@ -123,3 +123,12 @@ LogGen::loadBaseLogFields() log.getMarkers() = env->getAllStrings(EnvKeyAttr::LogSection::MARKER); } + +// Adding a suffix to log markers will allow for creating a unique log that won't be reduced +void +LogGen::addMarkerSuffix(const string &suffix) +{ + for (auto &marker : log.getMarkers()) { + marker.second += suffix; + } +} diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index 0e02dee..e5ac57e 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -96,6 +96,8 @@ public: client_key_path = metadata.getClientKeyPath(); is_dual_auth = true; } + + sni_hostname = metadata.getSniHostName(); } void @@ -329,6 +331,12 @@ private: if (!SSL_set1_host(ssl_socket, host)) { return genError("Failed to set host name verification. Host: " + string(host)); } + + if (sni_hostname.ok()) { + host = sni_hostname->c_str(); + } + + dbgDebug(D_CONNECTION) << "Setting TLS host name extension. Host: " << host; if (!SSL_set_tlsext_host_name(ssl_socket, host)) { return genError("Failed to set TLS host name extension. Host: " + string(host)); } @@ -656,7 +664,7 @@ private: printOut(const string &data) { string type = getConfigurationWithDefault("chopped", "message", "Data printout type"); - uint length = getConfigurationWithDefault(10, "message", "Data printout length"); + uint length = getConfigurationWithDefault(50, "message", "Data printout length"); if (type == "full") return data; if (type == "size") return to_string(data.size()) + " bytes"; if (type == "none") return ""; @@ -689,6 +697,7 @@ private: bool lock = false; bool should_close_connection = false; bool is_dual_auth = false; + Maybe sni_hostname = genError("Uninitialized"); }; Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata) diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc index 59c8a97..6505a48 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc @@ -112,9 +112,8 @@ MessagingBufferComponent::Impl::init() auto sub_path = getProfileAgentSettingWithDefault("nano_agent/event_buffer/", "eventBuffer.baseFolder"); buffer_root_path = getLogFilesPathConfig() + "/" + sub_path; - string full_executable_name = - Singleton::Consume::by()->get("Executable Name").unpack(); - string executable_name = full_executable_name.substr(full_executable_name.find_last_of("/") + 1); + string executable_name = + Singleton::Consume::by()->get("Base Executable Name").unpack(); removeLegacyBuffer(buffer_root_path, executable_name); mkdir(buffer_root_path.c_str(), 0644); diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc index 40323ba..0a87475 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc @@ -26,7 +26,7 @@ public: TestMessagingBuffer() { env.preload(); - Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "tmp_test_file"); config.preload(); config.init(); diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 15f9098..39ed949 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -17,6 +17,7 @@ #include "debug.h" #include "report/log_rest.h" #include "config.h" +#include "report_messaging.h" #include #include @@ -30,6 +31,24 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; } MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; } +// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage +static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); } +// LCOV_EXCL_STOP + +vector +MetricCalc::getAiopsMetrics() const +{ + float value = getValue(); + if (isnan(value)) return {}; + + string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName(); + string units = getMetircUnits(); + string description = getMetircDescription(); + string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter"; + + return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) }; +} + string MetricCalc::getMetadata(const string &key) const { @@ -54,6 +73,54 @@ MetricCalc::addMetric(GenericMetric *metric) if (metric != nullptr) metric->addCalc(this); } +vector +MetricCalc::getPrometheusMetrics() const +{ + float value = getValue(); + if (isnan(value)) return {}; + + PrometheusData res; + + res.name = getMetricDotName() != "" ? getMetricDotName() : getMetricName(); + res.type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter"; + res.desc = getMetircDescription(); + + stringstream labels; + const auto &label_pairs = getBasicLabels(); + bool first = true; + for (auto &pair : label_pairs) { + if (!first) labels << ','; + labels << pair.first << "=\"" << pair.second << '"'; + first = false; + } + res.label = labels.str(); + + stringstream value_str; + value_str << value; + res.value = value_str.str(); + + return {res}; +} + +map +MetricCalc::getBasicLabels() const +{ + map res; + + auto i_instance = Singleton::Consume::by(); + auto id = i_instance->getUniqueID(); + if (id.ok()) res["id"] = *id; + + auto details = Singleton::Consume::by(); + res["agent"] = details->getAgentId(); + + auto env = Singleton::Consume::by(); + auto executable = env->get("Base Executable Name"); + if (executable.ok()) res["process"] = *executable; + + return res; +} + static const string metric_file = "/tmp/metrics_output.txt"; class GenericMetric::MetricsRest : public ServerRest @@ -121,7 +188,9 @@ void GenericMetric::handleMetricStreamSending() { if (active_streams.isSet(Stream::DEBUG)) generateDebug(); + if (active_streams.isSet(Stream::PROMETHEUS)) generatePrometheus(); if (active_streams.isSet(Stream::FOG)) generateLog(); + if (active_streams.isSet(Stream::AIOPS)) generateAiopsLog(); if (reset) resetMetrics(); } @@ -244,6 +313,106 @@ GenericMetric::generateLog() sendLog(metric_client_rest); } +class PrometheusRest : public ClientRest +{ + class Metric : public ClientRest + { + public: + Metric(const string &n, const string &t, const string &d, const string &l, const string &v) + : + name(n), + type(t), + description(d), + labels(l), + value(v) + {} + + private: + C2S_PARAM(string, name); + C2S_PARAM(string, type); + C2S_PARAM(string, description); + C2S_PARAM(string, labels); + C2S_PARAM(string, value); + }; + +public: + PrometheusRest() : metrics(vector()) {} + + void + addMetric(const vector &vec) + { + auto &metric_vec = metrics.get(); + metric_vec.reserve(vec.size()); + for (auto &metric : vec) { + metric_vec.emplace_back(metric.name, metric.type, metric.desc, "{" + metric.label + "}", metric.value); + } + } + +private: + C2S_PARAM(vector, metrics); +}; + +void +GenericMetric::generatePrometheus() +{ + if (!getProfileAgentSettingWithDefault(false, "prometheus")) return; + + vector all_metrics; + for (auto &calc : calcs) { + const auto &cal_metrics = calc->getPrometheusMetrics(); + all_metrics.insert(all_metrics.end(), cal_metrics.begin(), cal_metrics.end()); + } + + PrometheusRest rest; + rest.addMetric(all_metrics); + + MessageMetadata new_config_req_md("127.0.0.1", 7465); + new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); + new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + Singleton::Consume::by()->sendSyncMessage( + HTTPMethod::POST, + "/set-prometheus-data", + rest, + MessageCategory::GENERIC, + new_config_req_md + ); +} + +void +GenericMetric::generateAiopsLog() +{ + if (!getConfigurationWithDefault(true, "metric", "aiopsMetricSendEnable")) return; + + AiopsMetricList aiops_metrics; + + for (auto &calc : calcs) { + aiops_metrics.addMetrics(calc->getAiopsMetrics()); + } + + set tags; + Report metric_to_fog( + "AIOPS Metric Data", + Singleton::Consume::by()->getWalltime(), + Type::PERIODIC, + Level::LOG, + LogLevel::INFO, + audience, + team, + Severity::INFO, + Priority::LOW, + report_interval, + LogField("agentId", Singleton::Consume::by()->getAgentId()), + tags, + Tags::INFORMATIONAL, + issuing_engine + ); + + metric_to_fog << LogField("eventObject", aiops_metrics); + + LogRest metric_client_rest(metric_to_fog); + sendLog(metric_client_rest); +} + void GenericMetric::generateDebug() { @@ -270,6 +439,7 @@ GenericMetric::preload() { registerExpectedConfiguration("metric", "fogMetricSendEnable"); registerExpectedConfiguration("metric", "debugMetricSendEnable"); + registerExpectedConfiguration("metric", "aiopsMetricSendEnable"); registerExpectedConfiguration("metric", "fogMetricUri"); registerExpectedConfiguration("metric", "metricsOutputTmpFile"); } diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index e96520c..a8d145a 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -10,6 +10,7 @@ #include "mock/mock_rest_api.h" #include "agent_details.h" #include "mock/mock_messaging.h" +#include "mock/mock_instance_awareness.h" #include "config.h" #include "config_component.h" @@ -67,6 +68,14 @@ public: top_usage.report(event.getCPU()); } + void + setAiopsMetric() + { + turnOnStream(Stream::AIOPS); + max.setMetricDotName("cpu.max"); + max.setMetircUnits("percrnt"); + } + Max max{this, "cpuMax"}; Min min{this, "cpuMin"}; Average avg{this, "cpuAvg"}; @@ -121,13 +130,15 @@ public: class HttpTransaction : public Event { public: - HttpTransaction(const string &_url, uint _bytes) : url(_url), bytes(_bytes) {} + HttpTransaction(const string &_url, const string &m, uint _bytes) : url(_url), method(m), bytes(_bytes) {} - const string & getUrl() const { return url;} + const string & getUrl() const { return url; } + const string & getMethod() const { return method; } uint getBytes() const { return bytes; } private: string url; + string method; uint bytes; }; @@ -144,9 +155,33 @@ public: total.report(event.getUrl(), 1); } + void + setAiopsMetric() + { + turnOnStream(Stream::AIOPS); + } + private: - MetricMap> avg{this, "PerUrlAvg"}; - MetricMap total{this, "TotalRequests"}; + MetricMap> avg{Average{nullptr, ""}, this, "url", "PerUrlAvg"}; + MetricMap total{NoResetCounter{nullptr, ""}, this, "url", "TotalRequests"}; +}; + +class UrlMetric2 : public GenericMetric, public Listener +{ +public: + void + upon(const HttpTransaction &event) override + { + total.report(event.getUrl(), event.getMethod(), 1); + } + +private: + MetricMap> total{ + MetricMap{NoResetCounter{nullptr, ""}, nullptr, "method", ""}, + this, + "url", + "request.total" + }; }; class MetricTest : public Test @@ -155,6 +190,8 @@ public: MetricTest() { EXPECT_CALL(rest, mockRestCall(RestAction::ADD, "declare-boolean-variable", _)).WillOnce(Return(true)); + ON_CALL(instance, getUniqueID()).WillByDefault(Return(string("87"))); + ON_CALL(instance, getFamilyID()).WillByDefault(Return(string(""))); env.init(); Debug::setNewDefaultStdout(&debug_output); Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE); @@ -177,6 +214,7 @@ public: StrictMock mock_ml; NiceMock timer; + NiceMock instance; StrictMock rest; ::Environment env; ConfigComponent conf; @@ -255,7 +293,9 @@ TEST_F(MetricTest, basicMetricTest) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 89,\n" @@ -326,7 +366,9 @@ TEST_F(MetricTest, basicMetricTest) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 90,\n" @@ -399,7 +441,9 @@ TEST_F(MetricTest, basicMetricTest) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 100,\n" @@ -485,6 +529,150 @@ TEST_F(MetricTest, printMetricsTest) GenericMetric::fini(); } +TEST_F(MetricTest, printPromeathus) +{ + conf.preload(); + + stringstream configuration; + configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n"; + + EXPECT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(configuration)); + + CPUMetric cpu_mt; + cpu_mt.init( + "CPU usage", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + seconds(5), + false + ); + cpu_mt.turnOffStream(GenericMetric::Stream::FOG); + cpu_mt.turnOffStream(GenericMetric::Stream::DEBUG); + cpu_mt.turnOnStream(GenericMetric::Stream::PROMETHEUS); + cpu_mt.registerListener(); + + CPUEvent cpu_event; + cpu_event.setProcessCPU(89); + cpu_event.notify(); + + string message_body; + EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse()))); + routine(); + + string res = + "{\n" + " \"metrics\": [\n" + " {\n" + " \"name\": \"cpuMax\",\n" + " \"type\": \"gauge\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"89\"\n" + " },\n" + " {\n" + " \"name\": \"cpuMin\",\n" + " \"type\": \"gauge\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"89\"\n" + " },\n" + " {\n" + " \"name\": \"cpuAvg\",\n" + " \"type\": \"gauge\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"89\"\n" + " },\n" + " {\n" + " \"name\": \"cpuCurrent\",\n" + " \"type\": \"gauge\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"89\"\n" + " },\n" + " {\n" + " \"name\": \"cpuCounter\",\n" + " \"type\": \"gauge\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"1\"\n" + " },\n" + " {\n" + " \"name\": \"cpuTotalCounter\",\n" + " \"type\": \"counter\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" + " \"value\": \"1\"\n" + " }\n" + " ]\n" + "}"; + + EXPECT_EQ(message_body, res); +} + +TEST_F(MetricTest, printPromeathusMultiMap) +{ + conf.preload(); + + stringstream configuration; + configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n"; + + EXPECT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(configuration)); + + UrlMetric2 metric; + metric.init( + "Bytes per URL", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + seconds(5), + true + ); + metric.turnOnStream(GenericMetric::Stream::PROMETHEUS); + metric.registerListener(); + + HttpTransaction("/index.html", "GET", 10).notify(); + HttpTransaction("/index2.html", "GET", 20).notify(); + HttpTransaction("/index.html", "POST", 40).notify(); + + string message_body; + EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _)) + .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse()))); + routine(); + + string res = + "{\n" + " \"metrics\": [\n" + " {\n" + " \"name\": \"request.total\",\n" + " \"type\": \"counter\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," + "method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n" + " \"value\": \"1\"\n" + " },\n" + " {\n" + " \"name\": \"request.total\",\n" + " \"type\": \"counter\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," + "method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n" + " \"value\": \"1\"\n" + " },\n" + " {\n" + " \"name\": \"request.total\",\n" + " \"type\": \"counter\",\n" + " \"description\": \"\",\n" + " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," + "method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n" + " \"value\": \"1\"\n" + " }\n" + " ]\n" + "}"; + + EXPECT_EQ(message_body, res); +} + TEST_F(MetricTest, metricTestWithReset) { CPUMetric cpu_mt; @@ -554,7 +742,9 @@ TEST_F(MetricTest, metricTestWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 89,\n" @@ -624,7 +814,9 @@ TEST_F(MetricTest, metricTestWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 90,\n" @@ -694,7 +886,9 @@ TEST_F(MetricTest, metricTestWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 100,\n" @@ -796,7 +990,9 @@ TEST_F(MetricTest, generateReportWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 89,\n" @@ -888,7 +1084,9 @@ TEST_F(MetricTest, generateReportWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"Unnamed Nano Service\"\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 90,\n" @@ -959,7 +1157,9 @@ TEST_F(MetricTest, generateReportWithReset) " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" - " \"serviceName\": \"My named nano service\"\n" + " \"serviceName\": \"My named nano service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" " },\n" " \"eventData\": {\n" " \"cpuMax\": 100,\n" @@ -1015,6 +1215,7 @@ TEST_F(MetricTest, allMetricTest) seconds(5), false ); + msg_size_mt.registerListener(); EXPECT_EQ(msg_size_mt.getMetricName(), "Message size"); @@ -1074,9 +1275,9 @@ TEST_F(MetricTest, testMapMetric) ); url_mt.registerListener(); - HttpTransaction("/index.html", 10).notify(); - HttpTransaction("/index2.html", 20).notify(); - HttpTransaction("/index.html", 40).notify(); + HttpTransaction("/index.html", "GET", 10).notify(); + HttpTransaction("/index2.html", "GET", 20).notify(); + HttpTransaction("/index.html", "POST", 40).notify(); string message_body; @@ -1191,3 +1392,309 @@ TEST_F(MetricTest, testManyValuesOutOfOrder) EXPECT_THAT(AllMetricEvent().query(), ElementsAre(HasSubstr(cpu_str))); } + +TEST_F(MetricTest, basicAIOPSMetricTest) +{ + EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087"))); + CPUMetric cpu_mt; + cpu_mt.init( + "CPU usage", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + seconds(5), + false + ); + cpu_mt.setAiopsMetric(); + cpu_mt.registerListener(); + + EXPECT_EQ(cpu_mt.getMetricName(), "CPU usage"); + EXPECT_EQ(cpu_mt.getReportInterval().count(), 5); + + routine(); + + CPUEvent cpu_event; + cpu_event.setProcessCPU(89); + cpu_event.notify(); + + string metric_str = + "{\n" + " \"Metric\": \"CPU usage\",\n" + " \"Reporting interval\": 5,\n" + " \"cpuMax\": 89.0,\n" + " \"cpuMin\": 89.0,\n" + " \"cpuAvg\": 89.0,\n" + " \"cpuCurrent\": 89.0,\n" + " \"cpuCounter\": 1,\n" + " \"cpuTotalCounter\": 1,\n" + " \"cpuTops\": [\n" + " 89.0\n" + " ]\n" + "}"; + + string message_body; + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); + + string expected_message = + "{\n" + " \"log\": {\n" + " \"eventTime\": \"2016-11-13T17:31:24.087\",\n" + " \"eventName\": \"AIOPS Metric Data\",\n" + " \"eventSeverity\": \"Info\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Periodic\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"Agent Core\",\n" + " \"eventFrequency\": 5,\n" + " \"eventTags\": [\n" + " \"Informational\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"issuingEngine\": \"Agent Core\",\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" + " },\n" + " \"eventData\": {\n" + " \"eventObject\": {\n" + " \"Metrics\": [\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpu.max\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"percrnt\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 89.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpuMin\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 89.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpuAvg\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 89.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpuCurrent\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 89.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpuCounter\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 1.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"cpuTotalCounter\",\n" + " \"MetricType\": \"counter\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 1.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {},\n" + " \"AssetID\": \"Unknown\"\n" + " }\n" + " ]\n" + " }\n" + " }\n" + " }\n" + "}"; + + routine(); + EXPECT_THAT(debug_output.str(), HasSubstr(metric_str)); + EXPECT_EQ(message_body, expected_message); + debug_output.str(""); +} + +TEST_F(MetricTest, testAIOPSMapMetric) +{ + EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087"))); + UrlMetric url_mt; + url_mt.init( + "Bytes per URL", + ReportIS::AudienceTeam::AGENT_CORE, + ReportIS::IssuingEngine::AGENT_CORE, + seconds(5), + true + ); + url_mt.registerListener(); + + url_mt.setAiopsMetric(); + + HttpTransaction("/index.html", "GET", 10).notify(); + HttpTransaction("/index2.html", "GET", 20).notify(); + HttpTransaction("/index.html", "POST", 40).notify(); + + string message_body; + + EXPECT_CALL(messaging_mock, sendAsyncMessage( + _, + "/api/v1/agents/events", + _, + MessageCategory::METRIC, + _, + _ + )).WillRepeatedly(SaveArg<2>(&message_body)); + routine(); + + string expected_message = + "{\n" + " \"log\": {\n" + " \"eventTime\": \"2016-11-13T17:31:24.087\",\n" + " \"eventName\": \"AIOPS Metric Data\",\n" + " \"eventSeverity\": \"Info\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Periodic\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"Agent Core\",\n" + " \"eventFrequency\": 5,\n" + " \"eventTags\": [\n" + " \"Informational\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"agentId\": \"Unknown\",\n" + " \"issuingEngine\": \"Agent Core\",\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"Unnamed Nano Service\",\n" + " \"serviceId\": \"87\",\n" + " \"serviceFamilyId\": \"\"\n" + " },\n" + " \"eventData\": {\n" + " \"eventObject\": {\n" + " \"Metrics\": [\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"/index.html\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 25.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {\n" + " \"url\": \"/index.html\"\n" + " },\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"/index2.html\",\n" + " \"MetricType\": \"gauge\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 20.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {\n" + " \"url\": \"/index2.html\"\n" + " },\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"/index.html\",\n" + " \"MetricType\": \"counter\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 2.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {\n" + " \"url\": \"/index.html\"\n" + " },\n" + " \"AssetID\": \"Unknown\"\n" + " },\n" + " {\n" + " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" + " \"MetricName\": \"/index2.html\",\n" + " \"MetricType\": \"counter\",\n" + " \"MetricUnit\": \"\",\n" + " \"MetricDescription\": \"\",\n" + " \"MetricValue\": 1.0,\n" + " \"ResourceAttributes\": {\n" + " \"agent\": \"Unknown\",\n" + " \"id\": \"87\"\n" + " },\n" + " \"MetricAttributes\": {\n" + " \"url\": \"/index2.html\"\n" + " },\n" + " \"AssetID\": \"Unknown\"\n" + " }\n" + " ]\n" + " }\n" + " }\n" + " }\n" + "}"; + + EXPECT_EQ(message_body, expected_message); +} diff --git a/core/rest/rest_conn.cc b/core/rest/rest_conn.cc index 4d06140..5e2d843 100644 --- a/core/rest/rest_conn.cc +++ b/core/rest/rest_conn.cc @@ -23,11 +23,12 @@ using namespace std; USE_DEBUG_FLAG(D_API); -RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke) +RestConn::RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external) : - fd(_fd), - mainloop(_mainloop), - invoke(_invoke) + fd(_fd), + mainloop(_mainloop), + invoke(_invoke), + is_external_ip(is_external) {} RestConn::~RestConn() @@ -101,6 +102,12 @@ RestConn::parseConn() const return sendResponse("200 OK", invoke->invokeGet(identifier), false); } + if (is_external_ip) { + dbgWarning(D_API) << "External IP tried to POST"; + sendResponse("500 Internal Server Error", "", false); + stop(); + } + stringstream body; body.str(readSize(len)); diff --git a/core/rest/rest_conn.h b/core/rest/rest_conn.h index bb7ecfa..c7489ef 100644 --- a/core/rest/rest_conn.h +++ b/core/rest/rest_conn.h @@ -21,7 +21,7 @@ class RestConn { public: - RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke); + RestConn(int _fd, I_MainLoop *_mainloop, const I_RestInvoke *_invoke, bool is_external = false); ~RestConn(); void parseConn() const; @@ -35,6 +35,7 @@ private: int fd; I_MainLoop *mainloop; const I_RestInvoke *invoke; + bool is_external_ip = false; }; #endif // __REST_CONN_H__ diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index 2ce76c8..159953a 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -47,6 +47,7 @@ public: void startNewConnection() const; bool bindRestServerSocket(struct sockaddr_in &addr, vector port_range); + bool bindRestServerSocket(struct sockaddr_in6 &addr, vector port_range); bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; bool addGetCall(const string &uri, const function &cb) override; uint16_t getListeningPort() const override { return listening_port; } @@ -73,10 +74,36 @@ private: bool RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector port_range) { + dbgFlow(D_API) << "Binding IPv4 socket"; for (uint16_t port : port_range) { addr.sin_port = htons(port); if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true; + + if (errno == EADDRINUSE) { + dbgDebug(D_API) << "Port " << port << " is already in use"; + } else { + dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + } + } + + return false; +} + +bool +RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector port_range) +{ + dbgFlow(D_API) << "Binding IPv6 socket"; + for (uint16_t port : port_range) { + addr.sin6_port = htons(port); + + if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true; + + if (errno == EADDRINUSE) { + dbgDebug(D_API) << "Port " << port << " is already in use"; + } else { + dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + } } return false; @@ -119,21 +146,55 @@ RestServer::Impl::init() mainloop = Singleton::Consume::by(); auto init_connection = [this] () { - fd = socket(AF_INET, SOCK_STREAM, 0); + auto allow_external_conn = "Nano service API Allow Get From External IP"; + auto conf_value = getConfiguration("connection", allow_external_conn); + bool accept_get_from_external_ip = false; + if (conf_value.ok()) { + accept_get_from_external_ip = *conf_value; + } else { + auto env_value = Singleton::Consume::by()->get(allow_external_conn); + if (env_value.ok()) { + accept_get_from_external_ip = *env_value; + } + } + + if (accept_get_from_external_ip) { + fd = socket(AF_INET6, SOCK_STREAM, 0); + } else { + fd = socket(AF_INET, SOCK_STREAM, 0); + } dbgAssert(fd >= 0) << alert << "Failed to open a socket"; + int socket_enable = 1; if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) { dbgWarning(D_API) << "Could not set the socket options"; } - struct sockaddr_in addr; - bzero(&addr, sizeof(addr)); + if (accept_get_from_external_ip) { + int option = 0; + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) { + dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option"; + } - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + struct sockaddr_in6 addr6; + bzero(&addr6, sizeof(addr6)); + addr6.sin6_family = AF_INET6; + addr6.sin6_addr = in6addr_any; - while (!bindRestServerSocket(addr, port_range)) { - mainloop->yield(bind_retry_interval_msec); + while (!bindRestServerSocket(addr6, port_range)) { + mainloop->yield(bind_retry_interval_msec); + } + listening_port = ntohs(addr6.sin6_port); + } else { + struct sockaddr_in addr; + bzero(&addr, sizeof(addr)); + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + + while (!bindRestServerSocket(addr, port_range)) { + mainloop->yield(bind_retry_interval_msec); + } + listening_port = ntohs(addr.sin_port); } listen(fd, listen_limit); @@ -146,9 +207,12 @@ RestServer::Impl::init() "REST server listener", is_primary.ok() && *is_primary ); - - listening_port = ntohs(addr.sin_port); - dbgInfo(D_API) << "REST server started: " << listening_port; + dbgInfo(D_API) + << "REST server started: " + << listening_port + << ". Accepting: " + << (accept_get_from_external_ip ? "external" : "loopback") + << " connections"; Singleton::Consume::by()->registerValue("Listening Port", listening_port); }; @@ -172,14 +236,30 @@ void RestServer::Impl::startNewConnection() const { dbgFlow(D_API) << "Starting a new connection"; - int new_socket = accept(fd, nullptr, nullptr); + struct sockaddr_storage addr; + socklen_t addr_len = sizeof(addr); + int new_socket = accept(fd, (struct sockaddr *)&addr, &addr_len); if (new_socket < 0) { - dbgWarning(D_API) << "Failed to accept a new socket"; + dbgWarning(D_API) << "Failed to accept a new socket: " << strerror(errno); return; } - dbgDebug(D_API) << "Starting a new socket: " << new_socket; - RestConn conn(new_socket, mainloop, this); + dbgDebug(D_API) << "Starting a new socket: " << new_socket; + bool is_external = false; + if (addr.ss_family == AF_INET6) { + struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *) &addr; + if (IN6_IS_ADDR_V4MAPPED(&addr_in6->sin6_addr)) { + struct in_addr ipv4_addr; + memcpy(&ipv4_addr, &addr_in6->sin6_addr.s6_addr[12], sizeof(ipv4_addr)); + is_external = ipv4_addr.s_addr != htonl(INADDR_LOOPBACK); + } else { + is_external = memcmp(&addr_in6->sin6_addr, &in6addr_loopback, sizeof(in6addr_loopback)) != 0; + } + } else { + struct sockaddr_in *addr_in = (struct sockaddr_in *)&addr; + is_external = addr_in->sin_addr.s_addr != htonl(INADDR_LOOPBACK); + } + RestConn conn(new_socket, mainloop, this, is_external); mainloop->addFileRoutine( I_MainLoop::RoutineType::Offline, new_socket, @@ -283,4 +363,5 @@ RestServer::preload() registerExpectedConfiguration("connection", "Nano service API Port Alternative"); registerExpectedConfiguration("connection", "Nano service API Port Range start"); registerExpectedConfiguration("connection", "Nano service API Port Range end"); + registerExpectedConfiguration("connection", "Nano service API Allow Get From External IP"); } diff --git a/core/rest/rest_ut/rest_config_ut.cc b/core/rest/rest_ut/rest_config_ut.cc index e699f10..4490696 100644 --- a/core/rest/rest_ut/rest_config_ut.cc +++ b/core/rest/rest_ut/rest_config_ut.cc @@ -14,10 +14,49 @@ #include "agent_details.h" #include "mock/mock_messaging.h" #include "tenant_manager.h" +#include +#include using namespace std; using namespace testing; +static const string config_json_allow_external = + "{\n" + " \"connection\": {\n" + " \"Nano service API Port Primary\": [\n" + " {\n" + " \"value\": 9777\n" + " }\n" + " ],\n" + " \"Nano service API Port Alternative\": [\n" + " {\n" + " \"value\": 9778\n" + " }\n" + " ],\n" + " \"Nano service API Allow Get From External IP\": [\n" + " {\n" + " \"value\": true\n" + " }\n" + " ]\n" + " }\n" + "}\n"; + +static const string config_json = + "{\n" + " \"connection\": {\n" + " \"Nano service API Port Primary\": [\n" + " {\n" + " \"value\": 9777\n" + " }\n" + " ],\n" + " \"Nano service API Port Alternative\": [\n" + " {\n" + " \"value\": 9778\n" + " }\n" + " ]\n" + " }\n" + "}\n"; + USE_DEBUG_FLAG(D_API); USE_DEBUG_FLAG(D_MAINLOOP); @@ -31,22 +70,6 @@ public: time_proxy.init(); mainloop_comp.init(); - string config_json = - "{\n" - " \"connection\": {\n" - " \"Nano service API Port Primary\": [\n" - " {\n" - " \"value\": 9777\n" - " }\n" - " ],\n" - " \"Nano service API Port Alternative\": [\n" - " {\n" - " \"value\": 9778\n" - " }\n" - " ]\n" - " }\n" - "}\n"; - istringstream ss(config_json); Singleton::Consume::from(config)->loadConfiguration(ss); @@ -58,6 +81,9 @@ public: ~RestConfigTest() { Debug::setNewDefaultStdout(&cout); + auto mainloop = Singleton::Consume::from(mainloop_comp); + mainloop->stopAll(); + rest_server.fini(); time_proxy.fini(); mainloop_comp.fini(); } @@ -133,7 +159,7 @@ int TestServer::g_num = 0; TEST_F(RestConfigTest, basic_flow) { env.preload(); - Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "tmp_test_file"); config.preload(); config.init(); @@ -165,11 +191,15 @@ TEST_F(RestConfigTest, basic_flow) auto mainloop = Singleton::Consume::from(mainloop_comp); I_MainLoop::Routine stop_routine = [&] () { - EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); + EXPECT_EQ(connect(file_descriptor1, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0) + << "file_descriptor1 Error: " + << strerror(errno); string msg1 = "GET /stuff HTTP/1.1\r\n\r\n"; EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast(msg1.size())); - EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0); + EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0) + << "file_descriptor2 Error: " + << strerror(errno); string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast(msg2.size())); @@ -204,3 +234,159 @@ TEST_F(RestConfigTest, basic_flow) "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla" ); } + +string +getLocalIPAddress() { + char hostname[1024]; + hostname[1024 - 1] = '\0'; + + // Get the hostname + if (gethostname(hostname, sizeof(hostname)) == -1) { + return ""; + } + + struct addrinfo hints, *info, *p; + int gai_result; + + memset(&hints, 0, sizeof(hints)); + hints.ai_family = AF_INET; // Use AF_INET for IPv4 + hints.ai_socktype = SOCK_STREAM; + + // Get the address info + if ((gai_result = getaddrinfo(hostname, nullptr, &hints, &info)) != 0) { + return ""; + } + + std::string ip_address; + for (p = info; p != nullptr; p = p->ai_next) { + void *addr; + char ipstr[INET_ADDRSTRLEN]; + + struct sockaddr_in *ipv4 = (struct sockaddr_in *)p->ai_addr; + addr = &(ipv4->sin_addr); + + // Convert the IP to a string and print it + inet_ntop(p->ai_family, addr, ipstr, sizeof(ipstr)); + if (std::string(ipstr) != "127.0.0.1") { + ip_address = ipstr; + break; + } + } + + freeaddrinfo(info); // Free the linked list + + return ip_address; +} + + +TEST_F(RestConfigTest, not_loopback_flow) +{ + env.preload(); + Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + + + istringstream ss(config_json_allow_external); + Singleton::Consume::from(config)->loadConfiguration(ss); + + config.preload(); + config.init(); + + rest_server.init(); + time_proxy.init(); + mainloop_comp.init(); + + auto i_rest = Singleton::Consume::from(rest_server); + ASSERT_TRUE(i_rest->addRestCall(RestAction::ADD, "test")); + ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); })); + + int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor1, -1); + int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0); + EXPECT_NE(file_descriptor2, -1); + + auto primary_port = getConfiguration("connection", "Nano service API Port Primary"); + auto second_port = getConfiguration("connection", "Nano service API Port Alternative"); + auto local_ip = getLocalIPAddress(); + struct sockaddr_in sa_primary; + sa_primary.sin_family = AF_INET; + sa_primary.sin_port = htons(primary_port.unpack()); + sa_primary.sin_addr.s_addr = inet_addr(local_ip.c_str()); + struct sockaddr_in sa_second; + sa_second.sin_family = AF_INET; + sa_second.sin_port = htons(second_port.unpack()); + sa_second.sin_addr.s_addr = inet_addr(local_ip.c_str()); + + int socket_enable = 1; + EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0); + + EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _)) + .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + Debug::setNewDefaultStdout(&cout); + auto mainloop = Singleton::Consume::from(mainloop_comp); + Debug::setNewDefaultStdout(&cout); + I_MainLoop::Routine stop_routine = [&] () { + int socket_client_2 = -1; + auto socket_client_1 = connect(file_descriptor1, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr)); + dbgDebug(D_API) << "socket_client_1: " << socket_client_1; + if (socket_client_1 == -1) { + dbgDebug(D_API) << "Error: " << strerror(errno); + socket_client_2 = connect(file_descriptor1, (struct sockaddr*)&sa_second, sizeof(struct sockaddr)); + dbgDebug(D_API) << "socket_client_2: " << socket_client_2; + if (socket_client_2 == -1) { + dbgDebug(D_API) << "Error: " << strerror(errno) << endl; + } else { + EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_second, sizeof(struct sockaddr)), 0); + string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; + EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast(msg2.size())); + } + } else { + EXPECT_EQ(connect(file_descriptor2, (struct sockaddr*)&sa_primary, sizeof(struct sockaddr)), 0); + string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}"; + EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast(msg2.size())); + } + EXPECT_TRUE(socket_client_1 != -1 || socket_client_2 != -1); + string msg1 = "GET /stuff HTTP/1.1\r\n\r\n"; + EXPECT_EQ(write(file_descriptor1, msg1.data(), msg1.size()), static_cast(msg1.size())); + + mainloop->yield(true); + + struct pollfd s_poll; + s_poll.fd = file_descriptor1; + s_poll.events = POLLIN; + s_poll.revents = 0; + while(poll(&s_poll, 1, 0) <= 0) { + mainloop->yield(true); + } + + struct pollfd s_poll2; + s_poll2.fd = file_descriptor2; + s_poll2.events = POLLIN; + s_poll2.revents = 0; + while(poll(&s_poll2, 1, 0) <= 0) { + mainloop->yield(true); + } + + mainloop->stopAll(); + }; + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::RealTime, + stop_routine, + "RestConfigTest-alternative_port_used stop routine", + true + ); + mainloop->run(); + + char respose[1000]; + EXPECT_EQ(read(file_descriptor1, respose, 1000), 76); + EXPECT_EQ( + string(respose, 76), + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla" + ); + + EXPECT_EQ(read(file_descriptor2, respose, 1000), 89); + EXPECT_EQ( + string(respose, 89), + "HTTP/1.1 500 Internal Server Error\r\nContent-Type: application/json\r\nContent-Length: 0\r\n\r\n" + ); +} diff --git a/core/rest/rest_ut/rest_schema_ut.cc b/core/rest/rest_ut/rest_schema_ut.cc index 1a2ab71..1d63cc6 100644 --- a/core/rest/rest_ut/rest_schema_ut.cc +++ b/core/rest/rest_ut/rest_schema_ut.cc @@ -437,7 +437,7 @@ TEST(RestSchema, server_schema) env.preload(); - Singleton::Consume::from(env)->registerValue("Executable Name", "tmp_test_file"); + Singleton::Consume::from(env)->registerValue("Base Executable Name", "tmp_test_file"); config.preload(); config.init(); diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index f27cca3..ee811ea 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -1452,12 +1452,12 @@ upload_ai() # Initials - uai if [ "$AI_VERBOSE" = "true" ]; then echo "Uploading file $file" fi - if [ -z "${is_gaia}" -o "$is_smb_release" = "1" ]; then - uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) + if [ -n "${is_gaia}" -o "$is_smb_release" = "1" ]; then + uai_curl_output=$(${curl_cmd} -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1) elif [ "${remove_curl_ld_path}" = "true" ]; then - uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) + uai_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1) else - uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1) + uai_curl_output=$(${curl_cmd} --cacert ${FILESYSTEM_PATH}/certs/fog.pem "${uai_proxy_val}" -o /dev/null -s -w "%{http_code}\n" --progress-bar --request PUT -T "${file}" -H "user-agent: Infinity Next (a7030abf93a4c13)" -H "Content-Type: application/json" -H "Authorization: Bearer ${uai_token}" "$uai_fog_address"/agents-core/storage/"$uai_tenant_id"/"$uai_agent_id"/"$uai_current_time"/"$uai_file_dir" 2>&1 | tail -1) fi if [ "$AI_UPLOAD_TOO_LARGE_FLAG" = "false" ] && [ "$uai_curl_output" = "413" ]; then AI_UPLOAD_TOO_LARGE_FLAG=true