From c20fa9f9664d5f61fbe0deff877bd842248daeab Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Thu, 21 Mar 2024 15:31:38 +0000 Subject: [PATCH] Mar 21st 2024 update --- components/CMakeLists.txt | 1 - .../nginx_attachment/nginx_attachment.cc | 3 +- components/include/i_generic_rulebase.h | 4 + components/include/i_waap_telemetry.h | 2 + components/include/report_messaging.h | 3 + components/include/telemetry.h | 1 + .../report_messaging/report_messaging.cc | 10 +- .../report_messaging_ut.cc | 47 +++ .../access_control_practice.cc | 36 +- .../appsec_practice_section.cc | 21 +- .../include/access_control_practice.h | 4 +- .../include/appsec_practice_section.h | 7 +- .../include/local_policy_common.h | 3 +- .../include/new_practice.h | 29 +- .../include/policy_maker_utils.h | 10 +- .../local_policy_mgmt_gen/k8s_policy_utils.cc | 10 +- .../new_appsec_linux_policy.cc | 2 +- .../local_policy_mgmt_gen/new_exceptions.cc | 2 +- .../local_policy_mgmt_gen/new_practice.cc | 246 +++++++------ .../policy_maker_utils.cc | 53 ++- .../checkpoint_product_handlers.h | 12 + .../details_resolver_impl.h | 44 ++- .../details_resolving_handler.cc | 18 + .../orchestration/orchestration_comp.cc | 3 +- .../orchestration_ut/orchestration_ut.cc | 1 + .../update_communication/fog_authenticator.cc | 5 +- .../security_apps/waap/include/i_serialize.h | 10 + .../waap/include/i_transaction.h | 3 + .../waap/waap_clib/CMakeLists.txt | 1 + .../waap/waap_clib/DeepParser.cc | 22 +- .../security_apps/waap/waap_clib/DeepParser.h | 1 + .../waap_clib/OASchemaUpdaterConfConstant.h | 13 + .../security_apps/waap/waap_clib/ParserPDF.cc | 111 ++++++ .../security_apps/waap/waap_clib/ParserPDF.h | 47 +++ .../waap/waap_clib/Serializator.cc | 1 + .../security_apps/waap/waap_clib/Telemetry.cc | 1 + .../waap/waap_clib/WaapOverrideFunctor.cc | 49 ++- .../waap_clib/WaapResponseInspectReasons.cc | 16 +- .../waap_clib/WaapResponseInspectReasons.h | 2 + .../waap/waap_clib/Waf2Engine.cc | 183 +++++++-- .../security_apps/waap/waap_clib/Waf2Engine.h | 17 +- .../waap/waap_clib/Waf2EngineGetters.cc | 23 ++ .../security_apps/waap/waap_clib/Waf2Util.h | 7 + .../security_apps/waap/waap_component_impl.cc | 15 +- components/utils/CMakeLists.txt | 1 + .../generic_rulebase/CMakeLists.txt | 0 .../utils/generic_rulebase/assets_config.cc | 137 +++++++ .../evaluators/CMakeLists.txt | 0 .../generic_rulebase/evaluators/asset_eval.cc | 52 +++ .../evaluators/connection_eval.cc | 299 +++++++++++++++ .../evaluators/http_transaction_data_eval.cc | 168 +++++++++ .../evaluators/parameter_eval.cc | 38 ++ .../evaluators/practice_eval.cc | 50 +++ .../generic_rulebase/evaluators/query_eval.cc | 136 +++++++ .../evaluators/trigger_eval.cc | 57 +++ .../generic_rulebase/evaluators/zone_eval.cc | 44 +++ .../generic_rulebase/generic_rulebase.cc | 149 ++++++++ .../generic_rulebase_context.cc | 109 ++++++ .../utils/generic_rulebase/match_query.cc | 347 ++++++++++++++++++ .../generic_rulebase/parameters_config.cc | 157 ++++++++ .../utils/generic_rulebase/rulebase_config.cc | 79 ++++ .../utils/generic_rulebase/triggers_config.cc | 243 ++++++++++++ components/utils/generic_rulebase/zone.cc | 179 +++++++++ .../utils/generic_rulebase/zones_config.cc | 114 ++++++ .../agent_details_reporter_ut.cc | 11 + core/debug_is/debug_is_ut/debug_ut.cc | 3 +- core/include/general/common.h | 13 + .../services_sdk/interfaces/i_messaging.h | 6 +- .../intelligence_is_v2/asset_replay_impl.h | 2 +- .../interfaces/messaging/interface_impl.h | 6 +- .../interfaces/mock/mock_messaging.h | 5 +- .../services_sdk/resources/debug_flags.h | 2 + .../services_sdk/resources/generic_metric.h | 4 +- .../resources/intelligence_invalidation.h | 13 +- .../include/intelligence_server.h | 2 +- .../intelligence_comp_v2.cc | 20 +- .../intelligence_comp_v2_ut.cc | 5 +- .../intelligence_is_v2_ut/invalidation_ut.cc | 110 ++++-- .../intelligence_is_v2/intelligence_server.cc | 5 +- core/intelligence_is_v2/invalidation.cc | 103 +++++- core/logging/k8s_svc_stream.cc | 3 + core/logging/logging_ut/logging_ut.cc | 6 + core/mainloop/mainloop_ut/mainloop_ut.cc | 2 +- core/messaging/connection/connection.cc | 79 +++- core/messaging/connection/connection_comp.cc | 6 +- .../connection_ut/connection_comp_ut.cc | 110 ++---- core/messaging/include/connection.h | 1 + core/messaging/include/dummy_socket.h | 100 +++++ core/messaging/include/messaging_comp.h | 3 +- core/messaging/messaging.cc | 5 +- .../messaging_buffer_comp.cc | 10 +- .../messaging_buffer_comp_ut.cc | 22 +- .../messaging_comp/messaging_comp.cc | 7 +- .../messaging_comp_ut/messaging_comp_ut.cc | 35 +- core/metric/generic_metric.cc | 8 +- core/metric/metric_ut/metric_ut.cc | 11 + core/rest/rest_server.cc | 4 +- .../package/local-default-policy.yaml | 2 +- .../orchestration/package/open-appsec-ctl.sh | 82 +++-- .../package/orchestration_package.sh | 40 +- 100 files changed, 3851 insertions(+), 453 deletions(-) create mode 100644 components/security_apps/waap/waap_clib/OASchemaUpdaterConfConstant.h create mode 100644 components/security_apps/waap/waap_clib/ParserPDF.cc create mode 100644 components/security_apps/waap/waap_clib/ParserPDF.h rename components/{ => utils}/generic_rulebase/CMakeLists.txt (100%) create mode 100644 components/utils/generic_rulebase/assets_config.cc rename components/{ => utils}/generic_rulebase/evaluators/CMakeLists.txt (100%) create mode 100644 components/utils/generic_rulebase/evaluators/asset_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/connection_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/http_transaction_data_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/parameter_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/practice_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/query_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/trigger_eval.cc create mode 100644 components/utils/generic_rulebase/evaluators/zone_eval.cc create mode 100644 components/utils/generic_rulebase/generic_rulebase.cc create mode 100644 components/utils/generic_rulebase/generic_rulebase_context.cc create mode 100644 components/utils/generic_rulebase/match_query.cc create mode 100644 components/utils/generic_rulebase/parameters_config.cc create mode 100644 components/utils/generic_rulebase/rulebase_config.cc create mode 100644 components/utils/generic_rulebase/triggers_config.cc create mode 100644 components/utils/generic_rulebase/zone.cc create mode 100644 components/utils/generic_rulebase/zones_config.cc create mode 100644 core/messaging/include/dummy_socket.h diff --git a/components/CMakeLists.txt b/components/CMakeLists.txt index 99be4a8..c48e065 100644 --- a/components/CMakeLists.txt +++ b/components/CMakeLists.txt @@ -1,6 +1,5 @@ add_subdirectory(report_messaging) add_subdirectory(http_manager) -add_subdirectory(generic_rulebase) add_subdirectory(signal_handler) add_subdirectory(gradual_deployment) add_subdirectory(packet) diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 6174c65..9f52aa5 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -1136,8 +1136,7 @@ private: NginxAttachmentOpaque &opaque = i_transaction_table->getState(); uuid = opaque.getSessionUUID(); } - web_response_data.uuid_size = - string("Incident Id: ").length() + uuid.size(); + web_response_data.uuid_size = uuid.size(); if (web_trigger_conf.getDetailsLevel() == "Redirect") { web_response_data.response_data.redirect_data.redirect_location_size = diff --git a/components/include/i_generic_rulebase.h b/components/include/i_generic_rulebase.h index b99ad02..a49a574 100755 --- a/components/include/i_generic_rulebase.h +++ b/components/include/i_generic_rulebase.h @@ -17,6 +17,7 @@ #include #include "generic_rulebase/parameters_config.h" +#include "generic_rulebase/triggers_config.h" #include "generic_rulebase/zone.h" #include "config.h" @@ -26,6 +27,9 @@ public: virtual Maybe getLocalZone() const = 0; virtual Maybe getOtherZone() const = 0; + virtual LogTriggerConf getLogTriggerConf(const std::string &trigger_Id) const = 0; + virtual ParameterException getParameterException(const std::string ¶meter_Id) const = 0; + using ParameterKeyValues = std::unordered_map>; virtual std::set getBehavior(const ParameterKeyValues &key_value_pairs) const = 0; diff --git a/components/include/i_waap_telemetry.h b/components/include/i_waap_telemetry.h index c47c229..6b3e7d3 100755 --- a/components/include/i_waap_telemetry.h +++ b/components/include/i_waap_telemetry.h @@ -25,6 +25,7 @@ struct DecisionTelemetryData std::string source; TrafficMethod method; int responseCode; + uint64_t elapsedTime; std::set attackTypes; DecisionTelemetryData() : @@ -36,6 +37,7 @@ struct DecisionTelemetryData source(), method(POST), responseCode(0), + elapsedTime(0), attackTypes() { } diff --git a/components/include/report_messaging.h b/components/include/report_messaging.h index 3d6d479..43f2f09 100755 --- a/components/include/report_messaging.h +++ b/components/include/report_messaging.h @@ -141,9 +141,12 @@ public: ReportMessaging & operator<<(const LogField &field); + void setForceBuffering(bool _force_buffering); + private: Report report; bool is_async_message; + bool force_buffering = false; MessageCategory message_type_tag; }; diff --git a/components/include/telemetry.h b/components/include/telemetry.h index ddbeebc..0461688 100755 --- a/components/include/telemetry.h +++ b/components/include/telemetry.h @@ -91,6 +91,7 @@ private: MetricCalculations::Counter response_2xx{this, "reservedNgenG"}; MetricCalculations::Counter response_4xx{this, "reservedNgenH"}; MetricCalculations::Counter response_5xx{this, "reservedNgenI"}; + MetricCalculations::Average average_latency{this, "reservedNgenJ"}; }; class WaapAttackTypesMetrics : public WaapTelemetryBase diff --git a/components/report_messaging/report_messaging.cc b/components/report_messaging/report_messaging.cc index b72cecf..290a1c2 100755 --- a/components/report_messaging/report_messaging.cc +++ b/components/report_messaging/report_messaging.cc @@ -33,7 +33,9 @@ ReportMessaging::~ReportMessaging() HTTPMethod::POST, url, log_rest, - message_type_tag + message_type_tag, + MessageMetadata(), + force_buffering ); } catch (...) {} } @@ -44,3 +46,9 @@ ReportMessaging::operator<<(const LogField &field) report << field; return *this; } + +void +ReportMessaging::setForceBuffering(bool _force_buffering) +{ + force_buffering = _force_buffering; +} diff --git a/components/report_messaging/report_messaging_ut/report_messaging_ut.cc b/components/report_messaging/report_messaging_ut/report_messaging_ut.cc index b3c594f..85a2743 100644 --- a/components/report_messaging/report_messaging_ut/report_messaging_ut.cc +++ b/components/report_messaging/report_messaging_ut/report_messaging_ut.cc @@ -99,12 +99,55 @@ TEST_F(ReportMessagingTest, title_only) " }\n" "}", _, + _, _ ) ).Times(1); ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL); } +TEST_F(ReportMessagingTest, with_buffering) +{ + EXPECT_CALL( + mock_messaging, + sendAsyncMessage( + _, + _, + "{\n" + " \"log\": {\n" + " \"eventTime\": \"Best Time ever\",\n" + " \"eventName\": \"test\",\n" + " \"eventSeverity\": \"Info\",\n" + " \"eventPriority\": \"Low\",\n" + " \"eventType\": \"Event Driven\",\n" + " \"eventLevel\": \"Log\",\n" + " \"eventLogLevel\": \"info\",\n" + " \"eventAudience\": \"Internal\",\n" + " \"eventAudienceTeam\": \"Agent Core\",\n" + " \"eventFrequency\": 0,\n" + " \"eventTags\": [\n" + " \"Access Control\"\n" + " ],\n" + " \"eventSource\": {\n" + " \"eventTraceId\": \"\",\n" + " \"eventSpanId\": \"\",\n" + " \"issuingEngineVersion\": \"\",\n" + " \"serviceName\": \"Unnamed Nano Service\"\n" + " },\n" + " \"eventData\": {\n" + " \"eventObject\": 1\n" + " }\n" + " }\n" + "}", + _, + _, + true + ) + ).Times(1); + ReportMessaging report("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL); + report.setForceBuffering(true); +} + TEST_F(ReportMessagingTest, with_dynamic_fields) { EXPECT_CALL( @@ -140,6 +183,7 @@ TEST_F(ReportMessagingTest, with_dynamic_fields) " }\n" "}", _, + _, _ ) ).Times(1); @@ -189,6 +233,7 @@ TEST_F(ReportMessagingTest, custom_event_object) " }\n" "}", _, + _, _ ) ).Times(1); @@ -243,6 +288,7 @@ TEST_F(ReportMessagingTest, custom_priority) " }\n" "}", _, + _, _ ) ).Times(1); @@ -309,6 +355,7 @@ TEST_F(ReportMessagingTest, with_env_details) " }\n" "}", _, + _, _ ) ).Times(1); diff --git a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc index 18b447c..08d46d2 100755 --- a/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/access_control_practice.cc @@ -12,20 +12,34 @@ // limitations under the License. #include "access_control_practice.h" +#include "new_practice.h" using namespace std; USE_DEBUG_FLAG(D_LOCAL_POLICY); // LCOV_EXCL_START Reason: no test exist -static const map valid_modes_to_key = { +static const set valid_modes = { + "prevent", + "detect", + "inactive", + "prevent-learn", + "detect-learn", + "as-top-level", + "inherited" +}; + +static const unordered_map valid_modes_to_key = { {"prevent", "Active"}, + {"prevent-learn", "Active"}, {"detect", "Detect"}, + {"detect-learn", "Detect"}, {"inactive", "Inactive"} }; + static const set valid_units = {"minute", "second"}; -static const std::unordered_map key_to_units_val = { +static const unordered_map key_to_units_val = { { "second", "Second"}, { "minute", "Minute"} }; @@ -177,13 +191,10 @@ void AccessControlRateLimit::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading Access control rate limit"; - string in_mode; - parseAppsecJSONKey("overrideMode", in_mode, archive_in, "detect"); - if (valid_modes_to_key.find(in_mode) == valid_modes_to_key.end()) { - dbgWarning(D_LOCAL_POLICY) << "AppSec access control rate limit override mode invalid: " << in_mode; - throw PolicyGenException("AppSec access control rate limit override mode invalid: " + in_mode); - } else { - mode = valid_modes_to_key.at(in_mode); + parseMandatoryAppsecJSONKey("overrideMode", mode, archive_in, "inactive"); + if (valid_modes.find(mode) == valid_modes.end()) { + dbgWarning(D_LOCAL_POLICY) << "AppSec access control rate limit override mode invalid: " << mode; + throw PolicyGenException("AppSec access control rate limit override mode invalid: " + mode); } parseAppsecJSONKey>("rules", rules, archive_in); } @@ -205,9 +216,10 @@ AccessControlRateLimit::getRules() const } const string & -AccessControlRateLimit::getMode() const +AccessControlRateLimit::getMode(const std::string &default_mode) const { - return mode; + const string &res = getModeWithDefault(mode, default_mode, valid_modes_to_key); + return res; } void @@ -227,7 +239,7 @@ AccessControlPracticeSpec::setName(const string &_name) } const AccessControlRateLimit & -AccessControlPracticeSpec::geRateLimit() const +AccessControlPracticeSpec::getRateLimit() const { return rate_limit; } diff --git a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc index bc1007c..2a24378 100755 --- a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc @@ -133,7 +133,7 @@ AppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec"; parseAppsecJSONKey("protections", protections, archive_in); - parseAppsecJSONKey("override-mode", mode, archive_in, "Unset"); + parseAppsecJSONKey("override-mode", mode, archive_in, "as-top-level"); if (valid_modes.count(mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode; } @@ -187,7 +187,7 @@ AppSecPracticeWebAttacks::getMinimumConfidence() const const string & AppSecPracticeWebAttacks::getMode(const string &default_mode) const { - if (mode == "Unset" || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) { + if (isModeInherited(mode) || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) { dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode; return default_mode; } @@ -429,6 +429,9 @@ WebAppSection::WebAppSection( context(_context), web_attack_mitigation_severity(parsed_appsec_spec.getWebAttacks().getMinimumConfidence()), web_attack_mitigation_mode(parsed_appsec_spec.getWebAttacks().getMode(default_mode)), + csrf_protection_mode("Disabled"), + open_redirect_mode("Disabled"), + error_disclosure_mode("Disabled"), practice_advanced_config(parsed_appsec_spec), anti_bots(parsed_appsec_spec.getAntiBot()), trusted_sources({ parsed_trusted_sources }) @@ -451,6 +454,7 @@ WebAppSection::WebAppSection( } } +// Used for V1Beta2 WebAppSection::WebAppSection( const string &_application_urls, const string &_asset_id, @@ -465,7 +469,8 @@ WebAppSection::WebAppSection( const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, - const AppSecTrustedSources &parsed_trusted_sources) + const AppSecTrustedSources &parsed_trusted_sources, + const NewAppSecWebAttackProtections &protections) : application_urls(_application_urls), asset_id(_asset_id), @@ -489,6 +494,10 @@ WebAppSection::WebAppSection( web_attack_mitigation_severity == "medium" ? "high" : "Error"; + csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode); + open_redirect_mode = protections.getOpenRedirectMode(_web_attack_mitigation_mode); + error_disclosure_mode = protections.getErrorDisclosureMode(_web_attack_mitigation_mode); + triggers.push_back(TriggersInWaapSection(parsed_log_trigger)); for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) { overrides.push_back(AppSecOverride(source_ident)); @@ -510,9 +519,9 @@ WebAppSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("webAttackMitigationAction", web_attack_mitigation_action), cereal::make_nvp("webAttackMitigationMode", web_attack_mitigation_mode), cereal::make_nvp("practiceAdvancedConfig", practice_advanced_config), - cereal::make_nvp("csrfProtection", disabled_str), - cereal::make_nvp("openRedirect", disabled_str), - cereal::make_nvp("errorDisclosure", disabled_str), + cereal::make_nvp("csrfProtection", csrf_protection_mode), + cereal::make_nvp("openRedirect", open_redirect_mode), + cereal::make_nvp("errorDisclosure", error_disclosure_mode), cereal::make_nvp("practiceId", practice_id), cereal::make_nvp("practiceName", practice_name), cereal::make_nvp("assetId", asset_id), diff --git a/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h b/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h index b8c263a..9050e6e 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/access_control_practice.h @@ -165,7 +165,7 @@ public: void load(cereal::JSONInputArchive &archive_in); const std::vector & getRules() const; - const std::string & getMode() const; + const std::string & getMode(const std::string &default_mode = "inactive") const; std::vector createRateLimitRulesSection(const RateLimitRulesTriggerSection &trigger) const; private: @@ -178,7 +178,7 @@ class AccessControlPracticeSpec public: void load(cereal::JSONInputArchive &archive_in); - const AccessControlRateLimit & geRateLimit() const; + const AccessControlRateLimit &getRateLimit() const; const std::string & getAppSecClassName() const; const std::string & getName() const; void setName(const std::string &_name); diff --git a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h index be30a59..50e939f 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h @@ -278,6 +278,7 @@ public: const std::vector &parsed_exceptions ); + // used for V1beta2 WebAppSection( const std::string &_application_urls, const std::string &_asset_id, @@ -292,7 +293,8 @@ public: const PracticeAdvancedConfig &_practice_advanced_config, const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, - const AppSecTrustedSources &parsed_trusted_sources); + const AppSecTrustedSources &parsed_trusted_sources, + const NewAppSecWebAttackProtections &protections); void save(cereal::JSONOutputArchive &out_ar) const; @@ -310,6 +312,9 @@ private: std::string web_attack_mitigation_action; std::string web_attack_mitigation_severity; std::string web_attack_mitigation_mode; + std::string csrf_protection_mode; + std::string open_redirect_mode; + std::string error_disclosure_mode; bool web_attack_mitigation; std::vector triggers; PracticeAdvancedConfig practice_advanced_config; diff --git a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h index 6da97b0..612687e 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h +++ b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h @@ -97,8 +97,7 @@ parseAppsecJSONKey( value = default_value; if (!mandatory) { dbgDebug(D_LOCAL_POLICY) - << "Could not parse the required key. Key: \""<< key_name - << "\", Error: " << e.what(); + << "Could not parse a non-mandatory key: \""<< key_name << "\", Error: " << e.what(); } else { throw PolicyGenException( "Could not parse a mandatory key: \"" + key_name + "\", Error: " + std::string(e.what()) diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h index 63658f7..9b098b1 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h @@ -24,6 +24,14 @@ #include "debug.h" #include "local_policy_common.h" +bool isModeInherited(const std::string &mode); + +const std::string &getModeWithDefault( + const std::string &mode, + const std::string &default_mode, + const std::unordered_map &key_to_val +); + class IpsProtectionsRulesSection { public: @@ -126,8 +134,8 @@ class NewIntrusionPrevention public: void load(cereal::JSONInputArchive &archive_in); - std::vector createIpsRules() const; - const std::string & getMode() const; + std::vector createIpsRules(const std::string &default_mode) const; + const std::string & getMode(const std::string &default_mode = "inactive") const; private: std::string override_mode; @@ -273,7 +281,8 @@ public: const std::string &asset_name, const std::string &asset_id, const std::string &practice_name, - const std::string &practice_id + const std::string &practice_id, + const std::string &default_mode ) const; private: @@ -486,7 +495,7 @@ public: void load(cereal::JSONInputArchive &archive_in); void addFile(const std::string &file_name); - const std::string & getOverrideMode() const; + const std::string & getOverrideMode(const std::string &default_mode = "inactive") const; const std::vector & getConfigMap() const; const std::vector & getFiles() const; bool isTemporary() const; @@ -530,10 +539,10 @@ class NewAppSecWebAttackProtections public: void load(cereal::JSONInputArchive &archive_in); - const std::string getCsrfProtectionMode() const; - const std::string & getErrorDisclosureMode() const; + const std::string & getCsrfProtectionMode(const std::string &default_mode = "inactive") const; + const std::string & getErrorDisclosureMode(const std::string &default_mode = "inactive") const; + const std::string & getOpenRedirectMode(const std::string &default_mode = "inactive") const; bool getNonValidHttpMethods() const; - const std::string getOpenRedirectMode() const; private: std::string csrf_protection; @@ -551,9 +560,9 @@ public: int getMaxHeaderSizeBytes() const; int getMaxObjectDepth() const; int getMaxUrlSizeBytes() const; - const std::string & getMinimumConfidence() const; - const NewAppSecWebAttackProtections & getprotections() const; - const std::string & getMode(const std::string &default_mode = "Inactive") const; + const std::string & getMinimumConfidence(const std::string &default_mode = "inactive") const; + const NewAppSecWebAttackProtections & getProtections() const; + const std::string & getMode(const std::string &default_mode = "inactive") const; private: int max_body_size_kb; diff --git a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h index 7a65e9d..7575720 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h +++ b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h @@ -158,7 +158,8 @@ private: const std::string &source_identifier, const std::string & context, const V1beta2AppsecLinuxPolicy &policy, - std::map &rule_annotations + std::map &rule_annotations, + const std::string &default_mode ); void createSnortProtecionsSection(const std::string &file_name, bool is_temporary); @@ -172,7 +173,8 @@ private: const std::string &practice_id, const std::string &source_identifier, const V1beta2AppsecLinuxPolicy &policy, - std::map &rule_annotations + std::map &rule_annotations, + const std::string &default_mode ); void @@ -183,7 +185,8 @@ private: const std::string &practice_name, const std::string & context, const V1beta2AppsecLinuxPolicy &policy, - std::map &rule_annotations + std::map &rule_annotations, + const std::string &default_mode ); void @@ -192,6 +195,7 @@ private: const std::string &url, const std::string &uri, const std::string &trigger_id, + const std::string &default_mode, const V1beta2AppsecLinuxPolicy &policy, std::map &rule_annotations ); diff --git a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc index 450f6b7..33f4522 100644 --- a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc @@ -414,7 +414,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds( vector access_control_practices = extractV1Beta2ElementsFromCluster( - "accesscontrolpractice", + "accesscontrolpractices", policy_elements_names[AnnotationTypes::ACCESS_CONTROL_PRACTICE] ); @@ -489,6 +489,8 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i !doesVersionExist(maybe_appsec_policy_spec.unpack().getMetaData().getAnnotations(), "v1beta1") ) { try { + std::string v1beta1_error = + maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr(); dbgWarning(D_LOCAL_POLICY ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2"; auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster>( @@ -498,7 +500,7 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i dbgWarning(D_LOCAL_POLICY) << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr(); return std::make_tuple( - genError("Failed to retrieve AppSec v1beta1 policy. Error: " + maybe_appsec_policy_spec.getErr()), + genError("Failed to retrieve AppSec v1beta1 policy. Error: " + v1beta1_error), genError( "Failed to retrieve AppSec v1beta2 policy. Error: " + maybe_v1beta2_appsec_policy_spec.getErr() ) @@ -584,7 +586,9 @@ K8sPolicyUtils::createAppsecPoliciesFromIngresses() ); if (!std::get<0>(maybe_appsec_policy).ok() && !std::get<1>(maybe_appsec_policy).ok()) { dbgWarning(D_LOCAL_POLICY) - << "Failed to create appsec policy. Error: " + << "Failed to create appsec policy. v1beta1 Error: " + << std::get<0>(maybe_appsec_policy).getErr() + << ". v1beta2 Error: " << std::get<1>(maybe_appsec_policy).getErr(); continue; } diff --git a/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc b/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc index 25fa7e5..c202016 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc @@ -99,7 +99,7 @@ V1beta2AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in) archive_in ); parseAppsecJSONKey>("logTriggers", log_triggers, archive_in); - parseAppsecJSONKey>("customResponse", custom_responses, archive_in); + parseAppsecJSONKey>("customResponses", custom_responses, archive_in); parseAppsecJSONKey>("exceptions", exceptions, archive_in); parseAppsecJSONKey>("trustedSources", trusted_sources, archive_in); parseAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc index 8a2ae01..d7dfc35 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc @@ -44,7 +44,7 @@ void NewAppsecException::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading New AppSec exception"; - parseAppsecJSONKey("name", name, archive_in, "exception"); + parseAppsecJSONKey("name", name, archive_in); parseMandatoryAppsecJSONKey("action", action, archive_in, "accept"); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); if (valid_actions.count(action) == 0) { diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index 36e74d1..572cec6 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -21,8 +21,16 @@ USE_DEBUG_FLAG(D_LOCAL_POLICY); static const set performance_impacts = {"low", "medium", "high"}; static const set severity_levels = {"low", "medium", "high", "critical"}; static const set size_unit = {"bytes", "KB", "MB", "GB"}; -static const set confidences_actions = {"prevent", "detect", "inactive"}; -static const set valid_modes = {"prevent", "detect", "inactive", "prevent-learn", "detect-learn"}; +static const set confidences_actions = {"prevent", "detect", "inactive", "as-top-level", "inherited"}; +static const set valid_modes = { + "prevent", + "detect", + "inactive", + "prevent-learn", + "detect-learn", + "as-top-level", + "inherited" +}; static const set valid_confidences = {"medium", "high", "critical"}; static const std::unordered_map key_to_performance_impact_val = { { "low", "Low or lower"}, @@ -48,6 +56,30 @@ static const std::unordered_map unit_to_int = { { "MB", 1048576}, { "GB", 1073741824} }; +static const std::string TRANSPARENT_MODE = "Transparent"; + +bool +isModeInherited(const string &mode) +{ + return mode == "as-top-level" || mode == "inherited"; +} + +const std::string & +getModeWithDefault( + const std::string &mode, + const std::string &default_mode, + const std::unordered_map &key_to_val) +{ + if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) { + dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode; + return key_to_val.at(default_mode); + } + else if (key_to_val.find(mode) == key_to_val.end()) { + dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid."; + return key_to_val.at("inactive"); + } + return key_to_val.at(mode); +} void NewAppSecWebBotsURI::load(cereal::JSONInputArchive &archive_in) @@ -84,7 +116,7 @@ NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in) dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots"; parseAppsecJSONKey>("injectedUris", injected_uris, archive_in); parseAppsecJSONKey>("validatedUris", validated_uris, archive_in); - parseAppsecJSONKey("overrideMode", override_mode, archive_in, "Inactive"); + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode; } @@ -110,26 +142,33 @@ NewAppSecWebAttackProtections::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("csrfProtection", csrf_protection, archive_in, "inactive"); parseAppsecJSONKey("errorDisclosure", error_disclosure, archive_in, "inactive"); parseAppsecJSONKey("openRedirect", open_redirect, archive_in, "inactive"); + if (valid_modes.count(csrf_protection) == 0 || + valid_modes.count(error_disclosure) == 0 || + valid_modes.count(open_redirect) == 0) { + string error_msg = "AppSec Attack Protections mode invalid. csrf_protection: " + csrf_protection + + " error_disclosure: " + error_disclosure + " open_redirect: " + open_redirect; + dbgWarning(D_LOCAL_POLICY) << error_msg; + throw PolicyGenException(error_msg); + } parseAppsecJSONKey("nonValidHttpMethods", non_valid_http_methods, archive_in, false); } -const string -NewAppSecWebAttackProtections::getCsrfProtectionMode() const +const string & +NewAppSecWebAttackProtections::getCsrfProtectionMode(const string &default_mode) const { - if (key_to_practices_val.find(csrf_protection) == key_to_practices_val.end()) { - dbgError(D_LOCAL_POLICY) - << "Failed to find a value for " - << csrf_protection - << ". Setting CSRF protection to Inactive"; - return "Inactive"; - } - return key_to_practices_val.at(csrf_protection); + return getModeWithDefault(csrf_protection, default_mode, key_to_practices_val2); } const string & -NewAppSecWebAttackProtections::getErrorDisclosureMode() const +NewAppSecWebAttackProtections::getErrorDisclosureMode(const string &default_mode) const { - return error_disclosure; + return getModeWithDefault(error_disclosure, default_mode, key_to_practices_val2); +} + +const string & +NewAppSecWebAttackProtections::getOpenRedirectMode(const string &default_mode) const +{ + return getModeWithDefault(open_redirect, default_mode, key_to_practices_val2); } bool @@ -138,40 +177,24 @@ NewAppSecWebAttackProtections::getNonValidHttpMethods() const return non_valid_http_methods; } -const string -NewAppSecWebAttackProtections::getOpenRedirectMode() const -{ - if (key_to_practices_val.find(open_redirect) == key_to_practices_val.end()) { - dbgError(D_LOCAL_POLICY) - << "Failed to find a value for " - << open_redirect - << ". Setting Open Redirect mode to Inactive"; - return "Inactive"; - } - return key_to_practices_val.at(open_redirect); -} - void NewAppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice web attacks spec"; parseAppsecJSONKey("protections", protections, archive_in); - parseAppsecJSONKey("overrideMode", mode, archive_in, "Unset"); + parseMandatoryAppsecJSONKey("overrideMode", mode, archive_in, "inactive"); if (valid_modes.count(mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode; } - if (getMode() == "Prevent") { - parseMandatoryAppsecJSONKey("minimumConfidence", minimum_confidence, archive_in, "critical"); - if (valid_confidences.count(minimum_confidence) == 0) { - dbgWarning(D_LOCAL_POLICY) - << "AppSec practice override minimum confidence invalid: " - << minimum_confidence; - throw PolicyGenException("AppSec practice override minimum confidence invalid: " + minimum_confidence); - } - } else { - minimum_confidence = "Transparent"; + parseAppsecJSONKey("minimumConfidence", minimum_confidence, archive_in, "critical"); + if (valid_confidences.count(minimum_confidence) == 0) { + dbgWarning(D_LOCAL_POLICY) + << "AppSec practice override minimum confidence invalid: " + << minimum_confidence; + throw PolicyGenException("AppSec practice override minimum confidence invalid: " + minimum_confidence); } + parseAppsecJSONKey("maxBodySizeKb", max_body_size_kb, archive_in, 1000000); parseAppsecJSONKey("maxHeaderSizeBytes", max_header_size_bytes, archive_in, 102400); parseAppsecJSONKey("maxObjectDepth", max_object_depth, archive_in, 40); @@ -203,19 +226,25 @@ NewAppSecPracticeWebAttacks::getMaxUrlSizeBytes() const } const string & -NewAppSecPracticeWebAttacks::getMinimumConfidence() const +NewAppSecPracticeWebAttacks::getMinimumConfidence(const string &default_mode) const { + if (getMode(default_mode) != "Prevent") { + return TRANSPARENT_MODE; + } return minimum_confidence; } const string & NewAppSecPracticeWebAttacks::getMode(const string &default_mode) const { - if (mode == "Unset" || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) { - dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode; - return default_mode; - } - return key_to_practices_val2.at(mode); + const string &res = getModeWithDefault(mode, default_mode, key_to_practices_val); + return res; +} + +const NewAppSecWebAttackProtections & +NewAppSecPracticeWebAttacks::getProtections() const +{ + return protections; } SnortProtectionsSection::SnortProtectionsSection( @@ -244,7 +273,7 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const { out_ar( cereal::make_nvp("context", context), - cereal::make_nvp("mode", key_to_mode_val.at(mode)), + cereal::make_nvp("mode", mode), cereal::make_nvp("files", files), cereal::make_nvp("assetName", asset_name), cereal::make_nvp("assetId", asset_id), @@ -440,8 +469,8 @@ void NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice"; - parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); - parseMandatoryAppsecJSONKey>("configmap", config_map, archive_in); + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); + parseAppsecJSONKey>("configmap", config_map, archive_in); parseAppsecJSONKey>("files", files, archive_in); is_temporary = false; if (valid_modes.count(override_mode) == 0) { @@ -457,9 +486,10 @@ NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name) } const string & -NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode() const +NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const { - return override_mode; + const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val); + return res; } const vector & @@ -491,7 +521,7 @@ IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const { vector protections; out_ar( - cereal::make_nvp("action", key_to_mode_val.at(action)), + cereal::make_nvp("action", action), cereal::make_nvp("confidenceLevel", confidence_level), cereal::make_nvp("clientProtections", true), cereal::make_nvp("serverProtections", true), @@ -541,7 +571,7 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("practiceName", practice_name), cereal::make_nvp("practiceId", practice_id), cereal::make_nvp("sourceIdentifier", source_identifier), - cereal::make_nvp("defaultAction", key_to_mode_val.at(mode)), + cereal::make_nvp("defaultAction", mode), cereal::make_nvp("rules", rules) ); } @@ -566,7 +596,7 @@ void NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Intrusion Prevention practice"; - parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention override mode invalid: " << override_mode; throw PolicyGenException("AppSec Intrusion Prevention override mode invalid: " + override_mode); @@ -580,13 +610,13 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) "AppSec Intrusion Prevention max performance impact invalid: " + max_performance_impact ); } - parseAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "low"); + parseAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "medium"); if (severity_levels.count(min_severity_level) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention min severity level invalid: " << min_severity_level; } - parseAppsecJSONKey("highConfidenceEventAction", high_confidence_event_action, archive_in, "prevent"); + parseAppsecJSONKey("highConfidenceEventAction", high_confidence_event_action, archive_in, "inherited"); if (confidences_actions.count(high_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention high confidence event invalid: " @@ -595,7 +625,9 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) "AppSec Intrusion Prevention high confidence event invalid: " + high_confidence_event_action ); } - parseAppsecJSONKey("mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "prevent"); + parseAppsecJSONKey( + "mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inherited" + ); if (confidences_actions.count(medium_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec Intrusion Prevention medium confidence event invalid: " @@ -613,16 +645,16 @@ NewIntrusionPrevention::load(cereal::JSONInputArchive &archive_in) "AppSec Intrusion Prevention low confidence event action invalid: " + low_confidence_event_action ); } - parseAppsecJSONKey("minCveYear", min_cve_Year, archive_in); + parseAppsecJSONKey("minCveYear", min_cve_Year, archive_in, 2016); } vector -NewIntrusionPrevention::createIpsRules() const +NewIntrusionPrevention::createIpsRules(const string &default_mode) const { vector ips_rules; IpsProtectionsRulesSection high_rule( min_cve_Year, - high_confidence_event_action, + getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val), string("High"), max_performance_impact, string(""), @@ -632,7 +664,7 @@ NewIntrusionPrevention::createIpsRules() const IpsProtectionsRulesSection med_rule( min_cve_Year, - medium_confidence_event_action, + getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val), string("Medium"), max_performance_impact, string(""), @@ -642,7 +674,7 @@ NewIntrusionPrevention::createIpsRules() const IpsProtectionsRulesSection low_rule( min_cve_Year, - low_confidence_event_action, + getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val), string("Low"), max_performance_impact, string(""), @@ -654,9 +686,10 @@ NewIntrusionPrevention::createIpsRules() const } const std::string & -NewIntrusionPrevention::getMode() const +NewIntrusionPrevention::getMode(const std::string &default_mode) const { - return override_mode; + const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val); + return res; } FileSecurityProtectionsSection::FileSecurityProtectionsSection( @@ -711,20 +744,20 @@ FileSecurityProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const cereal::make_nvp("assetId", asset_id), cereal::make_nvp("practiceName", practice_name), cereal::make_nvp("practiceId", practice_id), - cereal::make_nvp("action", key_to_mode_val.at(action)), - cereal::make_nvp("filesWithoutNameAction", key_to_mode_val.at(files_without_name_action)), + cereal::make_nvp("action", action), + cereal::make_nvp("filesWithoutNameAction", files_without_name_action), cereal::make_nvp("allowFilesWithoutName", allow_files_without_name), - cereal::make_nvp("highConfidence", key_to_mode_val.at(high_confidence_action)), - cereal::make_nvp("mediumConfidence", key_to_mode_val.at(medium_confidence_action)), - cereal::make_nvp("lowConfidence", key_to_mode_val.at(low_confidence_action)), + cereal::make_nvp("highConfidence", high_confidence_action), + cereal::make_nvp("mediumConfidence", medium_confidence_action), + cereal::make_nvp("lowConfidence", low_confidence_action), cereal::make_nvp("severityLevel", key_to_severity_level_val.at(severity_level)), - cereal::make_nvp("fileSizeLimitAction", key_to_mode_val.at(file_size_limit_action)), + cereal::make_nvp("fileSizeLimitAction", file_size_limit_action), cereal::make_nvp("fileSizeLimit", file_size_limit), cereal::make_nvp("requiredFileSizeLimit", required_file_size_limit), cereal::make_nvp("requiredArchiveExtraction", required_archive_extraction), cereal::make_nvp("archiveFileSizeLimit", archive_file_size_limit), - cereal::make_nvp("MultiLevelArchiveAction", key_to_mode_val.at(multi_level_archive_action)), - cereal::make_nvp("UnopenedArchiveAction", key_to_mode_val.at(unopened_archive_action)) + cereal::make_nvp("MultiLevelArchiveAction", multi_level_archive_action), + cereal::make_nvp("UnopenedArchiveAction", unopened_archive_action) ); } @@ -748,7 +781,7 @@ void NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security Archive Inspection practice"; - parseAppsecJSONKey("extractArchiveFiles", extract_archive_files, archive_in, true); + parseAppsecJSONKey("extractArchiveFiles", extract_archive_files, archive_in, false); parseAppsecJSONKey("scanMaxFileSize", scan_max_file_size, archive_in, 10); parseAppsecJSONKey("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "MB"); if (size_unit.count(scan_max_file_size_unit) == 0) { @@ -763,7 +796,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) "archivedFilesWithinArchivedFiles", archived_files_within_archived_files, archive_in, - "prevent"); + "inherited"); if (confidences_actions.count(archived_files_within_archived_files) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived files invalid: " @@ -777,7 +810,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) "archivedFilesWhereContentExtractionFailed", archived_files_where_content_extraction_failed, archive_in, - "prevent"); + "inherited"); if (confidences_actions.count(archived_files_where_content_extraction_failed) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived file invalid: " @@ -834,7 +867,7 @@ NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in) "filesExceedingSizeLimitAction", files_exceeding_size_limit_action, archive_in, - "prevent"); + "inherited"); if (confidences_actions.count(files_exceeding_size_limit_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security Archive Inspection archived files within archived files invalid: " @@ -869,18 +902,18 @@ void NewFileSecurity::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security practice"; - parseAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); + parseMandatoryAppsecJSONKey("overrideMode", override_mode, archive_in, "inactive"); if (valid_modes.count(override_mode) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security override mode invalid: " << override_mode; throw PolicyGenException("AppSec File Security override mode invalid: " + override_mode); } - parseMandatoryAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "low"); + parseAppsecJSONKey("minSeverityLevel", min_severity_level, archive_in, "medium"); if (severity_levels.count(min_severity_level) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security min severity level invalid: " << min_severity_level; min_severity_level = "low"; } - parseMandatoryAppsecJSONKey( - "highConfidenceEventAction", high_confidence_event_action, archive_in, "inactive" + parseAppsecJSONKey( + "highConfidenceEventAction", high_confidence_event_action, archive_in, "inherited" ); if (confidences_actions.count(high_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -888,8 +921,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in) << high_confidence_event_action; high_confidence_event_action = "inactive"; } - parseMandatoryAppsecJSONKey( - "mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inactive" + parseAppsecJSONKey( + "mediumConfidenceEventAction", medium_confidence_event_action, archive_in, "inherited" ); if (confidences_actions.count(medium_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -897,8 +930,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in) << medium_confidence_event_action; medium_confidence_event_action = "inactive"; } - parseMandatoryAppsecJSONKey( - "lowConfidenceEventAction", low_confidence_event_action, archive_in, "inactive" + parseAppsecJSONKey( + "lowConfidenceEventAction", low_confidence_event_action, archive_in, "detect" ); if (confidences_actions.count(low_confidence_event_action) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -906,7 +939,7 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in) << low_confidence_event_action; low_confidence_event_action = "inactive"; } - parseMandatoryAppsecJSONKey("unnamedFilesAction", unnamed_files_action, archive_in, "inactive"); + parseAppsecJSONKey("unnamedFilesAction", unnamed_files_action, archive_in, "inherited"); if (confidences_actions.count(unnamed_files_action) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec File Security low unnamed files action invalid: " @@ -914,10 +947,8 @@ NewFileSecurity::load(cereal::JSONInputArchive &archive_in) unnamed_files_action = "inactive"; } parseAppsecJSONKey("threatEmulationEnabled", threat_emulation_enabled, archive_in); - parseMandatoryAppsecJSONKey("archiveInspection", archive_inspection, archive_in); - parseMandatoryAppsecJSONKey( - "largeFileInspection", large_file_inspection, archive_in - ); + parseAppsecJSONKey("archiveInspection", archive_inspection, archive_in); + parseAppsecJSONKey("largeFileInspection", large_file_inspection, archive_in); } const string & @@ -944,28 +975,37 @@ NewFileSecurity::createFileSecurityProtectionsSection( const string &asset_name, const string &asset_id, const string &practice_name, - const string &practice_id) const + const string &practice_id, + const string &default_mode) const { + string practice_action = (isModeInherited(override_mode) ? default_mode : override_mode); + const string &unnamed_files_action_val = + getModeWithDefault(unnamed_files_action, practice_action, key_to_mode_val); + const string &large_file_action_val = getModeWithDefault( + getLargeFileInspection().getFileSizeLimitAction(), + practice_action, + key_to_mode_val + ); return FileSecurityProtectionsSection( getLargeFileInspection().getFileSizeLimit(), getArchiveInspection().getArchiveFileSizeLimit(), - unnamed_files_action == "prevent" ? true : false, - getLargeFileInspection().getFileSizeLimitAction() == "prevent" ? true : false, + unnamed_files_action_val == "Prevent" ? true : false, + large_file_action_val == "Prevent" ? true : false, getArchiveInspection().getrequiredArchiveExtraction(), context, asset_name, asset_id, practice_name, practice_id, - override_mode, - unnamed_files_action, - high_confidence_event_action, - medium_confidence_event_action, - low_confidence_event_action, + getModeWithDefault(override_mode, practice_action, key_to_mode_val), + unnamed_files_action_val, + getModeWithDefault(high_confidence_event_action, practice_action, key_to_mode_val), + getModeWithDefault(medium_confidence_event_action, practice_action, key_to_mode_val), + getModeWithDefault(low_confidence_event_action, practice_action, key_to_mode_val), min_severity_level, - getLargeFileInspection().getFileSizeLimitAction(), - getArchiveInspection().getMultiLevelArchiveAction(), - getArchiveInspection().getUnopenedArchiveAction() + large_file_action_val, + getModeWithDefault(getArchiveInspection().getMultiLevelArchiveAction(), practice_action, key_to_mode_val), + getModeWithDefault(getArchiveInspection().getUnopenedArchiveAction(), practice_action, key_to_mode_val) ); } @@ -974,14 +1014,14 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec"; parseAppsecJSONKey( - "openapi-schema-validation", + "schemaValidation", openapi_schema_validation, archive_in ); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); - parseAppsecJSONKey("fileSecurity", file_security, archive_in); - parseAppsecJSONKey("intrusionPrevention", intrusion_prevention, archive_in); - parseAppsecJSONKey("snortSignatures", snort_signatures, archive_in); + parseMandatoryAppsecJSONKey("fileSecurity", file_security, archive_in); + parseMandatoryAppsecJSONKey("intrusionPrevention", intrusion_prevention, archive_in); + parseMandatoryAppsecJSONKey("snortSignatures", snort_signatures, archive_in); parseMandatoryAppsecJSONKey("webAttacks", web_attacks, archive_in); parseAppsecJSONKey("antiBot", anti_bot, archive_in); parseAppsecJSONKey("name", practice_name, archive_in); diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index a96f0e4..89927c7 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -996,13 +996,15 @@ PolicyMakerUtils::createIpsSections( const string &source_identifier, const string & context, const V1beta2AppsecLinuxPolicy &policy, - map &rule_annotations) + map &rule_annotations, + const string &default_mode) { auto apssec_practice = getAppsecPracticeSpec( rule_annotations[AnnotationTypes::PRACTICE], policy); - if (apssec_practice.getIntrusionPrevention().getMode().empty()) return; + const string &override_mode = apssec_practice.getIntrusionPrevention().getMode(default_mode); + if (override_mode == "Inactive" || override_mode == "Disabled") return; IpsProtectionsSection ips_section = IpsProtectionsSection( context, @@ -1011,8 +1013,8 @@ PolicyMakerUtils::createIpsSections( practice_name, practice_id, source_identifier, - apssec_practice.getIntrusionPrevention().getMode(), - apssec_practice.getIntrusionPrevention().createIpsRules() + override_mode, + apssec_practice.getIntrusionPrevention().createIpsRules(override_mode) ); ips[asset_name] = ips_section; @@ -1068,13 +1070,16 @@ PolicyMakerUtils::createSnortSections( const string &practice_id, const string &source_identifier, const V1beta2AppsecLinuxPolicy &policy, - map &rule_annotations) + map &rule_annotations, + const string &default_mode) { auto apssec_practice = getAppsecPracticeSpec( rule_annotations[AnnotationTypes::PRACTICE], policy); - if (apssec_practice.getSnortSignatures().getOverrideMode() == "inactive" || + const string &override_mode = apssec_practice.getSnortSignatures().getOverrideMode(default_mode); + if (override_mode == "Inactive" || + override_mode == "Disabled" || apssec_practice.getSnortSignatures().getFiles().size() == 0) { return; } @@ -1094,7 +1099,7 @@ PolicyMakerUtils::createSnortSections( practice_name, practice_id, source_identifier, - apssec_practice.getSnortSignatures().getOverrideMode(), + override_mode, apssec_practice.getSnortSignatures().getFiles() ); @@ -1109,7 +1114,8 @@ PolicyMakerUtils::createFileSecuritySections( const string &practice_name, const string &context, const V1beta2AppsecLinuxPolicy &policy, - map &rule_annotations) + map &rule_annotations, + const string &default_mode) { auto apssec_practice = getAppsecPracticeSpec( rule_annotations[AnnotationTypes::PRACTICE], @@ -1122,7 +1128,8 @@ PolicyMakerUtils::createFileSecuritySections( asset_name, asset_id, practice_name, - practice_id + practice_id, + default_mode ); file_security[asset_name] = file_security_section; @@ -1134,6 +1141,7 @@ PolicyMakerUtils::createRateLimitSection( const string &url, const string &uri, const string &trigger_id, + const std::string &default_mode, const V1beta2AppsecLinuxPolicy &policy, map &rule_annotations) { @@ -1157,13 +1165,13 @@ PolicyMakerUtils::createRateLimitSection( trigger = RateLimitRulesTriggerSection(trigger_id, trigger_name, "Trigger"); } - auto rules = access_control_practice.geRateLimit().createRateLimitRulesSection(trigger); + auto rules = access_control_practice.getRateLimit().createRateLimitRulesSection(trigger); rate_limit[rule_annotations[AnnotationTypes::ACCESS_CONTROL_PRACTICE]] = RateLimitSection( asset_name, url, uri, - access_control_practice.geRateLimit().getMode(), + access_control_practice.getRateLimit().getMode(default_mode), practice_id, rule_annotations[AnnotationTypes::ACCESS_CONTROL_PRACTICE], rules @@ -1198,12 +1206,13 @@ PolicyMakerUtils::createWebAppSection( practice_id, rule_annotations[AnnotationTypes::PRACTICE], rule_config.getContext(), - apssec_practice.getWebAttacks().getMinimumConfidence(), + apssec_practice.getWebAttacks().getMinimumConfidence(default_mode), apssec_practice.getWebAttacks().getMode(default_mode), practice_advance_config, apssec_practice.getAntiBot(), log_triggers[rule_annotations[AnnotationTypes::TRIGGER]], - trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]] + trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]], + apssec_practice.getWebAttacks().getProtections() ); web_apps[rule_config.getAssetName()] = web_app; } @@ -1271,7 +1280,8 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( current_identifier, rule_config.getContext(), policy, - rule_annotations + rule_annotations, + default_mode ); createSnortSections( @@ -1282,7 +1292,8 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( practice_id, current_identifier, policy, - rule_annotations + rule_annotations, + default_mode ); createFileSecuritySections( @@ -1292,11 +1303,18 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( rule_annotations[AnnotationTypes::PRACTICE], "assetId(" + rule_config.getAssetId() + ")", policy, - rule_annotations + rule_annotations, + default_mode ); if (!web_apps.count(rule_config.getAssetName())) { - createWebAppSection(policy, rule_config, practice_id, asset_name, default_mode, rule_annotations); + createWebAppSection( + policy, + rule_config, + practice_id, + asset_name, + default_mode, + rule_annotations); } } @@ -1568,6 +1586,7 @@ PolicyMakerUtils::createPolicyElementsByRule(splited_host_name), std::get<2>(splited_host_name), log_triggers[rule_annotations[AnnotationTypes::TRIGGER]].getTriggerId(), + rule.getMode(), policy, rule_annotations ); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 1fde746..99310e1 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -216,6 +216,18 @@ getFecApplicable(const string &command_output) return genError("Could not determine if fec applicable"); } +Maybe +getSMCBasedMgmtId(const string &command_output) +{ + return getAttr(command_output, "Mgmt object UUID was not found"); +} + +Maybe +getSMCBasedMgmtName(const string &command_output) +{ + return getAttr(command_output, "Mgmt object Name was not found"); +} + Maybe getSmbObjectName(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index df0f164..55a3606 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -33,7 +33,10 @@ SHELL_PRE_CMD("read sdwan data", "(cpsdwan get_data > /tmp/cpsdwan_getdata_orch.json~) " "&& (mv /tmp/cpsdwan_getdata_orch.json~ /tmp/cpsdwan_getdata_orch.json)") -#endif +#endif //gaia || smb +#if defined(smb) +SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz > /tmp/local.cfg") +#endif //smb #endif #ifdef SHELL_CMD_HANDLER @@ -115,6 +118,22 @@ SHELL_CMD_HANDLER( "cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:VPN_1/ {print $3}' | head -n 1", getGWIPSecVPNBlade ) +SHELL_CMD_HANDLER( + "SMCBasedMgmtId", + "domain_uuid=$(jq -r .domain_uuid /tmp/cpsdwan_getdata_orch.json);" + "[ \"$domain_uuid\" != \"null\" ] && echo \"$domain_uuid\" ||" + "cat $FWDIR/database/myself_objects.C " + "| awk -F'[{}]' '/:masters/ { found=1; next } found && /:Uid/ { uid=tolower($2); print uid; exit }'", + getSMCBasedMgmtId +) +SHELL_CMD_HANDLER( + "SMCBasedMgmtName", + "domain_name=$(jq -r .domain_name /tmp/cpsdwan_getdata_orch.json);" + "[ \"$domain_name\" != \"null\" ] && echo \"$domain_name\" ||" + "cat $FWDIR/database/myself_objects.C " + "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'", + getSMCBasedMgmtName +) #endif //gaia #if defined(smb) @@ -148,6 +167,23 @@ SHELL_CMD_HANDLER( "cat $FWDIR/conf/active_blades.txt | grep -o 'IPS [01]' | cut -d ' ' -f2", getSmbGWIPSecVPNBlade ) +SHELL_CMD_HANDLER( + "SMCBasedMgmtId", + "domain_uuid=$(jq -r .domain_uuid /tmp/cpsdwan_getdata_orch.json);" + "[ \"$domain_uuid\" != \"null\" ] && echo \"$domain_uuid\" ||" + "cat /tmp/local.cfg " + "| awk -F'[{}]' '/:masters/ { found=1; next } found && /:Uid/ { uid=tolower($2); print uid; exit }'", + getSMCBasedMgmtId +) + +SHELL_CMD_HANDLER( + "SMCBasedMgmtName", + "domain_name=$(jq -r .domain_name /tmp/cpsdwan_getdata_orch.json);" + "[ \"$domain_name\" != \"null\" ] && echo \"$domain_name\" ||" + "cat /tmp/local.cfg " + "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'", + getSMCBasedMgmtName +) #endif//smb SHELL_CMD_OUTPUT("kernel_version", "uname -r") @@ -190,3 +226,9 @@ FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease) FILE_CONTENT_HANDLER("AppSecModelVersion", "/etc/cp/conf/waap/waap.data", getWaapModelVersion) #endif // FILE_CONTENT_HANDLER + +#ifdef SHELL_POST_CMD +#if defined(smb) +SHELL_POST_CMD("remove local.cfg", "rm -rf /tmp/local.cfg") +#endif //smb +#endif diff --git a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc index 97830e1..e5e486c 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolving_handler.cc @@ -64,6 +64,12 @@ private: #undef FILE_CONTENT_HANDLER }; +#define SHELL_POST_CMD(NAME, COMMAND) {NAME, COMMAND}, + map shell_post_commands = { + #include "details_resolver_impl.h" + }; +#undef SHELL_POST_CMD + map DetailsResolvingHanlder::Impl::getResolvedDetails() const { @@ -114,6 +120,18 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const in_file->close(); } + for (auto &shell_post_command : shell_post_commands) { + const string &name = shell_post_command.first; + const string &command = shell_post_command.second; + Maybe command_ret = shell->getExecReturnCode(command, timeout); + + if (!command_ret.ok()) { + dbgWarning(D_AGENT_DETAILS) << "Failed to run post-command " << name; + } else if (*command_ret) { + dbgWarning(D_AGENT_DETAILS) << "Post-command " << name << " failed (rc: " << *command_ret << ")"; + } + } + I_AgentDetailsReporter *reporter = Singleton::Consume::by(); reporter->addAttr(resolved_details, true); diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 614af4a..10c4b91 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1012,10 +1012,11 @@ private: HybridModeMetricEvent().notify(); if (!response.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to get the update. Error: " << response.getErr(); orch_status->setFieldStatus( OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::FAILED, - response.getErr() + "Warning: Agent/Gateway failed during the update process. Contact Check Point support." ); return genError(response.getErr()); diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 38a8b7a..4b7ebfd 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -90,6 +90,7 @@ public: "/api/v1/agents/events", _, MessageCategory::LOG, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index a827bc2..e1ab0ff 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -473,10 +473,11 @@ FogAuthenticator::authenticateAgent() auto orc_status = Singleton::Consume::by(); credentials = getCredentials(); if (!credentials.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to authenticate the agent: " << credentials.getErr(); orc_status->setFieldStatus( OrchestrationStatusFieldType::REGISTRATION, OrchestrationStatusResult::FAILED, - credentials.getErr() + "Warning: Agent/Gateway failed the authentication. Contact Check Point support." ); return genError(credentials.getErr()); } @@ -516,7 +517,7 @@ FogAuthenticator::authenticateAgent() orc_status->setFieldStatus( OrchestrationStatusFieldType::REGISTRATION, OrchestrationStatusResult::FAILED, - access_token.getErr() + "Warning: Agent/Gateway failed to receive access token. Contact Check Point support." ); } int next_session_req = max( diff --git a/components/security_apps/waap/include/i_serialize.h b/components/security_apps/waap/include/i_serialize.h index 00299a7..4723af2 100755 --- a/components/security_apps/waap/include/i_serialize.h +++ b/components/security_apps/waap/include/i_serialize.h @@ -154,6 +154,7 @@ protected: if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); auto req_status = messaging->sendSyncMessage( method, uri, @@ -161,6 +162,10 @@ protected: MessageCategory::GENERIC, req_md ); + if (!req_status.ok()) { + dbgWarning(D_WAAP) << "failed to send request to uri: " << uri + << ", error: " << req_status.getErr().toString(); + } return req_status.ok(); } auto req_status = messaging->sendSyncMessage( @@ -169,6 +174,10 @@ protected: obj, MessageCategory::GENERIC ); + if (!req_status.ok()) { + dbgWarning(D_WAAP) << "failed to send request to uri: " << uri + << ", error: " << req_status.getErr().toString(); + } return req_status.ok(); } @@ -204,6 +213,7 @@ protected: if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); return messaging->sendSyncMessageWithoutResponse( method, uri, diff --git a/components/security_apps/waap/include/i_transaction.h b/components/security_apps/waap/include/i_transaction.h index 1af9157..2c727c1 100755 --- a/components/security_apps/waap/include/i_transaction.h +++ b/components/security_apps/waap/include/i_transaction.h @@ -20,6 +20,7 @@ #include "../waap_clib/WaapOpenRedirect.h" #include "../waap_clib/FpMitigation.h" #include "../waap_clib/DeepParser.h" +#include "../waap_clib/OASchemaUpdaterConfConstant.h" #include "http_inspection_events.h" enum HeaderType { @@ -29,6 +30,7 @@ enum HeaderType { COOKIE_HEADER, REFERER_HEADER, CONTENT_TYPE_HEADER, + AUTHORIZATION_HEADER, CLEAN_HEADER, OTHER_KNOWN_HEADERS }; @@ -135,6 +137,7 @@ public: virtual void add_response_body_chunk(const char* data, int data_len) = 0; virtual void end_response_body() = 0; virtual void end_response() = 0; + virtual const std::string& getResponseBody() = 0; virtual void collectFoundPatterns() = 0; virtual ReportIS::Severity computeEventSeverityFromDecision() const = 0; diff --git a/components/security_apps/waap/waap_clib/CMakeLists.txt b/components/security_apps/waap/waap_clib/CMakeLists.txt index 09e249b..87ea950 100755 --- a/components/security_apps/waap/waap_clib/CMakeLists.txt +++ b/components/security_apps/waap/waap_clib/CMakeLists.txt @@ -86,6 +86,7 @@ add_library(waap_clib ParserPercentEncode.cc ParserPairs.cc Waf2Util2.cc + ParserPDF.cc ) add_definitions("-Wno-unused-function") diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index ee32993..3b16d37 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -26,6 +26,7 @@ #include "ParserPercentEncode.h" #include "ParserPairs.h" #include "ParserDelimiter.h" +#include "ParserPDF.h" #include "WaapAssetState.h" #include "Waf2Regex.h" #include "Waf2Util.h" @@ -1146,9 +1147,15 @@ DeepParser::createInternalParser( )); offset = 0; } else if (isTopData && (isBinaryType || m_pWaapAssetState->isBinarySampleType(cur_val))) { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file"; - m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); - offset = 0; + if (isPDFDetected(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a PDF file"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } } } if (offset < 0) { @@ -1473,3 +1480,12 @@ DeepParser::shouldEnforceDepthLimit(const std::shared_ptr &parser) c } return false; } + +bool +DeepParser::isPDFDetected(const std::string &cur_val) const +{ + static const std::string PDF_header("%PDF-"); + if (cur_val.size() < 10) + return false; + return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos; +} diff --git a/components/security_apps/waap/waap_clib/DeepParser.h b/components/security_apps/waap/waap_clib/DeepParser.h index 6cf5504..cda51e0 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.h +++ b/components/security_apps/waap/waap_clib/DeepParser.h @@ -173,6 +173,7 @@ private: bool shouldEnforceDepthLimit(const std::shared_ptr& parser) const; void setLocalMaxObjectDepth(size_t depth) { m_localMaxObjectDepth = depth; } void setGlobalMaxObjectDepthReached() { m_globalMaxObjectDepthReached = true; } + bool isPDFDetected(const std::string &cur_val) const; bool m_deepParserFlag; std::stack> m_splitTypesStack; // depth, splitIndex, splitType std::deque> m_parsersDeque; diff --git a/components/security_apps/waap/waap_clib/OASchemaUpdaterConfConstant.h b/components/security_apps/waap/waap_clib/OASchemaUpdaterConfConstant.h new file mode 100644 index 0000000..fceab31 --- /dev/null +++ b/components/security_apps/waap/waap_clib/OASchemaUpdaterConfConstant.h @@ -0,0 +1,13 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + diff --git a/components/security_apps/waap/waap_clib/ParserPDF.cc b/components/security_apps/waap/waap_clib/ParserPDF.cc new file mode 100644 index 0000000..d4fbfdd --- /dev/null +++ b/components/security_apps/waap/waap_clib/ParserPDF.cc @@ -0,0 +1,111 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ParserPDF.h" +#include "Waf2Util.h" +#include "debug.h" +#include + +USE_DEBUG_FLAG(D_WAAP_PARSER_PDF); +USE_DEBUG_FLAG(D_WAAP); + +const std::string ParserPDF::m_parserName = "ParserPDF"; +const char* PDF_TAIL = "%%EOF"; + +ParserPDF::ParserPDF( + IParserStreamReceiver &receiver, + size_t parser_depth +) : + m_receiver(receiver), + m_state(s_start), + m_parser_depth(parser_depth) +{} + +ParserPDF::~ParserPDF() +{} + +size_t +ParserPDF::push(const char *buf, size_t len) +{ + dbgTrace(D_WAAP_PARSER_PDF) + << "buf=" + << buf + << "len=" + << len; + + const char *c; + + if (m_state == s_error) { + return 0; + } + if (len == 0) + { + dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): end of stream. m_state=" << m_state; + + if (m_state == s_end) { + m_receiver.onKvDone(); + } else { + m_state = s_error; + } + return 0; + } + + switch (m_state) { + case s_start: + m_state = s_body; + CP_FALL_THROUGH; + case s_body: + c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL); + dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c; + if (c) { + m_state = s_end; + CP_FALL_THROUGH; + } else { + break; + } + case s_end: + if (m_receiver.onKey("PDF", 3) != 0) { + m_state = s_error; + return 0; + } + if (m_receiver.onValue("", 0) != 0) { + m_state = s_error; + return 0; + } + break; + case s_error: + break; + default: + dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): unknown state: " << m_state; + m_state = s_error; + return 0; + } + + return len; +} + + +void ParserPDF::finish() +{ + push(NULL, 0); +} + +const std::string& ParserPDF::name() const +{ + return m_parserName; +} + +bool ParserPDF::error() const +{ + return m_state == s_error; +} diff --git a/components/security_apps/waap/waap_clib/ParserPDF.h b/components/security_apps/waap/waap_clib/ParserPDF.h new file mode 100644 index 0000000..685c85c --- /dev/null +++ b/components/security_apps/waap/waap_clib/ParserPDF.h @@ -0,0 +1,47 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __PARSER_PDF_H__ +#define __PARSER_PDF_H__ + +#include "ParserBase.h" +#include + +#define MAX_HEADER_LOOKUP 64 +#define MAX_TAIL_LOOKUP 5 + +class ParserPDF : public ParserBase { +public: + ParserPDF(IParserStreamReceiver &receiver, size_t parser_depth); + virtual ~ParserPDF(); + virtual size_t push(const char *buf, size_t len); + virtual void finish(); + virtual const std::string &name() const; + virtual bool error() const; + virtual size_t depth() { return 1; } + +private: + enum state { + s_start, + s_body, + s_end, + s_error + }; + + IParserStreamReceiver &m_receiver; + enum state m_state; + static const std::string m_parserName; + size_t m_parser_depth; +}; + +#endif // __PARSER_PDF_H__ diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index 8452d0e..b80e5f3 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -706,6 +706,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() MessageMetadata req_md(getLearningHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); + req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); bool ok = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/api/sync", diff --git a/components/security_apps/waap/waap_clib/Telemetry.cc b/components/security_apps/waap/waap_clib/Telemetry.cc index 45e37ae..55e9386 100755 --- a/components/security_apps/waap/waap_clib/Telemetry.cc +++ b/components/security_apps/waap/waap_clib/Telemetry.cc @@ -156,6 +156,7 @@ void WaapTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data) { initMetrics(); + average_latency.report(data.elapsedTime); switch (data.method) { case POST: diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc index 092203b..1ef91de 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc @@ -12,6 +12,7 @@ // limitations under the License. #include +#include #include #include "WaapOverrideFunctor.h" #include "Waf2Engine.h" @@ -47,14 +48,16 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::C bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx) { boost::cmatch what; + std::string tagLower = tag; + std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower); try { - if (tag == "url") { - return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUri().c_str(), what, rx); + if (tagLower == "url") { + return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx); } - else if (tag == "hostname") { + else if (tagLower == "hostname") { return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx); } - else if (tag == "sourceidentifier") { + else if (tagLower == "sourceidentifier") { return NGEN::Regex::regexMatch( __FILE__, __LINE__, @@ -63,7 +66,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx ); } - else if (tag == "keyword") { + else if (tagLower == "keyword") { for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) { if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) { return true; @@ -71,7 +74,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& } return false; } - else if (tag == "paramname" || tag == "paramName") { + else if (tagLower == "paramname") { for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) { return true; @@ -85,7 +88,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& } return false; } - else if (tag == "paramvalue" || tag == "paramValue") { + else if (tagLower == "paramvalue") { for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) { if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) { return true; @@ -96,10 +99,10 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& } return false; } - else if (tag == "paramlocation" || tag == "paramLocation") { + else if (tagLower == "paramlocation") { return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx); } - else if (tag == "responsebody" || tag == "responseBody") { + else if (tagLower == "responsebody") { waf2Transaction.getResponseInspectReasons().setApplyOverride(true); if (!waf2Transaction.getResponseBody().empty()) { boost::smatch matcher; @@ -108,6 +111,32 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& } else { return false; } + } else if (tagLower == "headername") { + if (!waf2Transaction.checkIsHeaderOverrideScanRequired()) { + dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required"; + return false; + } + for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { + std::string value = hdr_pair.first; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) { + return true; + } + } + return false; + } else if (tagLower == "headervalue") { + if (!waf2Transaction.checkIsHeaderOverrideScanRequired()) { + dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required"; + return false; + } + for (auto& hdr_pair : waf2Transaction.getHdrPairs()) { + std::string value = hdr_pair.second; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) { + return true; + } + } + return false; } } catch (std::runtime_error & e) { @@ -116,6 +145,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& } // Unknown tag: should not occur - dbgWarning(D_WAAP) << "Invalid override tag:" << tag; + dbgWarning(D_WAAP) << "Invalid override tag: " << tag; return false; } diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc index 3dadfbb..a2892bc 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc @@ -26,7 +26,8 @@ namespace Waap { errorLimiter(false), rateLimiting(false), collectResponseForLog(false), - applyOverride(false) + applyOverride(false), + triggerReport(false) { } @@ -39,11 +40,12 @@ namespace Waap { " RateLimiting=" << rateLimiting << " ErrorLimiter=" << errorLimiter << " collectResponseForLog=" << collectResponseForLog << - " applyOverride=" << applyOverride; + " applyOverride=" << applyOverride << + " triggerReport=" << triggerReport; return openRedirect || errorDisclosure || rateLimiting || errorLimiter || - collectResponseForLog || applyOverride; + collectResponseForLog || applyOverride || triggerReport; } void @@ -91,6 +93,14 @@ namespace Waap { applyOverride = flag; } + void + ResponseInspectReasons::setTriggerReport(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setTriggerReport) " << triggerReport << " to " << + flag; + triggerReport = flag; + } + bool ResponseInspectReasons::getApplyOverride(void) { diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h index e2e58e0..51937fd 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h @@ -25,6 +25,7 @@ public: void setErrorLimiter(bool flag); void setCollectResponseForLog(bool flag); void setApplyOverride(bool flag); + void setTriggerReport(bool flag); bool getApplyOverride(void); private: @@ -34,6 +35,7 @@ private: bool rateLimiting; bool collectResponseForLog; bool applyOverride; + bool triggerReport; }; } diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index f41d9cc..22c3d6b 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -56,8 +56,10 @@ USE_DEBUG_FLAG(D_WAAP); USE_DEBUG_FLAG(D_WAAP_ULIMITS); USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION); USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); +USE_DEBUG_FLAG(D_WAAP_HEADERS); using namespace ReportIS; +using namespace std; #define MAX_REQUEST_BODY_SIZE (2*1024) #define MAX_RESPONSE_BODY_SIZE (2*1024) @@ -89,6 +91,9 @@ void Waf2Transaction::start_response(int response_status, int http_version) dbgTrace(D_WAAP) << "[transaction:" << this << "] start_response(response_status=" << response_status << "," << " http_version=" << http_version << ")"; m_responseStatus = response_status; + if (m_triggerReport) { + m_responseInspectReasons.setTriggerReport(false); + } if(m_responseStatus == 404) { @@ -233,6 +238,12 @@ void Waf2Transaction::end_response_body() dbgTrace(D_WAAP) << "[transaction:" << this << "] end_response_body"; } +const std::string & +Waf2Transaction::getResponseBody() +{ + return m_response_body; +} + void Waf2Transaction::scanErrDisclosureBuffer() { if (m_responseStatus >= 400 && m_responseStatus <= 599) { @@ -302,19 +313,25 @@ Waf2Transaction::Waf2Transaction() : m_deepParser(m_pWaapAssetState, m_scanner, this), m_deepParserReceiver(m_deepParser), m_scanResult(NULL), + m_response_body(), m_request_body_bytes_received(0), m_response_body_bytes_received(0), m_processedUri(false), m_processedHeaders(false), + m_isHeaderOverrideScanRequired(false), m_isScanningRequired(false), m_responseStatus(0), m_responseInspectReasons(), m_responseInjectReasons(), m_index(-1), m_triggerLog(), + m_triggerReport(false), is_schema_validation(false), m_waf2TransactionFlags() -{} +{ + I_TimeGet *timeGet = Singleton::Consume::by(); + m_entry_time = chrono::duration_cast(timeGet->getMonotonicTime()); +} Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState) : TableOpaqueSerialize(this), @@ -334,19 +351,25 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState m_deepParser(m_pWaapAssetState, m_scanner, this), m_deepParserReceiver(m_deepParser), m_scanResult(NULL), + m_response_body(), m_request_body_bytes_received(0), m_response_body_bytes_received(0), m_processedUri(false), m_processedHeaders(false), + m_isHeaderOverrideScanRequired(false), m_isScanningRequired(false), m_responseStatus(0), m_responseInspectReasons(), m_responseInjectReasons(), m_index(-1), m_triggerLog(), + m_triggerReport(false), is_schema_validation(false), m_waf2TransactionFlags() -{} +{ + I_TimeGet *timeGet = Singleton::Consume::by(); + m_entry_time = chrono::duration_cast(timeGet->getMonotonicTime()); +} Waf2Transaction::~Waf2Transaction() { dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser"; @@ -362,6 +385,8 @@ HeaderType Waf2Transaction::detectHeaderType(const char* name, int name_len) { static const char content_type[] = "content-Type"; static const char cookie[] = "cookie"; static const char referer[] = "referer"; + static const char authorization[] = "authorization"; + dbgTrace(D_WAAP_HEADERS) << "Detecting header type for header >>>" << std::string(name, name_len) << "<<<"; if (memcaseinsensitivecmp(name, name_len, host, sizeof(host) - 1)) { return HeaderType::HOST_HEADER; @@ -378,6 +403,9 @@ HeaderType Waf2Transaction::detectHeaderType(const char* name, int name_len) { if (memcaseinsensitivecmp(name, name_len, referer, sizeof(referer) - 1)) { return HeaderType::REFERER_HEADER; } + if (memcaseinsensitivecmp(name, name_len, authorization, sizeof(authorization) - 1)) { + return HeaderType::AUTHORIZATION_HEADER; + } return UNKNOWN_HEADER; } @@ -422,37 +450,100 @@ HeaderType Waf2Transaction::checkCleanHeader(const char* name, int name_len, con return CLEAN_HEADER; } } - - static const std::string authorization("authorization"); - if (memcaseinsensitivecmp(name, name_len, authorization.data(), authorization.size())) { - dbgTrace(D_WAAP) << "[transaction:" << this << "] special header '" << std::string(name, name_len) << - "' - detect base64 to determine cleanliness ..."; - - std::string result; - int decodedCount = 0; - int deletedCount = 0; - - std::string v(value, value_len); - boost::algorithm::to_lower(v); - const std::string negotiate("negotiate "); - - if (boost::algorithm::starts_with(v, negotiate)) { - v = v.substr(negotiate.size(), v.size() - negotiate.size()); - - // Detect potential base64 match after the "Negotiate " prefix - Waap::Util::b64Decode(v, b64DecodeChunk, decodedCount, deletedCount, result); - if (result.empty() && (deletedCount + decodedCount == 1)) { - // Decoded 1 base64 chunk and nothing left behind it - dbgTrace(D_WAAP) << "[transaction:" << this << "] special header '" << - std::string(name, name_len) << " is clean"; - return CLEAN_HEADER; - } - } - } } return UNKNOWN_HEADER; } +void Waf2Transaction::parseAuthorization(const char* value, int value_len, const std::string &header_name) +{ +#ifdef NO_HEADERS_SCAN + return; +#endif + static const std::string authorization("authorization"); + + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this + << "] special header '" << header_name; + + std::string result; + std::string v(value, value_len); + int decodedCount; + int deletedCount; + static const char *bearer = "bearer "; + static const std::string bearer_str(bearer); + if (memcaseinsensitivecmp(value, sizeof(bearer) - 1, bearer, sizeof(bearer) - 1)) { + dbgTrace(D_WAAP_HEADERS) << "Checking JWT chunk for alg = none" << v; + // parse buffer v till first dot, + // convert this chunk from base 64 to json string and get value for "alg" key + // if alg == "none" then this is JWT with no signature and we should block it + // if alg != "none" then this is JWT with signature and we should not block it + char* p = (char*)v.c_str() + bearer_str.size(); + char* p1 = strchr(p, '.'); + if (p1 != NULL) { + *p1 = '\0'; + std::string header(p); + alignBase64Chunk(header); + Waap::Util::b64Decode(header, b64DecodeChunk, decodedCount, deletedCount, result); + if (!result.empty()) { + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name + << " base64 encoded for JWT alg testing"; + // parse JSON string and get value for "alg" key and check its value to be equal to "none" + // if alg == "none" then this is JWT with no signature and we should block it + boost::algorithm::to_lower(result); + static bool err = false; + static const SingleRegex invalid_jwt_alg_re( + "\"alg\"\\s*:\\s*\"none\"", + err, + "invalid_jwt_alg_" + ); + if (invalid_jwt_alg_re.hasMatch(result)) { + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name + << " bad JWT algotitm detected, blocking"; + Waf2ScanResult new_res = Waf2ScanResult(); + new_res.location = "header"; + new_res.param_name = header_name; + new_res.unescaped_line = header_name + ": " + result; + new_res.score = 9.0; + new_res.keyword_matches.push_back("alg:none"); + new_res.scoreArray.push_back(9.0); + new_res.attack_types.insert("JWT"); + if (m_scanResult != NULL) { + dbgTrace(D_WAAP_HEADERS) << "LastScanResult = " << m_scanResult; + m_scanResult->mergeFrom(new_res); + } else { + dbgTrace(D_WAAP_HEADERS) << "LastScanResult is NULL"; + m_scanResult = new Waf2ScanResult(new_res); + } + return; + } + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name + << " clean JWT algotitm detected"; + return; + } else { + parseGenericHeaderValue(header_name, value, value_len); + } + } + return; + } + + static const char *negotiate = "negotiate "; + static const std::string negotiate_str(negotiate); + if (memcaseinsensitivecmp(value, sizeof(negotiate) - 1, negotiate, sizeof(negotiate) - 1)) { + v = v.substr(negotiate_str.size(), v.size() - negotiate_str.size()); + + // Detect potential base64 match after the "Negotiate " prefix + alignBase64Chunk(v); + Waap::Util::b64Decode(v, b64DecodeChunk, decodedCount, deletedCount, result); + if (result.empty() && (deletedCount + decodedCount == 1)) { + // Decoded 1 base64 chunk and nothing left behind it + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] special header '" << header_name + << " is clean - clean base64 for negotiate"; + return; + } else { + parseGenericHeaderValue(header_name, value, value_len); + } + } +} + // Methods below are callbacks that are called during HTTP transaction processing by the front-end server/proxy void Waf2Transaction::start() { dbgTrace(D_WAAP) << "[Waf2Transaction::start():" << this << "] start"; @@ -744,14 +835,14 @@ void Waf2Transaction::parseContentType(const char* value, int value_len) ctp.push(value, value_len); ctp.finish(); - dbgTrace(D_WAAP) << "[transaction:" << this << "] ctp detected content type: '" << + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] ctp detected content type: '" << ctp.contentTypeDetected.c_str() << "'"; // The above fills m_contentTypeDetected m_contentType = Waap::Util::detectContentType(ctp.contentTypeDetected.c_str()); // extract boundary string required for parsing multipart-form-data stream if (m_contentType == Waap::Util::CONTENT_TYPE_MULTIPART_FORM) { - dbgTrace(D_WAAP) << "content_type detected: " << Waap::Util::getContentTypeStr(m_contentType) << + dbgTrace(D_WAAP_HEADERS) << "content_type detected: " << Waap::Util::getContentTypeStr(m_contentType) << "; boundary='" << ctp.boundaryFound.c_str() << "'"; m_deepParser.setMultipartBoundary(ctp.boundaryFound); } @@ -773,7 +864,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len) #endif if (value_len > 0) { - dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the cookie value"; + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the cookie value"; m_deepParser.m_key.push("cookie", 6); ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';'); cookieValueParser.push(value, value_len); @@ -788,7 +879,7 @@ void Waf2Transaction::parseReferer(const char* value, int value_len) #ifdef NO_HEADERS_SCAN return; #endif - dbgTrace(D_WAAP) << "Parsed Referer. Referer URI: " << m_uriReferer; + dbgTrace(D_WAAP_HEADERS) << "Parsed Referer. Referer URI: " << m_uriReferer; std::string referer(value, value_len); std::vector regex_matches; @@ -815,7 +906,7 @@ void Waf2Transaction::parseUnknownHeaderName(const char* name, int name_len) // Apply signatures on all other, header names, unless they are considered "good" ones to skip scanning them. if (name_len && !m_pWaapAssetState->getSignatures()->good_header_name_re.hasMatch(std::string(name, name_len))) { - dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header name"; + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the header name"; m_deepParser.m_key.push("header", 6); ParserRaw headerNameParser(m_deepParserReceiver, 0, std::string(name, name_len)); headerNameParser.push(name, name_len); @@ -834,7 +925,7 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con return; } - dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header value"; + dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the header value"; m_deepParser.m_key.push("header", 6); ParserRaw headerValueParser(m_deepParserReceiver, 0, headerName); headerValueParser.push(value, value_len); @@ -844,11 +935,11 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con }; // Scan relevant headers to detect attacks inside them -void Waf2Transaction::scanSpecificHeder(const char* name, int name_len, const char* value, int value_len) +void Waf2Transaction::scanSpecificHeader(const char* name, int name_len, const char* value, int value_len) { HeaderType header_t = detectHeaderType(name, name_len); std::string headerName = std::string(name, name_len); - + dbgTrace(D_WAAP_HEADERS) << "Processing the header " << headerName; switch (header_t) { case HeaderType::COOKIE_HEADER: @@ -857,6 +948,9 @@ void Waf2Transaction::scanSpecificHeder(const char* name, int name_len, const ch case HeaderType::REFERER_HEADER: parseReferer(value, value_len); break; + case HeaderType::AUTHORIZATION_HEADER: + parseAuthorization(value, value_len, headerName); + break; case HeaderType::UNKNOWN_HEADER: { HeaderType headerType = checkCleanHeader(name, name_len, value, value_len); if(headerType == HeaderType::CLEAN_HEADER) { @@ -954,7 +1048,7 @@ void Waf2Transaction::scanHeaders() // Scan relevant headers for attacks for (auto it = hdrs_map.begin(); it != hdrs_map.end(); ++it) { - scanSpecificHeder(it->first.c_str(), it->first.size(), + scanSpecificHeader(it->first.c_str(), it->first.size(), it->second.c_str(), it->second.size()); } } @@ -1171,6 +1265,12 @@ void Waf2Transaction::end_request() { // Enable response headers processing if response scanning is enabled in policy auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL; m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable); + + auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL; + if (isTriggerReportExists(triggerPolicy)) { + m_responseInspectReasons.setTriggerReport(true); + dbgTrace(D_WAAP) << "setTriggerReport(true)"; + } } void Waf2Transaction::extractEnvSourceIdentifier() @@ -1393,6 +1493,7 @@ Waf2Transaction::decideAfterHeaders() return false; } + m_isHeaderOverrideScanRequired = true; m_overrideState = getOverrideState(sitePolicy); // Select scores pool by location (but use forced pool when forced) @@ -1648,6 +1749,12 @@ Waf2Transaction::sendLog() const auto& autonomousSecurityDecision = std::dynamic_pointer_cast( m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); + I_TimeGet *timeGet = Singleton::Consume::by(); + auto finish = timeGet->getMonotonicTime(); + auto diff = chrono::duration_cast(finish) - m_entry_time; + telemetryData.elapsedTime = diff.count(); + dbgTrace(D_WAAP) << "latency updated, time: " << diff.count() << "ms"; + if (m_methodStr == "POST") { telemetryData.method = POST; } else if (m_methodStr == "GET") { diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index e05087a..a3d514a 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -38,6 +38,7 @@ #include "i_transaction.h" #include "i_waap_telemetry.h" #include "i_deepAnalyzer.h" +#include "i_time_get.h" #include "table_opaque.h" #include "WaapResponseInspectReasons.h" #include "WaapResponseInjectReasons.h" @@ -45,6 +46,7 @@ #include "WaapOpenRedirectPolicy.h" #include "WaapScanner.h" #include "singleton.h" +#include "OASchemaUpdaterConfConstant.h" struct DecisionTelemetryData; class Waf2Transaction; @@ -60,6 +62,7 @@ class Waf2Transaction : public Singleton::Consume, private boost::noncopyable, Singleton::Consume, + Singleton::Consume, Singleton::Consume { public: @@ -76,6 +79,7 @@ public: void set_host(const char *host); // getters + uint64_t getElapsedTime() const; const std::string& getRemoteAddr() const; virtual const std::string getUri() const; const std::string getUriStr() const; @@ -143,6 +147,7 @@ public: void start_response_body(); void add_response_body_chunk(const char* data, int data_len); void end_response_body(); + const std::string & getResponseBody(); void end_response(); void extractEnvSourceIdentifier(); void finish(); @@ -212,19 +217,21 @@ public: } // LCOV_EXCL_STOP - bool reportScanResult(const Waf2ScanResult &res); bool shouldIgnoreOverride(const Waf2ScanResult &res); Waap::OpenRedirect::State &getOpenRedirectState() { return m_openRedirectState; } IWaapConfig* getSiteConfig() { return m_siteConfig; } void addNote(const std::string ¬e) { m_notes.push_back(note); } - const std::string &getResponseBody(void) const { return m_response_body; } Waap::ResponseInspectReasons &getResponseInspectReasons(void) { return m_responseInspectReasons; } + // LCOV_EXCL_START Reason: This function is tested in system tests + bool checkIsHeaderOverrideScanRequired(); + // LCOV_EXCL_STOP private: int finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock); const std::shared_ptr getTriggerLog(const std::shared_ptr& triggerPolicy) const; + bool isTriggerReportExists(const std::shared_ptr &triggerPolicy); void sendAutonomousSecurityLog( const std::shared_ptr& triggerLog, bool shouldBlock, @@ -261,15 +268,17 @@ private: void parseContentType(const char* value, int value_len); void parseCookie(const char* value, int value_len); void parseReferer(const char* value, int value_len); + void parseAuthorization(const char* value, int value_len, const std::string &header_name); void parseUnknownHeaderName(const char* name, int name_len); void parseGenericHeaderValue(const std::string &headerName, const char* value, int value_len); - void scanSpecificHeder(const char* name, int name_len, const char* value, int value_len); + void scanSpecificHeader(const char* name, int name_len, const char* value, int value_len); void detectSpecificHeader(const char* name, int name_len, const char* value, int value_len); void detectHeaders(); void scanHeaders(); void clearRequestParserState(); void scanErrDisclosureBuffer(); + std::chrono::milliseconds m_entry_time; std::shared_ptr m_pWaapAssetState; bool m_ignoreScore; // override the scoring filter and (effectively) take the last suspicious parameter, // instead of the one with highest score that is > SCORE_THRESHOLD @@ -334,6 +343,7 @@ private: bool m_processedUri; bool m_processedHeaders; + bool m_isHeaderOverrideScanRequired; bool m_isScanningRequired; int m_responseStatus; Waap::ResponseInspectReasons m_responseInspectReasons; @@ -345,6 +355,7 @@ private: // Cached pointer to const triggerLog (hence mutable) mutable std::shared_ptr m_triggerLog; + bool m_triggerReport; bool is_schema_validation = false; Waf2TransactionFlags m_waf2TransactionFlags; }; diff --git a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc index ad39478..89b420a 100755 --- a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc +++ b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc @@ -551,6 +551,11 @@ const std::set Waf2Transaction::getFoundPatterns() const return m_found_patterns; } +bool Waf2Transaction::checkIsHeaderOverrideScanRequired() +{ + return m_isHeaderOverrideScanRequired; +} + Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy) { Waap::Override::State overrideState; @@ -566,6 +571,7 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy) if (overridePolicy) { // later we will run response overrides overrideStateResponse.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false); } + m_isHeaderOverrideScanRequired = false; return overrideStateResponse; } @@ -592,6 +598,23 @@ const std::shared_ptr Waf2Transaction::getTriggerLog(const s return m_triggerLog; } +bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr< + Waap::Trigger::Policy> &triggerPolicy) +{ + if (!triggerPolicy) { + return false; + } + if (m_triggerReport) { + return m_triggerReport; + } + for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { + if (trigger.triggerType == "report") { + return m_triggerReport = true; + } + } + return m_triggerReport; +} + ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const { DecisionType type = m_waapDecision.getHighestPriorityDecisionToLog(); diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index 9222ede..b4f0971 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -732,6 +732,13 @@ inline void replaceAll(std::string& str, const std::string& from, const std::str start_pos += to.length(); // In case 'to' contains 'from', like replacing 'x' with 'yx' } } +inline void alignBase64Chunk (std::string &chunk) +{ + size_t len = chunk.length() % 4; + if (len >= 2) { + chunk.append(4-len, '='); + } +} // Count items in v that are not in ignored_set inline size_t countNotInSet(const std::vector &v, const std::set &ignored_set) { diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 8cd1d6b..f39b3e9 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -170,11 +170,15 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event) waf2Transaction.start(); - char sourceIpStr[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &(event.getSourceIP()), sourceIpStr, INET_ADDRSTRLEN); - - char listeningIpStr[INET_ADDRSTRLEN]; - inet_ntop(AF_INET, &(event.getListeningIP()), listeningIpStr, INET_ADDRSTRLEN); + char sourceIpStr[INET6_ADDRSTRLEN]; + char listeningIpStr[INET6_ADDRSTRLEN]; + if (event.getSourceIP().getType() == IPType::V4) { + inet_ntop(AF_INET, &(event.getSourceIP()), sourceIpStr, INET6_ADDRSTRLEN); + inet_ntop(AF_INET, &(event.getListeningIP()), listeningIpStr, INET6_ADDRSTRLEN); + } else { + inet_ntop(AF_INET6, &(event.getSourceIP()), sourceIpStr, INET6_ADDRSTRLEN); + inet_ntop(AF_INET6, &(event.getListeningIP()), listeningIpStr, INET6_ADDRSTRLEN); + } // Set envelope data waf2Transaction.set_transaction_remote(sourceIpStr, event.getSourcePort()); @@ -515,7 +519,6 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) dbgTrace(D_WAAP) << "HttpBodyResponse"; - // Push the response data chunk to the waf2 engine const char *dataBuf = (const char*)event.getData().data(); size_t dataBufLen = event.getData().size(); diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index 8bd9647..1873694 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(generic_rulebase) add_subdirectory(geo_location) add_subdirectory(http_transaction_data) add_subdirectory(ip_utilities) diff --git a/components/generic_rulebase/CMakeLists.txt b/components/utils/generic_rulebase/CMakeLists.txt similarity index 100% rename from components/generic_rulebase/CMakeLists.txt rename to components/utils/generic_rulebase/CMakeLists.txt diff --git a/components/utils/generic_rulebase/assets_config.cc b/components/utils/generic_rulebase/assets_config.cc new file mode 100644 index 0000000..64e0984 --- /dev/null +++ b/components/utils/generic_rulebase/assets_config.cc @@ -0,0 +1,137 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/assets_config.h" + +#include +#include +#include + +#include "generic_rulebase/generic_rulebase_utils.h" +#include "config.h" +#include "debug.h" +#include "ip_utilities.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +void +RuleAsset::load(cereal::JSONInputArchive &archive_in) +{ + archive_in(cereal::make_nvp("assetId", asset_id)); + archive_in(cereal::make_nvp("assetName", asset_name)); + archive_in(cereal::make_nvp("assetUrls", asset_urls)); + + dbgWarning(D_RULEBASE_CONFIG) << "Adding asset with UID: " << asset_id; +} + +void +RuleAsset::AssetUrl::load(cereal::JSONInputArchive &archive_in) +{ + archive_in(cereal::make_nvp("protocol", protocol)); + transform(protocol.begin(), protocol.end(), protocol.begin(), [](unsigned char c) { return tolower(c); }); + + archive_in(cereal::make_nvp("ip", ip)); + archive_in(cereal::make_nvp("port", port)); + + int value; + if (protocol == "*") { + is_any_proto = true; + } else { + is_any_proto = false; + try { + value = 0; + if(protocol == "udp") value = IPPROTO_UDP; + if(protocol == "tcp") value = IPPROTO_TCP; + if(protocol == "dccp") value = IPPROTO_DCCP; + if(protocol == "sctp") value = IPPROTO_SCTP; + if(protocol == "icmp") value = IPPROTO_ICMP; + if(protocol == "icmpv6") value = IPPROTO_ICMP; + + if (value > static_cast(UINT8_MAX) || value < 0) { + dbgWarning(D_RULEBASE_CONFIG) + << "provided value is not a legal IP protocol number. Value: " + << protocol; + } else { + parsed_proto = value; + } + } catch (...) { + dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal IP protocol. Value: " << protocol; + } + } + + if (port == "*") { + is_any_port = true; + } else { + is_any_port = false; + try { + value = stoi(port); + if (value > static_cast(UINT16_MAX) || value < 0) { + dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port number. Value: " << port; + } else { + parsed_port = value; + } + } catch (...) { + dbgWarning(D_RULEBASE_CONFIG) << "provided value is not a legal port. Value: " << port; + } + } + + if (ip == "*") { + is_any_ip = true; + } else { + is_any_ip = false; + auto ip_addr = IPAddr::createIPAddr(ip); + if (!ip_addr.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Could not create IP address. Error: " << ip_addr.getErr(); + } else { + parsed_ip = ConvertToIpAddress(ip_addr.unpackMove()); + } + } +} + +IpAddress +RuleAsset::AssetUrl::ConvertToIpAddress(const IPAddr &addr) +{ + IpAddress address; + switch (addr.getType()) { + case IPType::UNINITIALIZED: { + address.addr4_t = {0}; + address.ip_type = IP_VERSION_ANY; + break; + } + case IPType::V4: { + address.addr4_t = addr.getIPv4(); + address.ip_type = IP_VERSION_4; + break; + } + case IPType::V6: { + address.addr6_t = addr.getIPv6(); + address.ip_type = IP_VERSION_6; + break; + } + default: + address.addr4_t = {0}; + address.ip_type = IP_VERSION_ANY; + dbgWarning(D_RULEBASE_CONFIG) << "Unsupported IP type: " << static_cast(addr.getType()); + } + return address; +} + +const Assets Assets::empty_assets_config = Assets(); + +void +Assets::preload() +{ + registerExpectedSetting("rulebase", "usedAssets"); +} diff --git a/components/generic_rulebase/evaluators/CMakeLists.txt b/components/utils/generic_rulebase/evaluators/CMakeLists.txt similarity index 100% rename from components/generic_rulebase/evaluators/CMakeLists.txt rename to components/utils/generic_rulebase/evaluators/CMakeLists.txt diff --git a/components/utils/generic_rulebase/evaluators/asset_eval.cc b/components/utils/generic_rulebase/evaluators/asset_eval.cc new file mode 100644 index 0000000..ad0e35c --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/asset_eval.cc @@ -0,0 +1,52 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/asset_eval.h" + +#include +#include + +#include "generic_rulebase/assets_config.h" +#include "config.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +string AssetMatcher::ctx_key = "asset_id"; + +AssetMatcher::AssetMatcher(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams(AssetMatcher::getName(), params.size(), 1, 1); + asset_id = params[0]; +} + +Maybe +AssetMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto bc_asset_id_ctx = env->get(AssetMatcher::ctx_key); + + if (bc_asset_id_ctx.ok()) { + dbgTrace(D_RULEBASE_CONFIG) + << "Asset ID: " + << asset_id + << "; Current set assetId context: " + << *bc_asset_id_ctx; + } else { + dbgTrace(D_RULEBASE_CONFIG) << "Asset ID: " << asset_id << ". Empty context"; + } + + return bc_asset_id_ctx.ok() && *bc_asset_id_ctx == asset_id; +} diff --git a/components/utils/generic_rulebase/evaluators/connection_eval.cc b/components/utils/generic_rulebase/evaluators/connection_eval.cc new file mode 100644 index 0000000..589a96d --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/connection_eval.cc @@ -0,0 +1,299 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/connection_eval.h" + +#include +#include + +#include "generic_rulebase/rulebase_config.h" +#include "config.h" +#include "debug.h" +#include "ip_utilities.h" + +using namespace std; +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +string IpAddressMatcher::ctx_key = "ipAddress"; +string SourceIpMatcher::ctx_key = "sourceIP"; +string DestinationIpMatcher::ctx_key = "destinationIP"; +string SourcePortMatcher::ctx_key = "sourcePort"; +string ListeningPortMatcher::ctx_key = "listeningPort"; +string IpProtocolMatcher::ctx_key = "ipProtocol"; +string UrlMatcher::ctx_key = "url"; + +Maybe +getIpAddrFromEnviroment(I_Environment *env, Context::MetaDataType enum_data_type, const string &str_data_type) +{ + auto ip_str = env->get(enum_data_type); + if (!ip_str.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get " << str_data_type << " from the enviroment."; + return genError("Failed to get " + str_data_type + " from the enviroment."); + } + return IPAddr::createIPAddr(ip_str.unpack()); +} + +bool +checkIfIpInRangesVec(const vector> &values, const IPAddr &ip_to_check) +{ + if (values.size() == 0) { + dbgTrace(D_RULEBASE_CONFIG) << "Ip addersses vector empty. Match is true."; + return true; + } + for (const CustomRange &range : values) { + if (range.contains(ip_to_check)) { + dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss matched: " << ip_to_check; + return true; + } + } + dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss not match: " << ip_to_check; + return false; +} + + +IpAddressMatcher::IpAddressMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> ip_range = CustomRange::createRange(param); + if (!ip_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip. Error: " + ip_range.getErr(); + continue; + } + values.push_back(ip_range.unpack()); + } +} + +Maybe +IpAddressMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + Maybe subject_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::SubjectIpAddr, + "subject ip address" + ); + if (subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack())) return true; + + Maybe other_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::OtherIpAddr, + "other ip address" + ); + if (other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack())) return true; + if (!subject_ip.ok() && !other_ip.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Error in getting subject ip and other ip from the enviroment"; + return false; + } + dbgTrace(D_RULEBASE_CONFIG) << "Ip adderss didn't match"; + return false; +} + +SourceIpMatcher::SourceIpMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> ip_range = CustomRange::createRange(param); + if (!ip_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source ip. Error: " + ip_range.getErr(); + continue; + } + values.push_back(ip_range.unpack()); + } +} + +Maybe +SourceIpMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto direction_maybe = env->get(Context::MetaDataType::Direction); + if (!direction_maybe.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction from the enviroment."; + return false; + } + string direction = direction_maybe.unpack(); + if (direction == "incoming") { + Maybe other_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::OtherIpAddr, + "other ip address" + ); + return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack()); + } else if (direction == "outgoing") { + Maybe subject_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::SubjectIpAddr, + "subject ip address" + ); + return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack()); + } + dbgTrace(D_RULEBASE_CONFIG) << "Source ip adderss didn't match"; + return false; +} + +DestinationIpMatcher::DestinationIpMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> ip_range = CustomRange::createRange(param); + if (!ip_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create destination ip. Error: " + ip_range.getErr(); + continue; + } + values.push_back(ip_range.unpack()); + } +} + +Maybe +DestinationIpMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto direction_maybe = env->get(Context::MetaDataType::Direction); + if (!direction_maybe.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get direction."; + return false; + } + string direction = direction_maybe.unpack(); + if (direction == "outgoing") { + Maybe other_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::OtherIpAddr, + "other ip address" + ); + return other_ip.ok() && checkIfIpInRangesVec(values, other_ip.unpack()); + } else if (direction == "incoming") { + Maybe subject_ip = getIpAddrFromEnviroment( + env, + Context::MetaDataType::SubjectIpAddr, + "subject ip address" + ); + return subject_ip.ok() && checkIfIpInRangesVec(values, subject_ip.unpack()); + } + dbgTrace(D_RULEBASE_CONFIG) << "Destination ip adderss didn't match"; + return false; +} + +SourcePortMatcher::SourcePortMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> port_range = CustomRange::createRange(param); + if (!port_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create source port."; + continue; + } + values.push_back(port_range.unpack()); + } +} + +Maybe +SourcePortMatcher::evalVariable() const +{ + dbgTrace(D_RULEBASE_CONFIG) << "Source is not a match"; + return false; +} + + +ListeningPortMatcher::ListeningPortMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> port_range = CustomRange::createRange(param); + if (!port_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create listening port range."; + continue; + } + values.push_back(port_range.unpack()); + } +} + +Maybe +ListeningPortMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto port_str = env->get(Context::MetaDataType::Port); + if (!port_str.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get port from the enviroment."; + return false; + } + PortNumber port; + if (ConnKeyUtil::fromString(port_str.unpack(), port)) { + if (values.size() == 0) return true; + for (const CustomRange &port_range : values) { + if (port_range.contains(port)) { + dbgTrace(D_RULEBASE_CONFIG) << "Listening port is a match. Value: " << port_str.unpack(); + return true; + } + } + } + dbgTrace(D_RULEBASE_CONFIG) << "Listening port is not a match. Value: " << port_str.unpack(); + return false; +} + +IpProtocolMatcher::IpProtocolMatcher(const vector ¶ms) +{ + for (const string ¶m : params) { + Maybe> proto_range = CustomRange::createRange(param); + if (!proto_range.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to create ip protocol."; + continue; + } + values.push_back(proto_range.unpack()); + } +} + +Maybe +IpProtocolMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto proto_str = env->get(Context::MetaDataType::Protocol); + if (!proto_str.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get ip protocol from the enviroment."; + return false; + } + IPProto protocol; + if (ConnKeyUtil::fromString(proto_str.unpack(), protocol)) { + if (values.size() == 0) return true; + for (const CustomRange &proto_range : values) { + if (proto_range.contains(protocol)) { + dbgTrace(D_RULEBASE_CONFIG) << "Ip protocol is a match. Value: " << proto_str.unpack(); + return true; + } + } + } + dbgTrace(D_RULEBASE_CONFIG) << "Source port is not a match. Value: " << proto_str.unpack(); + return false; +} + +UrlMatcher::UrlMatcher(const vector ¶ms) : values(params) {} + +Maybe +UrlMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto curr_url_ctx = env->get(Context::MetaDataType::Url); + if (!curr_url_ctx.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to get URL from the enviroment."; + return false; + } + + if (values.size() == 0) { + dbgTrace(D_RULEBASE_CONFIG) << "Matched URL on \"any\". Url: " << *curr_url_ctx; + return true; + } + + for (const string &url : values) { + if (*curr_url_ctx == url) { + dbgTrace(D_RULEBASE_CONFIG) << "Matched URL. Value: " << *curr_url_ctx; + return true; + } + } + + dbgTrace(D_RULEBASE_CONFIG) << "URL is not a match. Value: " << *curr_url_ctx; + return false; +} diff --git a/components/utils/generic_rulebase/evaluators/http_transaction_data_eval.cc b/components/utils/generic_rulebase/evaluators/http_transaction_data_eval.cc new file mode 100644 index 0000000..d6560fb --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/http_transaction_data_eval.cc @@ -0,0 +1,168 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/http_transaction_data_eval.h" + +#include +#include + +#include "http_transaction_data.h" +#include "environment/evaluator_templates.h" +#include "i_environment.h" +#include "singleton.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; +using namespace EnvironmentHelper; + +EqualHost::EqualHost(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams("EqualHost", params.size(), 1, 1); + host = params[0]; +} + +Maybe +EqualHost::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto host_ctx = env->get(HttpTransactionData::host_name_ctx); + + if (!host_ctx.ok()) + { + return false; + } + + std::string lower_host_ctx = host_ctx.unpack(); + std::transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower); + + std::string lower_host = host; + std::transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower); + + + if (lower_host_ctx == lower_host) return true; + size_t pos = lower_host_ctx.find_last_of(':'); + if (pos == string::npos) return false; + lower_host_ctx = string(lower_host_ctx.data(), pos); + return lower_host_ctx == lower_host; +} + +WildcardHost::WildcardHost(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams("WildcardHost", params.size(), 1, 1); + host = params[0]; +} + +Maybe +WildcardHost::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto host_ctx = env->get(HttpTransactionData::host_name_ctx); + + if (!host_ctx.ok()) + { + return false; + } + + string lower_host_ctx = host_ctx.unpack(); + transform(lower_host_ctx.begin(), lower_host_ctx.end(), lower_host_ctx.begin(), ::tolower); + + dbgTrace(D_RULEBASE_CONFIG) << "found host in current context: " << lower_host_ctx; + + size_t pos = lower_host_ctx.find_first_of("."); + if (pos == string::npos) { + return false; + } + + lower_host_ctx = "*" + lower_host_ctx.substr(pos, lower_host_ctx.length()); + + string lower_host = host; + transform(lower_host.begin(), lower_host.end(), lower_host.begin(), ::tolower); + + dbgTrace(D_RULEBASE_CONFIG) + << "trying to match host context with its corresponding wildcard address: " + << lower_host_ctx + << ". Matcher host: " + << lower_host; + + if (lower_host_ctx == lower_host) return true; + pos = lower_host_ctx.find_last_of(':'); + if (pos == string::npos) return false; + lower_host_ctx = string(lower_host_ctx.data(), pos); + return lower_host_ctx == lower_host; +} + +EqualListeningIP::EqualListeningIP(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams("EqualListeningIP", params.size(), 1, 1); + + auto maybe_ip = IPAddr::createIPAddr(params[0]); + if (!maybe_ip.ok()) reportWrongParamType(getName(), params[0], "Not a valid IP Address"); + + listening_ip = maybe_ip.unpack(); +} + +Maybe +EqualListeningIP::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto listening_ip_ctx = env->get(HttpTransactionData::listening_ip_ctx); + return listening_ip_ctx.ok() && listening_ip_ctx.unpack() == listening_ip; +} + +EqualListeningPort::EqualListeningPort(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams("EqualListeningPort", params.size(), 1, 1); + + try { + listening_port = boost::lexical_cast(params[0]); + } catch (boost::bad_lexical_cast const&) { + reportWrongParamType(getName(), params[0], "Not a valid port number"); + } +} + +Maybe +EqualListeningPort::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto port_ctx = env->get(HttpTransactionData::listening_port_ctx); + + return port_ctx.ok() && port_ctx.unpack() == listening_port; +} + +BeginWithUri::BeginWithUri(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams("BeginWithUri", params.size(), 1, 1); + uri_prefix = params[0]; +} + +Maybe +BeginWithUri::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto uri_ctx = env->get(HttpTransactionData::uri_ctx); + + if (!uri_ctx.ok()) + { + return false; + } + + std::string lower_uri_ctx = uri_ctx.unpack(); + std::transform(lower_uri_ctx.begin(), lower_uri_ctx.end(), lower_uri_ctx.begin(), ::tolower); + + std::string lower_uri_prefix = uri_prefix; + std::transform(lower_uri_prefix.begin(), lower_uri_prefix.end(), lower_uri_prefix.begin(), ::tolower); + + return lower_uri_ctx.find(lower_uri_prefix) == 0; +} diff --git a/components/utils/generic_rulebase/evaluators/parameter_eval.cc b/components/utils/generic_rulebase/evaluators/parameter_eval.cc new file mode 100644 index 0000000..2430ba6 --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/parameter_eval.cc @@ -0,0 +1,38 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/parameter_eval.h" + +#include +#include + +#include "generic_rulebase/rulebase_config.h" +#include "config.h" +#include "debug.h" + +using namespace std; + +string ParameterMatcher::ctx_key = "parameters"; + +ParameterMatcher::ParameterMatcher(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams(ParameterMatcher::getName(), params.size(), 1, 1); + parameter_id = params[0]; +} + +Maybe +ParameterMatcher::evalVariable() const +{ + auto rule = getConfiguration("rulebase", "rulesConfig"); + return rule.ok() && rule.unpack().isParameterActive(parameter_id); +} diff --git a/components/utils/generic_rulebase/evaluators/practice_eval.cc b/components/utils/generic_rulebase/evaluators/practice_eval.cc new file mode 100644 index 0000000..fc570e1 --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/practice_eval.cc @@ -0,0 +1,50 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/practice_eval.h" + +#include +#include + +#include "generic_rulebase/rulebase_config.h" +#include "config.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +string PracticeMatcher::ctx_key = "practices"; + +PracticeMatcher::PracticeMatcher(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams(PracticeMatcher::getName(), params.size(), 1, 1); + practice_id = params[0]; +} + +Maybe +PracticeMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto bc_practice_id_ctx = env->get>(PracticeMatcher::ctx_key); + dbgTrace(D_RULEBASE_CONFIG) + << "Trying to match practice. ID: " + << practice_id << ", Current set IDs: " + << makeSeparatedStr(bc_practice_id_ctx.ok() ? *bc_practice_id_ctx : set(), ", "); + if (bc_practice_id_ctx.ok()) { + return bc_practice_id_ctx.unpack().count(practice_id) > 0; + } + + auto rule = getConfiguration("rulebase", "rulesConfig"); + return rule.ok() && rule.unpack().isPracticeActive(practice_id); +} diff --git a/components/utils/generic_rulebase/evaluators/query_eval.cc b/components/utils/generic_rulebase/evaluators/query_eval.cc new file mode 100644 index 0000000..23f722f --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/query_eval.cc @@ -0,0 +1,136 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/query_eval.h" + +#include +#include +#include + +#include "generic_rulebase/rulebase_config.h" +#include "generic_rulebase/zones_config.h" +#include "i_environment.h" +#include "singleton.h" +#include "config.h" +#include "debug.h" +#include "enum_range.h" + +using namespace std; + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +QueryMatcher::QueryMatcher(const vector ¶ms) +{ + if (params.size() < 1) reportWrongNumberOfParams(QueryMatcher::getName(), params.size(), 1); + + key = params.front(); + if (key == "any") { + is_any = true; + } else { + values.reserve(params.size() - 1); + for (uint i = 1; i < params.size() ; i++) { + if (params[i] == "any") { + values.clear(); + break; + } + values.insert(params[i]); + } + } +} + +const string +QueryMatcher::contextKeyToString(Context::MetaDataType type) +{ + if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip"; + return Context::convertToString(type); +} + +class QueryMatchSerializer +{ +public: + static const string req_attr_ctx_key; + + template + void + serialize(Archive &ar) + { + I_Environment *env = Singleton::Consume::by(); + auto req_attr = env->get(req_attr_ctx_key); + if (!req_attr.ok()) return; + + try { + ar(cereal::make_nvp(*req_attr, value)); + dbgDebug(D_RULEBASE_CONFIG) + << "Found value for requested attribute. Tag: " + << *req_attr + << ", Value: " + << value; + } catch (exception &e) { + dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << *req_attr; + ar.finishNode(); + } + } + + template + bool + matchValues(const Values &requested_vals) const + { + return value != "" && (requested_vals.empty() || requested_vals.count(value) > 0); + } + +private: + string value; +}; + +const string QueryMatchSerializer::req_attr_ctx_key = "requested attribute key"; + +Maybe +QueryMatcher::evalVariable() const +{ + if (is_any) return true; + + I_Environment *env = Singleton::Consume::by(); + auto local_asset_ctx = env->get("is local asset"); + bool is_remote_asset = local_asset_ctx.ok() && !(*local_asset_ctx); + + QueryRequest request; + for (Context::MetaDataType name : makeRange()) { + auto val = env->get(name); + if (val.ok()) { + if ((name == Context::MetaDataType::SubjectIpAddr && is_remote_asset) || + (name == Context::MetaDataType::OtherIpAddr && !is_remote_asset)) { + continue; + } + + request.addCondition(Condition::EQUALS, contextKeyToString(name), *val); + } + } + if (request.empty()) return false; + + request.setRequestedAttr(key); + ScopedContext req_attr_key; + req_attr_key.registerValue(QueryMatchSerializer::req_attr_ctx_key, key); + + I_Intelligence_IS_V2 *intelligence = Singleton::Consume::by(); + auto query_res = intelligence->queryIntelligence(request); + if (!query_res.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr(); + return false; + } + + for (const AssetReply &asset : query_res.unpack()) { + if (asset.matchValues>(values)) return true; + } + + return false; +} diff --git a/components/utils/generic_rulebase/evaluators/trigger_eval.cc b/components/utils/generic_rulebase/evaluators/trigger_eval.cc new file mode 100644 index 0000000..cd26ac4 --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/trigger_eval.cc @@ -0,0 +1,57 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/trigger_eval.h" + +#include +#include + +#include "generic_rulebase/rulebase_config.h" +#include "config.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +string TriggerMatcher::ctx_key = "triggers"; + +TriggerMatcher::TriggerMatcher(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams(TriggerMatcher::getName(), params.size(), 1, 1); + trigger_id = params[0]; +} + +Maybe +TriggerMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto ac_bc_trigger_id_ctx = env->get>("ac_trigger_id"); + dbgTrace(D_RULEBASE_CONFIG) + << "Trying to match trigger for access control rule. ID: " + << trigger_id << ", Current set IDs: " + << makeSeparatedStr(ac_bc_trigger_id_ctx.ok() ? *ac_bc_trigger_id_ctx : set(), ", "); + if (ac_bc_trigger_id_ctx.ok()) { + return ac_bc_trigger_id_ctx.unpack().count(trigger_id) > 0; + } + + auto bc_trigger_id_ctx = env->get>(TriggerMatcher::ctx_key); + dbgTrace(D_RULEBASE_CONFIG) + << "Trying to match trigger. ID: " + << trigger_id << ", Current set IDs: " + << makeSeparatedStr(bc_trigger_id_ctx.ok() ? *bc_trigger_id_ctx : set(), ", "); + if (bc_trigger_id_ctx.ok() && bc_trigger_id_ctx.unpack().count(trigger_id) > 0 ) return true; + + auto rule = getConfiguration("rulebase", "rulesConfig"); + return rule.ok() && rule.unpack().isTriggerActive(trigger_id); +} diff --git a/components/utils/generic_rulebase/evaluators/zone_eval.cc b/components/utils/generic_rulebase/evaluators/zone_eval.cc new file mode 100644 index 0000000..17eb328 --- /dev/null +++ b/components/utils/generic_rulebase/evaluators/zone_eval.cc @@ -0,0 +1,44 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/evaluators/zone_eval.h" + +#include +#include + +#include "generic_rulebase/zone.h" +#include "generic_rulebase/rulebase_config.h" +#include "config.h" + +using namespace std; + +string ZoneMatcher::ctx_key = "zone_id"; + +ZoneMatcher::ZoneMatcher(const vector ¶ms) +{ + if (params.size() != 1) reportWrongNumberOfParams(ZoneMatcher::getName(), params.size(), 1, 1); + zone_id = params[0]; +} + +Maybe +ZoneMatcher::evalVariable() const +{ + I_Environment *env = Singleton::Consume::by(); + auto bc_zone_id_ctx = env->get(ZoneMatcher::ctx_key); + if (bc_zone_id_ctx.ok() && *bc_zone_id_ctx == zone_id) return true; + + if (!getProfileAgentSettingWithDefault(false, "rulebase.enableQueryBasedMatch")) return false; + + auto zone = getConfiguration("rulebase", "zones"); + return zone.ok() && zone.unpack().getId() == zone_id; +} diff --git a/components/utils/generic_rulebase/generic_rulebase.cc b/components/utils/generic_rulebase/generic_rulebase.cc new file mode 100644 index 0000000..c7f98de --- /dev/null +++ b/components/utils/generic_rulebase/generic_rulebase.cc @@ -0,0 +1,149 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/generic_rulebase.h" + +#include + +#include "generic_rulebase/evaluators/trigger_eval.h" +#include "generic_rulebase/evaluators/practice_eval.h" +#include "generic_rulebase/evaluators/parameter_eval.h" +#include "generic_rulebase/evaluators/zone_eval.h" +#include "generic_rulebase/evaluators/asset_eval.h" +#include "generic_rulebase/evaluators/query_eval.h" +#include "generic_rulebase/evaluators/connection_eval.h" +#include "generic_rulebase/evaluators/http_transaction_data_eval.h" +#include "generic_rulebase/zone.h" +#include "generic_rulebase/triggers_config.h" +#include "generic_rulebase/parameters_config.h" +#include "singleton.h" +#include "common.h" +#include "debug.h" +#include "cache.h" +#include "config.h" +#include "i_environment.h" + + +using namespace std; + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +class GenericRulebase::Impl : Singleton::Provide::From +{ +public: + void init() {} + void fini() {} + + void preload(); + + Maybe getLocalZone() const override { return getZoneConfig(true); } + Maybe getOtherZone() const override { return getZoneConfig(false); } + + LogTriggerConf getLogTriggerConf(const string &trigger_Id) const override; + ParameterException getParameterException(const string ¶meter_Id) const override; + set getBehavior(const ParameterKeyValues &key_value_pairs) const override; + +private: + Maybe + getZoneConfig(bool is_local_zone) const + { + ScopedContext asset_location_ctx; + asset_location_ctx.registerValue("is local asset", is_local_zone); + return getConfiguration("rulebase", "zones"); + } +}; + +void +GenericRulebase::Impl::preload() +{ + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + addMatcher(); + BasicRuleConfig::preload(); + LogTriggerConf::preload(); + ParameterException::preload(); + registerExpectedConfiguration("rulebase", "zones"); + registerExpectedConfigFile("zones", Config::ConfigFileType::Policy); + registerExpectedConfigFile("triggers", Config::ConfigFileType::Policy); + registerExpectedConfigFile("rules", Config::ConfigFileType::Policy); + registerExpectedConfigFile("parameters", Config::ConfigFileType::Policy); + registerExpectedConfigFile("exceptions", Config::ConfigFileType::Policy); + +} + +LogTriggerConf +GenericRulebase::Impl::getLogTriggerConf(const string& trigger_Id) const +{ + ScopedContext ctx; + set triggers = {trigger_Id}; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers); + return getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log"); +} + +ParameterException +GenericRulebase::Impl::getParameterException(const string& parameter_Id) const +{ + ScopedContext ctx; + set exceptions = {parameter_Id}; + ctx.registerValue>(ParameterMatcher::ctx_key, exceptions); + return getConfigurationWithDefault(ParameterException(), "rulebase", "exception"); +} + +set +GenericRulebase::Impl::getBehavior(const ParameterKeyValues &key_value_pairs) const +{ + auto &exceptions = getConfiguration("rulebase", "exception"); + + if (!exceptions.ok()) { + dbgTrace(D_RULEBASE_CONFIG) << "Could not find any exception with the current rule's context"; + return {}; + } + return (*exceptions).getBehavior(key_value_pairs); +} + +GenericRulebase::GenericRulebase() : Component("GenericRulebase"), pimpl(make_unique()) {} + +GenericRulebase::~GenericRulebase() {} + +void +GenericRulebase::init() +{ + pimpl->init(); +} + +void +GenericRulebase::fini() +{ + pimpl->fini(); +} + +void +GenericRulebase::preload() +{ + pimpl->preload(); +} diff --git a/components/utils/generic_rulebase/generic_rulebase_context.cc b/components/utils/generic_rulebase/generic_rulebase_context.cc new file mode 100644 index 0000000..7dd98cb --- /dev/null +++ b/components/utils/generic_rulebase/generic_rulebase_context.cc @@ -0,0 +1,109 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/generic_rulebase_context.h" + +#include + +#include "context.h" +#include "config.h" +#include "generic_rulebase/evaluators/trigger_eval.h" +#include "generic_rulebase/evaluators/parameter_eval.h" +#include "generic_rulebase/evaluators/practice_eval.h" +#include "generic_rulebase/evaluators/zone_eval.h" +#include "generic_rulebase/evaluators/asset_eval.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +template +set +extractIds(const vector &configurations) +{ + set ids; + for (const Configs &conf : configurations) { + ids.insert(conf.getId()); + } + return ids; +} + +void +GenericRulebaseContext::activate(const BasicRuleConfig &rule) +{ + switch(registration_state) { + case RuleRegistrationState::UNINITIALIZED: { + registration_state = RuleRegistrationState::REGISTERED; + ctx.registerValue>( + TriggerMatcher::ctx_key, + extractIds(rule.getTriggers()) + ); + ctx.registerValue>( + PracticeMatcher::ctx_key, + extractIds(rule.getPractices()) + ); + dbgTrace(D_RULEBASE_CONFIG) + << "Activating current practices. Current practice IDs: " + << makeSeparatedStr(extractIds(rule.getPractices()), ", "); + + ctx.registerValue>( + ParameterMatcher::ctx_key, + extractIds(rule.getParameters()) + ); + ctx.registerValue( + ZoneMatcher::ctx_key, + rule.getZoneId() + ); + ctx.registerValue( + AssetMatcher::ctx_key, + rule.getAssetId() + ); + ctx.activate(); + break; + } + case RuleRegistrationState::REGISTERED: { + dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values"; + ctx.activate(); + break; + } + case RuleRegistrationState::UNREGISTERED: { + dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values"; + } + } +} + +void +GenericRulebaseContext::activate() +{ + switch(registration_state) { + case RuleRegistrationState::UNINITIALIZED: { + auto maybe_rule = getConfiguration("rulebase", "rulesConfig"); + if (!maybe_rule.ok()) { + registration_state = RuleRegistrationState::UNREGISTERED; + return; + } + dbgTrace(D_RULEBASE_CONFIG) << "Registering new rule values"; + activate(maybe_rule.unpack()); + registration_state = RuleRegistrationState::REGISTERED; + break; + } + case RuleRegistrationState::REGISTERED: { + dbgTrace(D_RULEBASE_CONFIG) << "Activating registered rule values"; + ctx.activate(); + break; + } + case RuleRegistrationState::UNREGISTERED: { + dbgTrace(D_RULEBASE_CONFIG) << "Failed to register rule values"; + } + } +} diff --git a/components/utils/generic_rulebase/match_query.cc b/components/utils/generic_rulebase/match_query.cc new file mode 100644 index 0000000..ef985dc --- /dev/null +++ b/components/utils/generic_rulebase/match_query.cc @@ -0,0 +1,347 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/match_query.h" + +#include "cereal/types/set.hpp" + +#include "generic_rulebase/generic_rulebase_utils.h" +#include "config.h" +#include "ip_utilities.h" +#include "agent_core_utilities.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +static const unordered_map string_to_match_type = { + { "condition", MatchQuery::MatchType::Condition }, + { "operator", MatchQuery::MatchType::Operator } +}; + +static const unordered_map string_to_operator = { + { "and", MatchQuery::Operators::And }, + { "or", MatchQuery::Operators::Or } +}; + +static const unordered_map string_to_condition = { + { "equals", MatchQuery::Conditions::Equals }, + { "not-equals", MatchQuery::Conditions::NotEquals }, + { "not equals", MatchQuery::Conditions::NotEquals }, + { "in", MatchQuery::Conditions::In }, + { "not-in", MatchQuery::Conditions::NotIn }, + { "not in", MatchQuery::Conditions::NotIn }, + { "exist", MatchQuery::Conditions::Exist } +}; + +static const string ip_addr_type_name = "IP address"; +static const string port_type_name = "port"; +static const string ip_proto_type_name = "IP protocol"; + +static const unordered_map string_to_key = { + { "sourceIP", MatchQuery::StaticKeys::SrcIpAddress }, + { "sourceIpAddr", MatchQuery::StaticKeys::SrcIpAddress }, + { "destinationIP", MatchQuery::StaticKeys::DstIpAddress }, + { "destinationIpAddr", MatchQuery::StaticKeys::DstIpAddress }, + { "ipAddress", MatchQuery::StaticKeys::IpAddress }, + { "sourcePort", MatchQuery::StaticKeys::SrcPort }, + { "listeningPort", MatchQuery::StaticKeys::ListeningPort }, + { "ipProtocol", MatchQuery::StaticKeys::IpProtocol }, + { "domain", MatchQuery::StaticKeys::Domain } +}; + +MatchQuery::MatchQuery(const string &match) : is_specific_label(false), is_ignore_keyword(false) +{ + try { + stringstream ss; + ss.str(match); + cereal::JSONInputArchive archive_in(ss); + load(archive_in); + } catch (const exception &e) { + dbgWarning(D_RULEBASE_CONFIG) + << "Unable to load match query JSON. JSON content: " + << match + << ", Error: " + << e.what(); + } +} + +void +MatchQuery::load(cereal::JSONInputArchive &archive_in) +{ + string type_as_string; + archive_in(cereal::make_nvp("type", type_as_string)); + + string op_as_string; + archive_in(cereal::make_nvp("op", op_as_string)); + + auto maybe_type = string_to_match_type.find(type_as_string); + if (maybe_type == string_to_match_type.end()) { + reportConfigurationError("Illegal Zone match query type. Provided type in configuration: " + type_as_string); + } + + type = maybe_type->second; + switch (type) { + case (MatchType::Condition): { + auto maybe_condition = string_to_condition.find(op_as_string); + if (maybe_condition == string_to_condition.end()) { + reportConfigurationError( + "Illegal op provided for condition. Provided op in configuration: " + + op_as_string + ); + } + condition_type = maybe_condition->second; + operator_type = Operators::None; + archive_in(cereal::make_nvp("key", key)); + key_type = getKeyByName(key); + if (key_type == StaticKeys::NotStatic) { + if (key.rfind("containerLabels.", 0) == 0) { + is_specific_label = true; + } else { + is_specific_label = false; + } + } + is_ignore_keyword = (key == "indicator"); + + if (condition_type != Conditions::Exist) { + archive_in(cereal::make_nvp("value", value)); + for(const auto &val: value) { + if (isKeyTypeIp()) { + auto ip_range = IPUtilities::createRangeFromString(val, ip_addr_type_name); + if (ip_range.ok()) { + ip_addr_value.push_back(ip_range.unpack()); + } else { + dbgWarning(D_RULEBASE_CONFIG) + << "Failed to parse IP address range. Error: " + << ip_range.getErr(); + } + } else if (isKeyTypePort()) { + auto port_range = IPUtilities::createRangeFromString( + val, + port_type_name + ); + if (port_range.ok()) { + port_value.push_back(port_range.unpack()); + } else { + dbgWarning(D_RULEBASE_CONFIG) + << "Failed to parse port range. Error: " + << port_range.getErr(); + } + } else if (isKeyTypeProtocol()) { + auto proto_range = IPUtilities::createRangeFromString( + val, + ip_proto_type_name + ); + if (proto_range.ok()) { + ip_proto_value.push_back(proto_range.unpack()); + } else { + dbgWarning(D_RULEBASE_CONFIG) + << "Failed to parse IP protocol range. Error: " + << proto_range.getErr(); + } + } + + try { + regex_values.insert(boost::regex(val)); + } catch (const exception &e) { + dbgDebug(D_RULEBASE_CONFIG) << "Failed to compile regex. Error: " << e.what(); + } + } + first_value = *(value.begin()); + } + break; + } + case (MatchType::Operator): { + auto maybe_operator = string_to_operator.find(op_as_string); + if (maybe_operator == string_to_operator.end()) { + reportConfigurationError( + "Illegal op provided for operator. Provided op in configuration: " + + op_as_string + ); + } + operator_type = maybe_operator->second; + condition_type = Conditions::None; + archive_in(cereal::make_nvp("items", items)); + break; + } + } +} + +MatchQuery::StaticKeys +MatchQuery::getKeyByName(const string &key_type_name) +{ + auto key = string_to_key.find(key_type_name); + if (key == string_to_key.end()) return StaticKeys::NotStatic; + return key->second; +} + +bool +MatchQuery::isKeyTypeIp() const +{ + return (key_type >= StaticKeys::IpAddress && key_type <= StaticKeys::DstIpAddress); +} + +bool +MatchQuery::isKeyTypePort() const +{ + return (key_type == StaticKeys::SrcPort || key_type == StaticKeys::ListeningPort); +} + +bool +MatchQuery::isKeyTypeProtocol() const +{ + return (key_type == StaticKeys::IpProtocol); +} + +bool +MatchQuery::isKeyTypeDomain() const +{ + return (key_type == StaticKeys::Domain); +} + +bool +MatchQuery::isKeyTypeSpecificLabel() const +{ + return is_specific_label; +} + +bool +MatchQuery::isKeyTypeStatic() const +{ + return (key_type != StaticKeys::NotStatic); +} + +set +MatchQuery::getAllKeys() const +{ + set keys; + if (type == MatchType::Condition) { + if (!key.empty()) keys.insert(key); + return keys; + } + + for (const MatchQuery &inner_match: items) { + set iner_keys = inner_match.getAllKeys(); + keys.insert(iner_keys.begin(), iner_keys.end()); + } + + return keys; +} + +bool +MatchQuery::matchAttributes( + const unordered_map> &key_value_pairs, + set &matched_override_keywords ) const +{ + + if (type == MatchType::Condition) { + auto key_value_pair = key_value_pairs.find(key); + if (key_value_pair == key_value_pairs.end()) { + dbgTrace(D_RULEBASE_CONFIG) << "Ignoring irrelevant key: " << key; + return false; + } + return matchAttributes(key_value_pair->second, matched_override_keywords); + } else if (type == MatchType::Operator && operator_type == Operators::And) { + for (const MatchQuery &inner_match: items) { + if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) { + return false; + } + } + return true; + } else if (type == MatchType::Operator && operator_type == Operators::Or) { + // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched + set inner_override_keywords; + bool res = false; + for (const MatchQuery &inner_match: items) { + inner_override_keywords.clear(); + if (inner_match.matchAttributes(key_value_pairs, inner_override_keywords)) { + matched_override_keywords.insert(inner_override_keywords.begin(), inner_override_keywords.end()); + res = true; + } + } + return res; + } else { + dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type"; + } + return false; +} + +MatchQuery::MatchResult +MatchQuery::getMatch( const unordered_map> &key_value_pairs) const +{ + MatchQuery::MatchResult matches; + matches.matched_keywords = make_shared>(); + matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords); + return matches; +} + +bool +MatchQuery::matchAttributes( + const unordered_map> &key_value_pairs) const +{ + return getMatch(key_value_pairs).is_match; +} + +bool +MatchQuery::matchAttributes( + const set &values, + set &matched_override_keywords) const +{ + auto &type = condition_type; + bool negate = type == MatchQuery::Conditions::NotEquals || type == MatchQuery::Conditions::NotIn; + bool match = isRegEx() ? matchAttributesRegEx(values, matched_override_keywords) : matchAttributesString(values); + return negate ? !match : match; +} + +bool +MatchQuery::matchAttributesRegEx( + const set &values, + set &matched_override_keywords) const +{ + bool res = false; + boost::cmatch value_matcher; + for (const boost::regex &val_regex : regex_values) { + for (const string &requested_match_value : values) { + if (NGEN::Regex::regexMatch( + __FILE__, + __LINE__, + requested_match_value.c_str(), + value_matcher, + val_regex)) + { + res = true; + if (is_ignore_keyword) { + matched_override_keywords.insert(requested_match_value); + } else { + return res; + } + } + } + } + return res; +} + +bool +MatchQuery::matchAttributesString(const set &values) const +{ + for (const string &requested_value : values) { + if (value.find(requested_value) != value.end()) return true; + } + return false; +} + +bool +MatchQuery::isRegEx() const +{ + return key != "protectionName"; +} diff --git a/components/utils/generic_rulebase/parameters_config.cc b/components/utils/generic_rulebase/parameters_config.cc new file mode 100644 index 0000000..7677b86 --- /dev/null +++ b/components/utils/generic_rulebase/parameters_config.cc @@ -0,0 +1,157 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/parameters_config.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +bool ParameterException::is_geo_location_exception_exists(false); +bool ParameterException::is_geo_location_exception_being_loaded(false); + +void +ParameterOverrides::load(cereal::JSONInputArchive &archive_in) +{ + parseJSONKey>("parsedBehavior", parsed_behaviors, archive_in); +} + +void +ParameterTrustedSources::load(cereal::JSONInputArchive &archive_in) +{ + parseJSONKey("numOfSources", num_of_sources, archive_in); + parseJSONKey>("sourcesIdentifiers", sources_identidiers, archive_in); +} + +void +ParameterBehavior::load(cereal::JSONInputArchive &archive_in) +{ + string key_string; + string val_string; + parseJSONKey("id", id, archive_in); + parseJSONKey("key", key_string, archive_in); + parseJSONKey("value", val_string, archive_in); + if (string_to_behavior_key.find(key_string) == string_to_behavior_key.end()) { + dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior key: " << key_string; + return; + } + key = string_to_behavior_key.at(key_string); + + if (string_to_behavior_val.find(val_string) == string_to_behavior_val.end()) { + dbgWarning(D_RULEBASE_CONFIG) << "Unsupported behavior value: " << val_string; + return; + } + value = string_to_behavior_val.at(val_string); +} + +void +ParameterAntiBot::load(cereal::JSONInputArchive &archive_in) +{ + parseJSONKey>("injected", injected, archive_in); + parseJSONKey>("validated", validated, archive_in); +} + +void +ParameterOAS::load(cereal::JSONInputArchive &archive_in) +{ + parseJSONKey("value", value, archive_in); +} + +void +ParameterException::MatchBehaviorPair::load(cereal::JSONInputArchive &archive_in) +{ + parseJSONKey("match", match, archive_in); + parseJSONKey("behavior", behavior, archive_in); +} + +void +ParameterException::load(cereal::JSONInputArchive &archive_in) +{ + try { + archive_in( + cereal::make_nvp("match", match), + cereal::make_nvp("behavior", behavior) + ); + } catch (...) { + parseJSONKey>("exceptions", match_queries, archive_in); + } + + function isGeoLocationExists = + [&](const MatchQuery &query) + { + if (query.getKey() == "countryCode" || query.getKey() == "countryName") { + is_geo_location_exception_being_loaded = true; + return true; + } + + for (const MatchQuery &query_item : query.getItems()) { + if (isGeoLocationExists(query_item)) return true; + } + + return false; + }; + + if (isGeoLocationExists(match)) return; + for (const MatchBehaviorPair &match_query : match_queries) { + if (isGeoLocationExists(match_query.match)) return; + } +} + +set +ParameterException::getBehavior( + const unordered_map> &key_value_pairs, + set &matched_override_keywords) const +{ + set matched_behaviors; + + matched_override_keywords.clear(); + dbgTrace(D_RULEBASE_CONFIG) << "Matching exception"; + for (const MatchBehaviorPair &match_behavior_pair: match_queries) { + MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs); + if (match_res.is_match) { + dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches."; + // When matching indicators with action=ignore, we expect no behavior override. + // Instead, a matched keywords list should be returned which will be later removed from score calculation + if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) { + matched_override_keywords.insert(match_res.matched_keywords->begin(), + match_res.matched_keywords->end()); + } else { + matched_behaviors.insert(match_behavior_pair.behavior); + } + } + } + + if (match_queries.empty()) { + MatchQuery::MatchResult match_res = match.getMatch(key_value_pairs); + if (match_res.is_match) { + dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception."; + // When matching indicators with action=ignore, we expect no behavior override. + // Instead, a matched keywords list should be returned which will be later removed from score calculation + if (match_res.matched_keywords->size() > 0 && behavior == action_ignore) { + matched_override_keywords.insert(match_res.matched_keywords->begin(), + match_res.matched_keywords->end()); + } else { + matched_behaviors.insert(behavior); + } + } + } + + return matched_behaviors; +} + +set +ParameterException::getBehavior(const unordered_map> &key_value_pairs) const +{ + set keywords; + return getBehavior(key_value_pairs, keywords); +} diff --git a/components/utils/generic_rulebase/rulebase_config.cc b/components/utils/generic_rulebase/rulebase_config.cc new file mode 100644 index 0000000..2dd018f --- /dev/null +++ b/components/utils/generic_rulebase/rulebase_config.cc @@ -0,0 +1,79 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/rulebase_config.h" + +#include "telemetry.h" +#include "config.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +set BasicRuleConfig::assets_ids{}; +set BasicRuleConfig::assets_ids_aggregation{}; + +void +BasicRuleConfig::load(cereal::JSONInputArchive &ar) +{ + parseJSONKey>("practices", practices, ar); + parseJSONKey>("triggers", triggers, ar); + parseJSONKey>("parameters", parameters, ar); + parseJSONKey("priority", priority, ar); + parseJSONKey("ruleId", rule_id, ar); + parseJSONKey("ruleName", rule_name, ar); + parseJSONKey("assetId", asset_id, ar); + parseJSONKey("assetName", asset_name, ar); + parseJSONKey("zoneId", zone_id, ar); + parseJSONKey("zoneName", zone_name, ar); + + assets_ids_aggregation.insert(asset_id); +} + +void +BasicRuleConfig::updateCountMetric() +{ + BasicRuleConfig::assets_ids = BasicRuleConfig::assets_ids_aggregation; + AssetCountEvent(AssetType::ALL, BasicRuleConfig::assets_ids.size()).notify(); +} + +bool +BasicRuleConfig::isPracticeActive(const string &practice_id) const +{ + for (auto practice: practices) { + if (practice.getId() == practice_id) return true; + } + return false; +} + +bool +BasicRuleConfig::isTriggerActive(const string &trigger_id) const +{ + for (auto trigger: triggers) { + if (trigger.getId() == trigger_id) { + return true; + } + } + return false; +} + +bool +BasicRuleConfig::isParameterActive(const string ¶meter_id) const +{ + for (auto param: parameters) { + if (param.getId() == parameter_id) { + return true; + } + } + return false; +} diff --git a/components/utils/generic_rulebase/triggers_config.cc b/components/utils/generic_rulebase/triggers_config.cc new file mode 100644 index 0000000..01429ec --- /dev/null +++ b/components/utils/generic_rulebase/triggers_config.cc @@ -0,0 +1,243 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "generic_rulebase/triggers_config.h" +#include "generic_rulebase/generic_rulebase_utils.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +WebTriggerConf::WebTriggerConf() : response_title(""), response_body(""), response_code(0) {} +WebTriggerConf::WebTriggerConf(const string &title, const string &body, uint code) + : + response_title(title), + response_body(body), + response_code(code) +{} + +WebTriggerConf WebTriggerConf::default_trigger_conf = WebTriggerConf( + "Attack blocked by web application protection", // title + "Check Point's Application Security has detected an attack and blocked it.", // body + 403 +); + +void +WebTriggerConf::load(cereal::JSONInputArchive &archive_in) +{ + try { + parseJSONKey("details level", details_level, archive_in); + if (details_level == "Redirect") { + parseJSONKey("redirect URL", redirect_url, archive_in); + parseJSONKey("xEventId", add_event_id_to_header, archive_in); + parseJSONKey("eventIdInHeader", add_event_id_to_header, archive_in); + return; + } + parseJSONKey("response code", response_code, archive_in); + if (response_code < 100 || response_code > 599) { + throw cereal::Exception( + "illegal web trigger response code: " + + to_string(response_code) + + " is out of range (100-599)" + ); + } + + if (details_level == "Response Code") return; + + parseJSONKey("response body", response_body, archive_in); + parseJSONKey("response title", response_title, archive_in); + } catch (const exception &e) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the web trigger configuration: '" << e.what() << "'"; + archive_in.setNextName(nullptr); + } +} + +bool +WebTriggerConf::operator==(const WebTriggerConf &other) const +{ + return + response_code == other.response_code && + response_title == other.response_title && + response_body == other.response_body; +} + +LogTriggerConf::LogTriggerConf(string trigger_name, bool log_detect, bool log_prevent) : name(trigger_name) +{ + if (log_detect) should_log_on_detect.setAll(); + if (log_prevent) should_log_on_prevent.setAll(); + active_streams.setFlag(ReportIS::StreamType::JSON_FOG); + active_streams.setFlag(ReportIS::StreamType::JSON_LOG_FILE); +} + +ReportIS::Severity +LogTriggerConf::getSeverity(bool is_action_drop_or_prevent) const +{ + return is_action_drop_or_prevent ? ReportIS::Severity::MEDIUM : ReportIS::Severity::LOW; +} + +ReportIS::Priority +LogTriggerConf::getPriority(bool is_action_drop_or_prevent) const +{ + return is_action_drop_or_prevent ? ReportIS::Priority::HIGH : ReportIS::Priority::MEDIUM; +} + +Flags +LogTriggerConf::getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const +{ + if (is_action_drop_or_prevent && should_log_on_prevent.isSet(security_type)) return active_streams; + if (!is_action_drop_or_prevent && should_log_on_detect.isSet(security_type)) return active_streams; + + return Flags(); +} + +Flags +LogTriggerConf::getEnrechments(SecurityType security_type) const +{ + Flags enreachments; + + if (log_geo_location.isSet(security_type)) enreachments.setFlag(ReportIS::Enreachments::GEOLOCATION); + if (should_format_output) enreachments.setFlag(ReportIS::Enreachments::BEAUTIFY_OUTPUT); + + return enreachments; +} + +template +static void +setTriggersFlag(const string &key, cereal::JSONInputArchive &ar, EnumClass flag, Flags &flags) +{ + bool value = false; + parseJSONKey(key, value, ar); + if (value) flags.setFlag(flag); +} + +static void +setLogConfiguration( + const ReportIS::StreamType &log_type, + const string &log_server_url = "", + const string &protocol = "" +) +{ + dbgTrace(D_RULEBASE_CONFIG) << "log server url:" << log_server_url; + if (log_server_url != "" && protocol != "") { + Singleton::Consume::by()->addStream(log_type, log_server_url, protocol); + } else { + Singleton::Consume::by()->addStream(log_type); + } +} + +static string +parseProtocolWithDefault( + const std::string &default_value, + const std::string &key_name, + cereal::JSONInputArchive &archive_in +) +{ + string value; + try { + archive_in(cereal::make_nvp(key_name, value)); + } catch (const cereal::Exception &e) { + return default_value; + } + return value; +} + +void +LogTriggerConf::load(cereal::JSONInputArchive& archive_in) +{ + try { + parseJSONKey("triggerName", name, archive_in); + parseJSONKey("verbosity", verbosity, archive_in); + parseJSONKey("urlForSyslog", url_for_syslog, archive_in); + parseJSONKey("urlForCef", url_for_cef, archive_in); + parseJSONKey("syslogProtocol", syslog_protocol, archive_in); + syslog_protocol = parseProtocolWithDefault("UDP", "syslogProtocol", archive_in); + cef_protocol = parseProtocolWithDefault("UDP", "cefProtocol", archive_in); + + setTriggersFlag("webBody", archive_in, WebLogFields::webBody, log_web_fields); + setTriggersFlag("webHeaders", archive_in, WebLogFields::webHeaders, log_web_fields); + setTriggersFlag("webRequests", archive_in, WebLogFields::webRequests, log_web_fields); + setTriggersFlag("webUrlPath", archive_in, WebLogFields::webUrlPath, log_web_fields); + setTriggersFlag("webUrlQuery", archive_in, WebLogFields::webUrlQuery, log_web_fields); + setTriggersFlag("logToAgent", archive_in, ReportIS::StreamType::JSON_LOG_FILE, active_streams); + setTriggersFlag("logToCloud", archive_in, ReportIS::StreamType::JSON_FOG, active_streams); + setTriggersFlag("logToK8sService", archive_in, ReportIS::StreamType::JSON_K8S_SVC, active_streams); + setTriggersFlag("logToSyslog", archive_in, ReportIS::StreamType::SYSLOG, active_streams); + setTriggersFlag("logToCef", archive_in, ReportIS::StreamType::CEF, active_streams); + setTriggersFlag("acAllow", archive_in, SecurityType::AccessControl, should_log_on_detect); + setTriggersFlag("acDrop", archive_in, SecurityType::AccessControl, should_log_on_prevent); + setTriggersFlag("tpDetect", archive_in, SecurityType::ThreatPrevention, should_log_on_detect); + setTriggersFlag("tpPrevent", archive_in, SecurityType::ThreatPrevention, should_log_on_prevent); + setTriggersFlag("complianceWarnings", archive_in, SecurityType::Compliance, should_log_on_detect); + setTriggersFlag("complianceViolations", archive_in, SecurityType::Compliance, should_log_on_prevent); + setTriggersFlag("acLogGeoLocation", archive_in, SecurityType::AccessControl, log_geo_location); + setTriggersFlag("tpLogGeoLocation", archive_in, SecurityType::ThreatPrevention, log_geo_location); + setTriggersFlag("complianceLogGeoLocation", archive_in, SecurityType::Compliance, log_geo_location); + + bool extend_logging = false; + parseJSONKey("extendLogging", extend_logging, archive_in); + if (extend_logging) { + setTriggersFlag("responseCode", archive_in, WebLogFields::responseCode, log_web_fields); + setTriggersFlag("responseBody", archive_in, WebLogFields::responseBody, log_web_fields); + + string severity; + static const map extend_logging_severity_strings = { + {"High", extendLoggingSeverity::High}, + {"Critical", extendLoggingSeverity::Critical} + }; + parseJSONKey("extendLoggingMinSeverity", severity, archive_in); + auto extended_severity = extend_logging_severity_strings.find(severity); + if (extended_severity != extend_logging_severity_strings.end()) { + extend_logging_severity = extended_severity->second; + } else { + dbgWarning(D_RULEBASE_CONFIG) + << "Failed to parse the extendLoggingMinSeverityfield: '" + << severity + << "'"; + } + } + + for (ReportIS::StreamType log_stream : makeRange()) { + if (!active_streams.isSet(log_stream)) continue; + switch (log_stream) { + case ReportIS::StreamType::JSON_DEBUG: + setLogConfiguration(ReportIS::StreamType::JSON_DEBUG); + break; + case ReportIS::StreamType::JSON_FOG: + setLogConfiguration(ReportIS::StreamType::JSON_FOG); + break; + case ReportIS::StreamType::JSON_LOG_FILE: + setLogConfiguration(ReportIS::StreamType::JSON_LOG_FILE); + break; + case ReportIS::StreamType::JSON_K8S_SVC: + setLogConfiguration(ReportIS::StreamType::JSON_K8S_SVC); + break; + case ReportIS::StreamType::SYSLOG: + setLogConfiguration(ReportIS::StreamType::SYSLOG, getUrlForSyslog(), syslog_protocol); + break; + case ReportIS::StreamType::CEF: + setLogConfiguration(ReportIS::StreamType::CEF, getUrlForCef(), cef_protocol); + break; + case ReportIS::StreamType::NONE: break; + case ReportIS::StreamType::COUNT: break; + } + } + + parseJSONKey("formatLoggingOutput", should_format_output, archive_in); + } catch (const exception &e) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to parse the log trigger configuration: '" << e.what() << "'"; + archive_in.setNextName(nullptr); + } +} diff --git a/components/utils/generic_rulebase/zone.cc b/components/utils/generic_rulebase/zone.cc new file mode 100644 index 0000000..c9f902c --- /dev/null +++ b/components/utils/generic_rulebase/zone.cc @@ -0,0 +1,179 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/zone.h" + +#include +#include +#include + +using namespace std; + +static const unordered_map string_to_direction = { + { "to", Zone::Direction::To }, + { "from", Zone::Direction::From }, + { "bidirectional", Zone::Direction::Bidirectional } +}; + +class AdjacentZone +{ +public: + void + load(cereal::JSONInputArchive &archive_in) + { + string direction_as_string; + archive_in(cereal::make_nvp("direction", direction_as_string)); + archive_in(cereal::make_nvp("zoneId", id)); + auto maybe_direction = string_to_direction.find(direction_as_string); + if (maybe_direction == string_to_direction.end()) { + reportConfigurationError( + "Illegal direction provided for adjacency. Provided direction in configuration: " + + direction_as_string + ); + } + dir = maybe_direction->second; + } + + pair getValue() const { return make_pair(dir, id); } + +private: + Zone::Direction dir; + GenericConfigId id; +}; + +class TagsValues +{ +public: + static const string req_attrs_ctx_key; + + TagsValues() {} + + template + void + serialize(Archive &ar) + { + I_Environment *env = Singleton::Consume::by(); + auto req_attrs = env->get>(req_attrs_ctx_key); + if (!req_attrs.ok()) return; + + for (const string &req_attr : *req_attrs) { + try { + string data; + ar(cereal::make_nvp(req_attr, data)); + dbgDebug(D_RULEBASE_CONFIG) + << "Found value for requested attribute. Tag: " + << req_attr + << ", Value: " + << data; + + tags_set[req_attr].insert(data); + } catch (const exception &e) { + dbgDebug(D_RULEBASE_CONFIG) << "Could not find values for requested attribute. Tag: " << req_attr; + ar.setNextName(nullptr); + } + } + } + + bool + matchValueByKey(const string &requested_key, const unordered_set &possible_values) const + { + auto values = tags_set.find(requested_key); + if (values == tags_set.end()) return false; + + for (const string &val : possible_values) { + if (values->second.count(val)) return true; + } + return false; + } + + void + insert(const TagsValues &other) + { + for (auto &single_tags_value : other.getData()) { + tags_set[single_tags_value.first].insert(single_tags_value.second.begin(), single_tags_value.second.end()); + } + } + + const unordered_map> & getData() const { return tags_set; } + +private: + unordered_map> tags_set; +}; + +const string TagsValues::req_attrs_ctx_key = "requested attributes key"; + +void +Zone::load(cereal::JSONInputArchive &archive_in) +{ + archive_in(cereal::make_nvp("id", zone_id)); + archive_in(cereal::make_nvp("name", zone_name)); + vector adjacency; + try { + archive_in(cereal::make_nvp("adjacentZones", adjacency)); + } catch (const cereal::Exception &) { + dbgTrace(D_RULEBASE_CONFIG) + << "List of adjacentZones does not exist for current object. Zone id: " + << zone_id + << ", Zone name: " + << zone_name; + + archive_in.setNextName(nullptr); + } + + for (const AdjacentZone &zone : adjacency) { + adjacent_zones.push_back(zone.getValue()); + } + + archive_in(cereal::make_nvp("match", match_query)); + + is_any = + match_query.getType() == MatchQuery::MatchType::Condition && + match_query.getKey() == "any" && + match_query.getValue().count("any") > 0; + + set keys = match_query.getAllKeys(); +} + +const string +contextKeyToString(Context::MetaDataType type) +{ + if (type == Context::MetaDataType::SubjectIpAddr || type == Context::MetaDataType::OtherIpAddr) return "ip"; + return Context::convertToString(type); +} + +bool +Zone::contains(const Asset &asset) +{ + QueryRequest request; + + for (const auto &main_attr : asset.getAttrs()) { + request.addCondition(Condition::EQUALS, contextKeyToString(main_attr.first), main_attr.second); + } + + ScopedContext req_attrs_key; + req_attrs_key.registerValue>(TagsValues::req_attrs_ctx_key, match_query.getAllKeys()); + + I_Intelligence_IS_V2 *intelligence = Singleton::Consume::by(); + auto query_res = intelligence->queryIntelligence(request); + if (!query_res.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to perform intelligence query. Error: " << query_res.getErr(); + return false; + } + + for (const AssetReply &asset : query_res.unpack()) { + TagsValues tag_values = asset.mergeReplyData(); + + if (match_query.matchAttributes(tag_values.getData())) return true; + } + return false; +} diff --git a/components/utils/generic_rulebase/zones_config.cc b/components/utils/generic_rulebase/zones_config.cc new file mode 100644 index 0000000..97bd88e --- /dev/null +++ b/components/utils/generic_rulebase/zones_config.cc @@ -0,0 +1,114 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "generic_rulebase/zones_config.h" + +#include +#include + +#include "generic_rulebase/generic_rulebase_utils.h" +#include "config.h" +#include "ip_utilities.h" +#include "connkey.h" +#include "i_generic_rulebase.h" + +USE_DEBUG_FLAG(D_RULEBASE_CONFIG); + +using namespace std; + +void +ZonesConfig::load(cereal::JSONInputArchive &archive_in) +{ + dbgFlow(D_RULEBASE_CONFIG) << "Saving active zones"; + set used_zones; + cereal::load(archive_in, used_zones); + + dbgTrace(D_RULEBASE_CONFIG) << "Loading all zones"; + auto all_zones_maybe = getSetting("rulebase", "zones"); + if (!all_zones_maybe.ok()) { + dbgWarning(D_RULEBASE_CONFIG) << "Failed to load zones"; + return; + } + + dbgTrace(D_RULEBASE_CONFIG) << "Creating cache of all zones by ID"; + map all_zones; + for (const auto &single_zone : all_zones_maybe.unpack().zones) { + if (used_zones.count(single_zone.getId()) > 0 && single_zone.isAnyZone()) { + dbgTrace(D_RULEBASE_CONFIG) << "Found used zone of type \"Any\": saving all zones as active zones"; + zones = all_zones_maybe.unpack().zones; + return; + } + + dbgDebug(D_RULEBASE_CONFIG) + << "Adding specific zone to cache. Zone ID: " + << single_zone.getId() + << ", name: " + << single_zone.getName(); + all_zones.emplace(single_zone.getId(), single_zone); + } + + dbgTrace(D_RULEBASE_CONFIG) << "Creating list of active zones"; + map active_zones_set; + for (const auto &single_used_zone_id : used_zones) { + const auto &found_zone = all_zones[single_used_zone_id]; + dbgTrace(D_RULEBASE_CONFIG) + << "Adding zone to list of active zones. Zone ID: " + << single_used_zone_id + << ", zone name: " + << found_zone.getName(); + active_zones_set.emplace(found_zone.getId(), found_zone); + + for (const auto &adjacent_zone : found_zone.getAdjacentZones()) { + const auto &adjacent_zone_obj = all_zones[adjacent_zone.second]; + dbgTrace(D_RULEBASE_CONFIG) + << "Adding adjacent zone to list of active zones. Zone ID: " + << adjacent_zone_obj.getId() + << ", zone name: " + << adjacent_zone_obj.getName(); + active_zones_set.emplace(adjacent_zone_obj.getId(), adjacent_zone_obj); + } + } + + vector implied_zones = { + "impliedAzure", + "impliedDNS", + "impliedSSH", + "impliedProxy", + "impliedFog" + }; + + GenericConfigId any_zone_id = ""; + for (const auto &single_zone : all_zones_maybe.unpack().zones) { + if (single_zone.isAnyZone()) any_zone_id = single_zone.getId(); + } + for (GenericConfigId &implied_id: implied_zones) { + if (all_zones.find(implied_id) != all_zones.end()) { + dbgDebug(D_RULEBASE_CONFIG) << "Adding implied zone to cache. Zone ID: " << implied_id; + active_zones_set.emplace(implied_id, all_zones[implied_id]); + if (any_zone_id != "" && active_zones_set.count(any_zone_id) == 0) { + active_zones_set.emplace(any_zone_id, all_zones[any_zone_id]); + } + } + } + + for (const auto &single_id_zone_pair : active_zones_set) { + zones.push_back(single_id_zone_pair.second); + } +} + +void +ZonesConfig::preload() +{ + registerExpectedSetting("rulebase", "zones"); + registerExpectedSetting("rulebase", "usedZones"); +} diff --git a/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc b/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc index 248a0d4..37bca7b 100644 --- a/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc +++ b/core/agent_details_reporter/agent_details_reporter_ut/agent_details_reporter_ut.cc @@ -95,6 +95,7 @@ TEST_F(AgentReporterTest, dataReport) " }" "\n}", MessageCategory::GENERIC, + _, _ )).Times(1); AgentDataReport() << AgentReportField(custom_data);; @@ -112,6 +113,7 @@ TEST_F(AgentReporterTest, labeledDataReport) " }" "\n}", MessageCategory::GENERIC, + _, _ )).Times(1); AgentDataReport() << AgentReportFieldWithLabel("this_is_custom_label", data); @@ -131,6 +133,7 @@ TEST_F(AgentReporterTest, multiDataReport) " }" "\n}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -158,6 +161,7 @@ TEST_F(AgentReporterTest, multiDataReportWithRegistrationData) " \"architecture\": \"aaa\"\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -181,6 +185,7 @@ TEST_F(AgentReporterTest, basicAttrTest) " \"additionalMetaData\": {}\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -200,6 +205,7 @@ TEST_F(AgentReporterTest, basicAttrTest) " }\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -219,6 +225,7 @@ TEST_F(AgentReporterTest, basicAttrTest) " \"additionalMetaData\": {}\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -247,6 +254,7 @@ TEST_F(AgentReporterTest, advancedAttrTest) " }\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -272,6 +280,7 @@ TEST_F(AgentReporterTest, advancedAttrTest) " }\n" "}", MessageCategory::GENERIC, + _, _ )).Times(1); @@ -297,6 +306,7 @@ TEST_F(AgentReporterTest, RestDetailsTest) "/agents", rest_call_parameters.str(), MessageCategory::GENERIC, + _, _ )).Times(1); @@ -366,6 +376,7 @@ TEST_F(AgentReporterTest, PersistenceAttrTest) "/agents", expected_attributes, MessageCategory::GENERIC, + _, _ )).Times(1); EXPECT_TRUE(report->sendAttributes()); diff --git a/core/debug_is/debug_is_ut/debug_ut.cc b/core/debug_is/debug_is_ut/debug_ut.cc index 3fffa62..fe0357c 100644 --- a/core/debug_is/debug_is_ut/debug_ut.cc +++ b/core/debug_is/debug_is_ut/debug_ut.cc @@ -824,6 +824,7 @@ TEST(DebugFogTest, fog_stream) "/api/v1/agents/events/bulk", _, _, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -980,7 +981,7 @@ TEST(DebugFogTest, fog_stream) .WillOnce(DoAll(InvokeMainLoopCB(), Return(0))); string message_body_1, message_body_2; - EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _)) + EXPECT_CALL(messaging_mock, sendAsyncMessage(_, "/api/v1/agents/events", _, MessageCategory::DEBUG, _, _)) .WillOnce(SaveArg<2>(&message_body_1)) .WillOnce(SaveArg<2>(&message_body_2)); diff --git a/core/include/general/common.h b/core/include/general/common.h index 81095ae..984bf56 100644 --- a/core/include/general/common.h +++ b/core/include/general/common.h @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -132,6 +133,18 @@ operator<<(ostream &os, const pair &printable_pair) return os; } +template +ostream & +operator<<(ostream &os, const vector &obj) +{ + bool first = true; + for (const auto &val : obj) { + os << (first ? "" : ", ") << val; + first = false; + } + return os; +} + } // namespace std #endif // __COMMON_H__ diff --git a/core/include/services_sdk/interfaces/i_messaging.h b/core/include/services_sdk/interfaces/i_messaging.h index 52a3bf6..f8e7171 100644 --- a/core/include/services_sdk/interfaces/i_messaging.h +++ b/core/include/services_sdk/interfaces/i_messaging.h @@ -56,14 +56,16 @@ public: const std::string &uri, serializableObject &req_obj, const MessageCategory category = MessageCategory::GENERIC, - MessageMetadata message_metadata = MessageMetadata()); + MessageMetadata message_metadata = MessageMetadata(), + bool force_buffering = true); virtual void sendAsyncMessage( const HTTPMethod method, const std::string &uri, const std::string &body, const MessageCategory category, - MessageMetadata message_metadata + const MessageMetadata &message_metadata = MessageMetadata(), + bool force_buffering = true ) = 0; virtual Maybe sendSyncMessage( diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h index 9d73631..6fb0460 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h @@ -28,6 +28,7 @@ AssetReply::load(cereal::JSONInputArchive &ar) SerializableMultiMap> tmp_multimap; ar( cereal::make_nvp("schemaVersion", asset_schema_version), + cereal::make_nvp("assetType", asset_type), cereal::make_nvp("assetTypeSchemaVersion", asset_type_schema_version), cereal::make_nvp("class", asset_class), cereal::make_nvp("category", asset_category), @@ -65,7 +66,6 @@ AssetReply::load(cereal::JSONInputArchive &ar) } catch(...) {} ar(cereal::make_nvp("sources", sources)); - ar(cereal::make_nvp("assetType", asset_type)); } template diff --git a/core/include/services_sdk/interfaces/messaging/interface_impl.h b/core/include/services_sdk/interfaces/messaging/interface_impl.h index 1995dc1..a571bd3 100644 --- a/core/include/services_sdk/interfaces/messaging/interface_impl.h +++ b/core/include/services_sdk/interfaces/messaging/interface_impl.h @@ -128,7 +128,8 @@ I_Messaging::sendAsyncMessage( const std::string &uri, serializableObject &req_obj, const MessageCategory category, - MessageMetadata message_metadata) + MessageMetadata message_metadata, + bool force_buffering) { Maybe req_body = req_obj.genJson(); if (!req_body.ok()) { @@ -141,7 +142,8 @@ I_Messaging::sendAsyncMessage( uri, req_body.unpack(), category, - message_metadata + message_metadata, + force_buffering ); } diff --git a/core/include/services_sdk/interfaces/mock/mock_messaging.h b/core/include/services_sdk/interfaces/mock/mock_messaging.h index 8060db6..061a0e2 100644 --- a/core/include/services_sdk/interfaces/mock/mock_messaging.h +++ b/core/include/services_sdk/interfaces/mock/mock_messaging.h @@ -18,14 +18,15 @@ public: MessageMetadata ) ); - MOCK_METHOD5( + MOCK_METHOD6( sendAsyncMessage, void ( HTTPMethod, const string &, const string &, MessageCategory, - MessageMetadata + const MessageMetadata &, + bool ) ); diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 9c30422..cc07c30 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -80,6 +80,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_JSON, D_WAAP) DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP) DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP) + DEFINE_FLAG(D_WAAP_HEADERS, D_WAAP) DEFINE_FLAG(D_WAAP_PARSER, D_WAAP) DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_HTML, D_WAAP_PARSER) @@ -96,6 +97,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER) + DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP) DEFINE_FLAG(D_IPS, D_COMPONENT) diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h index 1698824..08b90b9 100644 --- a/core/include/services_sdk/resources/generic_metric.h +++ b/core/include/services_sdk/resources/generic_metric.h @@ -58,7 +58,8 @@ public: const ReportIS::IssuingEngine &_issuing_engine, std::chrono::seconds _report_interval, bool _reset, - ReportIS::Audience _audience = ReportIS::Audience::INTERNAL + ReportIS::Audience _audience = ReportIS::Audience::INTERNAL, + bool _force_buffering = false ); template @@ -107,6 +108,7 @@ private: std::chrono::seconds report_interval; std::vector calcs; bool reset; + bool force_buffering = false; Context ctx; }; diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h index 4b2bb92..3d5d121 100644 --- a/core/include/services_sdk/resources/intelligence_invalidation.h +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -29,6 +29,7 @@ namespace Intelligence enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND }; enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED }; +enum class InvalidationType { ADD, DELETE, UPDATE }; class Invalidation { @@ -36,16 +37,20 @@ public: Invalidation(const std::string &class_value); Invalidation & setClassifier(ClassifierType type, const std::string &val); - Invalidation & setStringAttr(const std::string &attr, const std::string &val); - Invalidation & setStringSetAttr(const std::string &attr, const std::set &val); + Invalidation & setStringAttr(const std::string &attr, const std::string &val, bool is_main = true); + Invalidation & setStringSetAttr(const std::string &attr, const std::set &val, bool is_main = true); Invalidation & setSourceId(const std::string &id); Invalidation & setObjectType(ObjectType type); + Invalidation & setInvalidationType(InvalidationType type); std::string getClassifier(ClassifierType type) const { return classifiers[type]; } + Maybe getStringMainAttr(const std::string &attr) const; + Maybe, void> getStringSetMainAttr(const std::string &attr) const; Maybe getStringAttr(const std::string &attr) const; Maybe, void> getStringSetAttr(const std::string &attr) const; const Maybe & getSourceId() const { return source_id; } const Maybe & getObjectType() const { return object_type; } + InvalidationType getInvalidationType() const { return invalidation_type; } bool report(I_Intelligence_IS_V2 *interface) const; @@ -59,13 +64,17 @@ public: bool matches(const Invalidation &other) const; private: + bool hasMainAttr(const std::string &key, const std::string &value) const; bool hasAttr(const std::string &key, const std::string &value) const; EnumArray classifiers; std::map string_main_attr; std::map> set_string_main_attr; + std::map string_attr; + std::map> set_string_attr; Maybe source_id; Maybe object_type; + InvalidationType invalidation_type = InvalidationType::ADD; Maybe listening_id; }; diff --git a/core/intelligence_is_v2/include/intelligence_server.h b/core/intelligence_is_v2/include/intelligence_server.h index 54e84f7..41856b9 100644 --- a/core/intelligence_is_v2/include/intelligence_server.h +++ b/core/intelligence_is_v2/include/intelligence_server.h @@ -37,7 +37,7 @@ private: IntelligenceRequest request; Flags conn_flags; - bool is_local_intelligence; + bool is_local_intelligence = false; Maybe server_ip = genError("No server ip set"); Maybe server_port = genError("No port is set"); I_Messaging * i_message = nullptr; diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 3ccc394..a75bcd2 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -244,7 +244,12 @@ public: bool sendInvalidation(const Invalidation &invalidation) const override { - return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation); + if (hasLocalInvalidationSupport()) { + return sendLocalInvalidation(invalidation); + } + else { + return sendGlobalInvalidation(invalidation); + } } Maybe @@ -287,9 +292,14 @@ public: private: bool - hasLocalIntelligence() const + hasLocalInvalidationSupport() const { - return getProfileAgentSettingWithDefault(false, "agent.config.useLocalIntelligence"); + auto is_supported = getProfileAgentSettingWithDefault(false, "agent.config.useLocalIntelligence"); + + if (!is_supported) { + is_supported = getProfileAgentSettingWithDefault(false, "agent.config.supportInvalidation"); + } + return is_supported; } bool @@ -330,6 +340,7 @@ private: MessageMetadata invalidation_req_md(server, *port); invalidation_req_md.insertHeaders(getHTTPHeaders()); + invalidation_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); return message->sendSyncMessageWithoutResponse( HTTPMethod::POST, invalidation_uri, @@ -411,6 +422,7 @@ private: dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson(); MessageMetadata registration_req_md(server, *port); + registration_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); return message->sendSyncMessageWithoutResponse( HTTPMethod::POST, registration_uri, @@ -423,7 +435,7 @@ private: void sendReccurringInvalidationRegistration() const { - if (!hasLocalIntelligence() || invalidations.empty()) return; + if (!hasLocalInvalidationSupport() || invalidations.empty()) return; sendLocalRegistrationImpl(invalidations.getRegistration()); } diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index 06883f5..d47486f 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -411,8 +411,9 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest) "}\n" ); + MessageMetadata md; EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) - ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))); + ).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str)))); auto maybe_ans = intell->queryIntelligence(request); EXPECT_TRUE(maybe_ans.ok()); @@ -421,6 +422,7 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest) auto iter = vec.begin(); EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry"); EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing"); + EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest) @@ -490,6 +492,7 @@ TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest) EXPECT_TRUE(maybe_ans.ok()); EXPECT_EQ(md.getHostName(), "127.0.0.1"); + EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest) diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index 8cd7763..3881c18 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -25,59 +25,106 @@ TEST(InvalidationBasic, SettersAndGetters) EXPECT_EQ(invalidation.getClassifier(ClassifierType::GROUP), ""); EXPECT_EQ(invalidation.getClassifier(ClassifierType::ORDER), ""); EXPECT_EQ(invalidation.getClassifier(ClassifierType::KIND), ""); + EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::ADD); + EXPECT_FALSE(invalidation.getStringMainAttr("main_attr1").ok()); + EXPECT_FALSE(invalidation.getStringSetMainAttr("main_attr2").ok()); EXPECT_FALSE(invalidation.getStringAttr("attr1").ok()); EXPECT_FALSE(invalidation.getStringSetAttr("attr2").ok()); EXPECT_FALSE(invalidation.getSourceId().ok()); EXPECT_FALSE(invalidation.getObjectType().ok()); - set vals = { "2", "3" }; + set main_vals = { "2", "3" }; + set vals = { "5", "6" }; invalidation .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") - .setStringAttr("attr1", "1") - .setStringSetAttr("attr2", vals) + .setStringAttr("main_attr1", "1") + .setStringSetAttr("main_attr2", main_vals) + .setStringAttr("attr1", "4", false) + .setStringSetAttr("attr2", vals, false) .setSourceId("id") - .setObjectType(Intelligence::ObjectType::ASSET); + .setObjectType(Intelligence::ObjectType::ASSET) + .setInvalidationType(InvalidationType::DELETE); EXPECT_EQ(invalidation.getClassifier(ClassifierType::CATEGORY), "bbb"); EXPECT_EQ(invalidation.getClassifier(ClassifierType::FAMILY), "ccc"); - EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "1"); + EXPECT_EQ(invalidation.getStringMainAttr("main_attr1").unpack(), "1"); + EXPECT_EQ(invalidation.getStringSetMainAttr("main_attr2").unpack(), main_vals); + EXPECT_EQ(invalidation.getStringAttr("attr1").unpack(), "4"); EXPECT_EQ(invalidation.getStringSetAttr("attr2").unpack(), vals); EXPECT_EQ(invalidation.getSourceId().unpack(), "id"); EXPECT_EQ(invalidation.getObjectType().unpack(), Intelligence::ObjectType::ASSET); + EXPECT_EQ(invalidation.getInvalidationType(), InvalidationType::DELETE); } TEST(InvalidationBasic, Matching) { - set vals = { "2", "3" }; + set main_vals = { "2", "3" }; + set vals = { "5", "6" }; auto base_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") - .setStringAttr("attr1", "1") - .setStringSetAttr("attr2", vals); + .setStringAttr("main_attr1", "1") + .setStringSetAttr("main_attr2", main_vals) + .setStringAttr("attr1", "4", false) + .setStringSetAttr("attr2", vals, false); auto matching_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("attr1", "1") - .setStringSetAttr("attr2", vals) - .setStringAttr("attr3", "6") + .setStringAttr("main_attr1", "1") + .setStringSetAttr("main_attr2", main_vals) + .setStringAttr("attr1", "4", false) + .setStringSetAttr("attr2", vals, false) + .setStringAttr("main_attr3", "6") + .setStringAttr("attr3", "7", false) + .setSourceId("id") + .setObjectType(Intelligence::ObjectType::ASSET) + .setInvalidationType(InvalidationType::ADD); + + EXPECT_TRUE(base_invalidation.matches(matching_invalidation)); + + auto not_matching_invalidation_type = Invalidation("aaa") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::GROUP, "ddd") + .setStringAttr("main_attr1", "1") + .setStringSetAttr("main_attr2", main_vals) + .setSourceId("id") + .setObjectType(Intelligence::ObjectType::ASSET) + .setInvalidationType(InvalidationType::DELETE); + + EXPECT_FALSE(base_invalidation.matches(not_matching_invalidation_type)); + + auto missing_attr_invalidation_main = Invalidation("aaa") + .setClassifier(ClassifierType::CATEGORY, "bbb") + .setClassifier(ClassifierType::FAMILY, "ccc") + .setClassifier(ClassifierType::GROUP, "ddd") + .setStringAttr("main_attr1", "1") + .setStringAttr("main_attr2", "2") + .setStringAttr("main_attr3", "6") + .setStringAttr("attr1", "4", false) + .setStringSetAttr("attr2", vals, false) + .setStringAttr("attr3", "7", false) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); - EXPECT_TRUE(base_invalidation.matches(matching_invalidation)); + EXPECT_FALSE(base_invalidation.matches(missing_attr_invalidation_main)); auto missing_attr_invalidation = Invalidation("aaa") .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringAttr("attr1", "1") - .setStringAttr("attr2", "2") - .setStringAttr("attr3", "6") + .setStringAttr("main_attr1", "1") + .setStringSetAttr("main_attr2", main_vals) + .setStringAttr("main_attr3", "6") + .setStringAttr("attr1", "4", false) + .setStringAttr("attr2", "2", false) + .setStringAttr("attr3", "7", false) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); @@ -88,9 +135,12 @@ TEST(InvalidationBasic, Matching) .setClassifier(ClassifierType::CATEGORY, "bbb") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::GROUP, "ddd") - .setStringSetAttr("attr1", vals2) - .setStringSetAttr("attr2", vals) - .setStringAttr("attr3", "6") + .setStringSetAttr("main_attr1", vals2) + .setStringSetAttr("main_attr2", main_vals) + .setStringAttr("main_attr3", "6") + .setStringAttr("attr1", "4", false) + .setStringSetAttr("attr2", vals, false) + .setStringAttr("attr3", "7", false) .setSourceId("id") .setObjectType(Intelligence::ObjectType::ASSET); @@ -170,17 +220,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) { auto invalidation = Invalidation("aaa") .setStringAttr("attr2", "2") + .setStringAttr("attr3", "3", false) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") .setObjectType(Intelligence::ObjectType::ASSET); string invalidation_json; + MessageMetadata md; EXPECT_CALL( messaging_mock, sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) ).WillOnce(DoAll( SaveArg<2>(&invalidation_json), + SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); @@ -192,16 +245,20 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation) "\"category\": \"bbb\", " "\"family\": \"ccc\", " "\"objectType\": \"asset\", " + "\"invalidationType\": \"add\", " "\"sourceId\": \"id\", " - "\"mainAttributes\": [ { \"attr2\": \"2\" } ]" + "\"mainAttributes\": [ { \"attr2\": \"2\" } ], " + "\"attributes\": [ { \"attr3\": \"3\" } ]" " } ] }"; EXPECT_EQ(invalidation_json, expected_json); + EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } TEST_F(IntelligenceInvalidation, sending_private_invalidation) { auto invalidation = Invalidation("aaa") .setStringAttr("attr2", "2") + .setStringAttr("attr3", "3", false) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") @@ -221,11 +278,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation) Singleton::Consume::from(conf)->loadConfiguration(configuration); string invalidation_json; + MessageMetadata md; EXPECT_CALL( messaging_mock, sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _) ).WillOnce(DoAll( SaveArg<2>(&invalidation_json), + SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); @@ -237,10 +296,13 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation) "\"category\": \"bbb\", " "\"family\": \"ccc\", " "\"objectType\": \"asset\", " + "\"invalidationType\": \"add\", " "\"sourceId\": \"id\", " - "\"mainAttributes\": [ { \"attr2\": \"2\" } ]" + "\"mainAttributes\": [ { \"attr2\": \"2\" } ], " + "\"attributes\": [ { \"attr3\": \"3\" } ]" " } ] }"; EXPECT_EQ(invalidation_json, expected_json); + EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } TEST_F(IntelligenceInvalidation, register_for_invalidation) @@ -257,19 +319,23 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) configuration << "}"; Singleton::Consume::from(conf)->loadConfiguration(configuration); + set vals = { "11", "55", "22" }; auto invalidation = Invalidation("aaa") .setStringAttr("attr2", "2") + .setStringSetAttr("attr3", vals, false) .setSourceId("id") .setClassifier(ClassifierType::FAMILY, "ccc") .setClassifier(ClassifierType::CATEGORY, "bbb") .setObjectType(Intelligence::ObjectType::ASSET); string body; + MessageMetadata md; EXPECT_CALL( messaging_mock, sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) ).WillOnce(DoAll( SaveArg<2>(&body), + SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")) )); @@ -278,6 +344,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) EXPECT_THAT(body, HasSubstr("\"url\": \"http://127.0.0.1:7000/set-new-invalidation\"")); EXPECT_THAT(body, HasSubstr("\"apiVersion\": \"v2\", \"communicationType\": \"sync\"")); EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]")); + EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": [ \"11\", \"22\", \"55\" ] } ]")); + EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); } TEST_F(IntelligenceInvalidation, invalidation_callback) @@ -321,7 +389,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 1); - EXPECT_EQ(recieved_invalidations[0].getStringSetAttr("attr2").unpack(), vals); + EXPECT_EQ(recieved_invalidations[0].getStringSetMainAttr("attr2").unpack(), vals); } TEST_F(IntelligenceInvalidation, delete_invalidation_callback) diff --git a/core/intelligence_is_v2/intelligence_server.cc b/core/intelligence_is_v2/intelligence_server.cc index 9fed3a0..ee46b3d 100644 --- a/core/intelligence_is_v2/intelligence_server.cc +++ b/core/intelligence_is_v2/intelligence_server.cc @@ -43,7 +43,7 @@ Sender::Sender(IntelligenceRequest request) : request(request) } auto setting_server_ip = getSetting("intelligence", "local intelligence server ip"); - if (setting_server_ip.ok()) server_ip = *setting_server_ip; + if (setting_server_ip.ok() && is_local_intelligence) server_ip = *setting_server_ip; } Maybe @@ -80,6 +80,7 @@ Sender::sendQueryObjectToLocalServer(bool is_primary_port) server_port = *local_port; conn_flags.reset(); + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); auto res = sendQueryMessage(); @@ -173,7 +174,7 @@ Sender::sendMessage() MessageMetadata req_md(*server_ip, *server_port, conn_flags); auto req_status = i_message->sendSyncMessage( HTTPMethod::POST, - query_uri, + request.isBulk() ? queries_uri : query_uri, request, MessageCategory::INTELLIGENCE, req_md diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc index 6115118..025e163 100644 --- a/core/intelligence_is_v2/invalidation.cc +++ b/core/intelligence_is_v2/invalidation.cc @@ -37,16 +37,16 @@ Invalidation::setClassifier(ClassifierType type, const string &val) } Invalidation & -Invalidation::setStringAttr(const string &attr, const string &val) +Invalidation::setStringAttr(const string &attr, const string &val, bool is_main) { - string_main_attr[attr] = val; + is_main ? string_main_attr[attr] = val : string_attr[attr] = val; return *this; } Invalidation & -Invalidation::setStringSetAttr(const string &attr, const set &val) +Invalidation::setStringSetAttr(const string &attr, const set &val, bool is_main) { - set_string_main_attr[attr] = val; + is_main ? set_string_main_attr[attr] = val : set_string_attr[attr] = val; return *this; } @@ -64,8 +64,15 @@ Invalidation::setObjectType(ObjectType type) return *this; } +Invalidation & +Invalidation::setInvalidationType(InvalidationType type) +{ + invalidation_type = type; + return *this; +} + Maybe -Invalidation::getStringAttr(const string &attr) const +Invalidation::getStringMainAttr(const string &attr) const { auto val_ref = string_main_attr.find(attr); if (val_ref == string_main_attr.end()) return genError(); @@ -73,13 +80,29 @@ Invalidation::getStringAttr(const string &attr) const } Maybe, void> -Invalidation::getStringSetAttr(const string &attr) const +Invalidation::getStringSetMainAttr(const string &attr) const { auto val_ref = set_string_main_attr.find(attr); if (val_ref == set_string_main_attr.end()) return genError(); return val_ref->second; } +Maybe +Invalidation::getStringAttr(const string &attr) const +{ + auto val_ref = string_attr.find(attr); + if (val_ref == string_attr.end()) return genError(); + return val_ref->second; +} + +Maybe, void> +Invalidation::getStringSetAttr(const string &attr) const +{ + auto val_ref = set_string_attr.find(attr); + if (val_ref == set_string_attr.end()) return genError(); + return val_ref->second; +} + bool Invalidation::report(I_Intelligence_IS_V2 *interface) const { @@ -110,6 +133,12 @@ static const map convertObjectType = { { Intelligence::ObjectType::SHORTLIVED, "shortLived" } }; +static const map convertInvalidationType = { + { Intelligence::InvalidationType::ADD, "add" }, + { Intelligence::InvalidationType::DELETE, "delete" }, + { Intelligence::InvalidationType::UPDATE, "update" } +}; + Maybe Invalidation::genJson() const { @@ -145,6 +174,7 @@ Invalidation::genObject() const } if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"'; + invalidation << ", \"invalidationType\": \"" << convertInvalidationType.at(invalidation_type) << '"'; if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"'; if (!string_main_attr.empty() || !set_string_main_attr.empty()) { @@ -161,9 +191,35 @@ Invalidation::genObject() const auto val = makeSeparatedStr(attr.second, ", "); invalidation << "{ \"" << attr.first << "\": [ "; bool internal_first = true; - for (auto &val : attr.second) { + for (auto &value : attr.second) { if (!internal_first) invalidation << ", "; - invalidation << "\"" << val << "\""; + invalidation << "\"" << value << "\""; + internal_first = false; + } + invalidation << " ] }"; + first = false; + } + + invalidation << " ]"; + } + + if (!string_attr.empty() || !set_string_attr.empty()) { + invalidation << ", \"attributes\": [ "; + bool first = true; + for (auto &attr : string_attr) { + if (!first) invalidation << ", "; + invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }"; + first = false; + } + + for (auto &attr : set_string_attr) { + if (!first) invalidation << ", "; + auto val = makeSeparatedStr(attr.second, ", "); + invalidation << "{ \"" << attr.first << "\": [ "; + bool internal_first = true; + for (auto &value : attr.second) { + if (!internal_first) invalidation << ", "; + invalidation << "\"" << value << "\""; internal_first = false; } invalidation << " ] }"; @@ -208,16 +264,29 @@ Invalidation::matches(const Invalidation &other) const if (!other.object_type.ok() || *object_type != *other.object_type) return false; } + if (invalidation_type != other.invalidation_type) return false; + if (source_id.ok()) { if (!other.source_id.ok() || *source_id != *other.source_id) return false; } for (auto &key_value : string_main_attr) { - if (!other.hasAttr(key_value.first, key_value.second)) return false; + if (!other.hasMainAttr(key_value.first, key_value.second)) return false; } for (auto &key_values : set_string_main_attr) { + for (auto &value : key_values.second) { + if (!other.hasMainAttr(key_values.first, value)) return false; + } + } + + for (auto &key_value : string_attr) { + if (!other.hasAttr(key_value.first, key_value.second)) return false; + } + + + for (auto &key_values : set_string_attr) { for (auto &value : key_values.second) { if (!other.hasAttr(key_values.first, value)) return false; } @@ -227,7 +296,7 @@ Invalidation::matches(const Invalidation &other) const } bool -Invalidation::hasAttr(const string &key, const string &value) const +Invalidation::hasMainAttr(const string &key, const string &value) const { auto string_elem = string_main_attr.find(key); if (string_elem != string_main_attr.end()) return string_elem->second == value; @@ -239,3 +308,17 @@ Invalidation::hasAttr(const string &key, const string &value) const return false; } + +bool +Invalidation::hasAttr(const string &key, const string &value) const +{ + auto string_elem = string_attr.find(key); + if (string_elem != string_attr.end()) return string_elem->second == value; + + auto set_string_elem = set_string_attr.find(key); + if (set_string_elem != set_string_attr.end()) { + return set_string_elem->second.find(value) != set_string_elem->second.end(); + } + + return false; +} diff --git a/core/logging/k8s_svc_stream.cc b/core/logging/k8s_svc_stream.cc index 96f080b..f9bf877 100644 --- a/core/logging/k8s_svc_stream.cc +++ b/core/logging/k8s_svc_stream.cc @@ -41,6 +41,8 @@ K8sSvcStream::sendLog(const Report &log) MessageMetadata rest_req_md(svc_host, 80); rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); + rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + bool ok = i_msg->sendSyncMessageWithoutResponse( HTTPMethod::POST, K8sSvc_log_uri, @@ -69,6 +71,7 @@ K8sSvcStream::sendLog(const LogBulkRest &logs, bool persistence_only) MessageMetadata rest_req_md(svc_host, 80); rest_req_md.insertHeader("X-Tenant-Id", Singleton::Consume::by()->getTenantId()); + rest_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); bool ok = i_msg->sendSyncMessageWithoutResponse( HTTPMethod::POST, K8sSvc_log_uri, diff --git a/core/logging/logging_ut/logging_ut.cc b/core/logging/logging_ut/logging_ut.cc index 447ebbf..3518f09 100644 --- a/core/logging/logging_ut/logging_ut.cc +++ b/core/logging/logging_ut/logging_ut.cc @@ -171,6 +171,7 @@ public: _, _, MessageCategory::LOG, + _, _ )).WillRepeatedly(SaveArg<2>(&body)); } @@ -745,6 +746,7 @@ TEST_F(LogTest, FogBulkLogs) _, _, MessageCategory::LOG, + _, _ )).WillRepeatedly(SaveArg<2>(&local_body)); @@ -1143,6 +1145,7 @@ TEST(LogTestInstanceAwareness, LogGenInstanceAwareness) _, _, MessageCategory::LOG, + _, _ )).Times(AnyNumber()); @@ -1248,6 +1251,7 @@ TEST(LogTestWithoutComponent, RegisterBasicConfig) _, _, MessageCategory::LOG, + _, _ )).Times(AnyNumber()); @@ -1295,6 +1299,7 @@ TEST(LogTestWithoutComponent, RegisterAdvancedConfig) _, _, MessageCategory::LOG, + _, _ )).Times(AnyNumber()); @@ -1366,6 +1371,7 @@ TEST_F(LogTest, BulkModification) _, _, MessageCategory::LOG, + _, _ )).WillRepeatedly(SaveArg<2>(&local_body)); diff --git a/core/mainloop/mainloop_ut/mainloop_ut.cc b/core/mainloop/mainloop_ut/mainloop_ut.cc index 44667d5..2978c51 100644 --- a/core/mainloop/mainloop_ut/mainloop_ut.cc +++ b/core/mainloop/mainloop_ut/mainloop_ut.cc @@ -51,7 +51,7 @@ public: { EXPECT_CALL( mock_msg, - sendAsyncMessage(_, "/api/v1/agents/events", _, _, _) + sendAsyncMessage(_, "/api/v1/agents/events", _, _, _, _) ).Times(2).WillRepeatedly( WithArgs<2, 3>( Invoke( diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index 640ebcc..9ceac41 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -38,6 +38,10 @@ USE_DEBUG_FLAG(D_CONNECTION); static const HTTPResponse sending_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to send all data in time"); static const HTTPResponse receving_timeout(HTTPStatusCode::HTTP_UNKNOWN, "Failed to receive all data in time"); static const HTTPResponse parsing_error(HTTPStatusCode::HTTP_UNKNOWN, "Failed to parse the HTTP response"); +static const HTTPResponse close_error( + HTTPStatusCode::HTTP_UNKNOWN, + "The previous request failed to receive a response. Closing the connection" +); const string & MessageConnectionKey::getHostName() const @@ -125,6 +129,12 @@ public: return key; } + bool + shouldCloseConnection() const + { + return should_close_connection; + } + bool isOverProxy() const { @@ -152,12 +162,12 @@ public: if (establishConnection().ok()) { dbgDebug(D_MESSAGING) << "Reestablish connection"; - return true; + return false; } dbgWarning(D_MESSAGING) << "Reestablish connection failed"; active = genError(curr_time + chrono::seconds(300)); - return false; + return true; } Maybe @@ -200,6 +210,7 @@ public: << key.getPort() << (isOverProxy() ? ", Over proxy: " + settings.getProxyHost() + ":" + to_string(key.getPort()) : ""); active = Maybe(); + should_close_connection = false; return Maybe(); } @@ -443,25 +454,40 @@ private: I_MainLoop *i_mainloop = Singleton::Consume::by(); I_TimeGet *i_time = Singleton::Consume::by(); - auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + auto bio_connect = BIO_do_connect(bio.get()); uint attempts_count = 0; - while (i_time->getMonotonicTime() < conn_end_time) { - attempts_count++; - if (BIO_do_connect(bio.get()) > 0) { - if (isUnsecure() || isOverProxy()) return Maybe(); - return performHandshakeAndVerifyCert(i_time, i_mainloop); - } - + auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + while (i_time->getMonotonicTime() < conn_end_time && bio_connect <= 0) { if (!BIO_should_retry(bio.get())) { + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); + active = genError(curr_time + chrono::seconds(60)); string bio_error = ERR_error_string(ERR_get_error(), nullptr); - return genError("Failed to connect (BIO won't retry!): " + bio_error); + return genError( + "Failed to connect to: " + + full_address + + ", error: " + + bio_error + + ". Connection suspended for 60 seconds"); } - - if ((attempts_count % 10) == 0) i_mainloop->yield(true); + attempts_count++; + if (!isBioSocketReady()) { + i_mainloop->yield((attempts_count % 10) == 0); + continue; + } + bio_connect = BIO_do_connect(bio.get()); } - - return genError("Failed to establish new connection to " + full_address + " after reaching timeout."); + if (bio_connect > 0) { + if (isUnsecure() || isOverProxy()) return Maybe(); + return performHandshakeAndVerifyCert(i_time, i_mainloop); + } + auto curr_time = chrono::duration_cast(i_time->getMonotonicTime()); + active = genError(curr_time + chrono::seconds(60)); + return genError( + "Failed to establish new connection to: " + + full_address + + " after reaching timeout." + + " Connection suspended for 60 seconds"); } Maybe @@ -544,6 +570,7 @@ private: Maybe sendAndReceiveData(const string &request, bool is_connect) { + dbgFlow(D_CONNECTION) << "Sending and receiving data"; I_MainLoop *i_mainloop = Singleton::Consume::by(); while (lock) { i_mainloop->yield(true); @@ -551,6 +578,11 @@ private: lock = true; auto unlock = make_scope_exit([&] () { lock = false; }); + if (should_close_connection) { + dbgWarning(D_CONNECTION) << close_error.getBody(); + return genError(close_error); + } + I_TimeGet *i_time = Singleton::Consume::by(); auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); size_t data_left_to_send = request.length(); @@ -567,9 +599,15 @@ private: HTTPResponseParser http_parser; dbgTrace(D_CONNECTION) << "Sent the message, now waiting for response"; while (!http_parser.hasReachedError()) { - if (i_time->getMonotonicTime() > receiving_end_time) return genError(receving_timeout); + if (i_time->getMonotonicTime() > receiving_end_time) { + should_close_connection = true; + return genError(receving_timeout); + }; auto receieved = receiveData(); - if (!receieved.ok()) return receieved.passErr(); + if (!receieved.ok()) { + should_close_connection = true; + return receieved.passErr(); + } auto response = http_parser.parseData(*receieved, is_connect); if (response.ok()) { dbgTrace(D_MESSAGING) << printOut(response.unpack().toString()); @@ -611,6 +649,7 @@ private: uint failed_attempts = 0; bool lock = false; + bool should_close_connection = false; }; Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata) @@ -663,6 +702,12 @@ Connection::getConnKey() const return pimpl->getConnKey(); } +bool +Connection::shouldCloseConnection() const +{ + return pimpl->shouldCloseConnection(); +} + bool Connection::isOverProxy() const { diff --git a/core/messaging/connection/connection_comp.cc b/core/messaging/connection/connection_comp.cc index 26421f8..a033d79 100644 --- a/core/messaging/connection/connection_comp.cc +++ b/core/messaging/connection/connection_comp.cc @@ -53,6 +53,10 @@ public: { auto conn = persistent_connections.find(MessageConnectionKey(host_name, port, category)); if (conn == persistent_connections.end()) return genError("No persistent connection found"); + if (conn->second.shouldCloseConnection()) { + persistent_connections.erase(conn); + return genError("The connection needs to reestablish"); + } return conn->second; } @@ -93,6 +97,7 @@ private: if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate); auto connected = conn.establishConnection(); + persistent_connections.emplace(conn_key, conn); if (!connected.ok()) { string connection_err = "Failed to establish connection. Error: " + connected.getErr(); @@ -101,7 +106,6 @@ private: } dbgTrace(D_CONNECTION) << "Connection establish succssesfuly"; - persistent_connections.emplace(conn_key, conn); return conn; } diff --git a/core/messaging/connection/connection_ut/connection_comp_ut.cc b/core/messaging/connection/connection_ut/connection_comp_ut.cc index af4f3a9..bc1bc4f 100644 --- a/core/messaging/connection/connection_ut/connection_comp_ut.cc +++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc @@ -23,83 +23,13 @@ #include "mock/mock_encryptor.h" #include "rest.h" #include "rest_server.h" +#include "dummy_socket.h" using namespace std; using namespace testing; USE_DEBUG_FLAG(D_CONNECTION); -class DummySocket : Singleton::Consume -{ -public: - ~DummySocket() - { - if (server_fd != -1) close(server_fd); - if (connection_fd != -1) close(connection_fd); - } - - void - init() - { - server_fd = socket(AF_INET, SOCK_STREAM, 0); - dbgAssert(server_fd >= 0) << "Failed to open a socket"; - int socket_enable = 1; - setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)); - - struct sockaddr_in addr; - bzero(&addr, sizeof(addr)); - - addr.sin_family = AF_INET; - addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - addr.sin_port = htons(8080); - bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); - listen(server_fd, 100); - } - - void - acceptSocket() - { - if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr); - } - - string - readFromSocket() - { - acceptSocket(); - - string res; - char buffer[1024]; - while (int bytesRead = readRaw(buffer, sizeof(buffer))) { - res += string(buffer, bytesRead); - } - return res; - } - - void - writeToSocket(const string &msg) - { - acceptSocket(); - EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size()); - } - -private: - int - readRaw(char *buf, uint len) - { - struct pollfd s_poll; - s_poll.fd = connection_fd; - s_poll.events = POLLIN; - s_poll.revents = 0; - - if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0; - - return read(connection_fd, buf, len); - } - - int server_fd = -1; - int connection_fd = -1; -}; - static ostream & operator<<(ostream &os, const BufferedMessage &) { @@ -179,7 +109,6 @@ TEST_F(TestConnectionComp, testEstablishNewConnection) conn_metadata.setExternalCertificate("external cert"); auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); - cout << "[OREN] read: " << endl; ASSERT_TRUE(maybe_connection.ok()); auto get_conn = maybe_connection.unpack(); EXPECT_EQ(get_conn.getConnKey().getHostName(), "127.0.0.1"); @@ -228,6 +157,43 @@ TEST_F(TestConnectionComp, testSendRequest) EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg); } +TEST_F(TestConnectionComp, testCloseConnectionBeforeResponse) +{ + // Create a connection + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata("127.0.0.1", 8080, conn_flags); + + // Insert the connection to the map + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + ASSERT_TRUE(maybe_connection.ok()); + + // Get the connection from the map - Should be successful + auto maybe_get_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG); + ASSERT_TRUE(maybe_get_connection.ok()); + auto conn = maybe_get_connection.unpack(); + + auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body"); + ASSERT_TRUE(req.ok()); + + // force the connection to be closed + ON_CALL(mock_mainloop, yield(false)).WillByDefault(InvokeWithoutArgs([&] () { return; })); + + EXPECT_CALL(mock_timer, getMonotonicTime()) + .WillRepeatedly(Invoke([] () {static int j = 0; return chrono::microseconds(++j * 1000 * 1000);})); + + auto maybe_response = i_conn->sendRequest(conn, *req); + ASSERT_TRUE(!maybe_response.ok()); + ASSERT_EQ( + maybe_response.getErr().toString(), + "[Status-code]: -1 - HTTP_UNKNOWN, [Body]: Failed to receive all data in time" + ); + + auto maybe_get_closed_connection = i_conn->getPersistentConnection("127.0.0.1", 8080, MessageCategory::LOG); + ASSERT_TRUE(!maybe_get_closed_connection.ok()); + ASSERT_EQ(maybe_get_closed_connection.getErr(), "The connection needs to reestablish"); +} + TEST_F(TestConnectionComp, testSendRequestReplyChunked) { Flags conn_flags; diff --git a/core/messaging/include/connection.h b/core/messaging/include/connection.h index 28ca1b5..46485ca 100644 --- a/core/messaging/include/connection.h +++ b/core/messaging/include/connection.h @@ -63,6 +63,7 @@ public: bool isOverProxy() const; bool isUnsecure() const; bool isSuspended(); + bool shouldCloseConnection() const; Maybe establishConnection(); Maybe sendRequest(const std::string &request); diff --git a/core/messaging/include/dummy_socket.h b/core/messaging/include/dummy_socket.h new file mode 100644 index 0000000..101797f --- /dev/null +++ b/core/messaging/include/dummy_socket.h @@ -0,0 +1,100 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __DUMMY_SOCKET_H__ +#define __DUMMY_SOCKET_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "singleton.h" +#include "i_mainloop.h" +#include "agent_core_utilities.h" + +class DummySocket : Singleton::Consume +{ +public: + ~DummySocket() + { + if (server_fd != -1) close(server_fd); + if (connection_fd != -1) close(connection_fd); + } + + void + init() + { + server_fd = socket(AF_INET, SOCK_STREAM, 0); + dbgAssert(server_fd >= 0) << "Failed to open a socket"; + int socket_enable = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)); + + struct sockaddr_in addr; + bzero(&addr, sizeof(addr)); + + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + addr.sin_port = htons(8080); + bind(server_fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)); + listen(server_fd, 100); + } + + void + acceptSocket() + { + if (connection_fd == -1) connection_fd = accept(server_fd, nullptr, nullptr); + } + + std::string + readFromSocket() + { + acceptSocket(); + + std::string res; + char buffer[1024]; + while (int bytesRead = readRaw(buffer, sizeof(buffer))) { + res += std::string(buffer, bytesRead); + } + return res; + } + + void + writeToSocket(const std::string &msg) + { + acceptSocket(); + EXPECT_EQ(write(connection_fd, msg.data(), msg.size()), msg.size()); + } + +private: + int + readRaw(char *buf, uint len) + { + struct pollfd s_poll; + s_poll.fd = connection_fd; + s_poll.events = POLLIN; + s_poll.revents = 0; + + if (poll(&s_poll, 1, 0) <= 0 || (s_poll.revents & POLLIN) == 0) return 0; + + return read(connection_fd, buf, len); + } + + int server_fd = -1; + int connection_fd = -1; +}; + +#endif // __DUMMY_SOCKET_H__ diff --git a/core/messaging/include/messaging_comp.h b/core/messaging/include/messaging_comp.h index f432004..210254a 100644 --- a/core/messaging/include/messaging_comp.h +++ b/core/messaging/include/messaging_comp.h @@ -50,7 +50,8 @@ public: const std::string &uri, const std::string &body, MessageCategory category, - const MessageMetadata &message_metadata + const MessageMetadata &message_metadata, + bool force_buffering = true ); Maybe downloadFile( diff --git a/core/messaging/messaging.cc b/core/messaging/messaging.cc index bacc71e..5e200d4 100644 --- a/core/messaging/messaging.cc +++ b/core/messaging/messaging.cc @@ -55,10 +55,11 @@ public: const std::string &uri, const std::string &body, const MessageCategory category, - MessageMetadata message_metadata + const MessageMetadata &message_metadata, + bool force_buffering ) override { - return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata); + return messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, force_buffering); } Maybe diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc index a5d7fc3..59c8a97 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc @@ -153,8 +153,6 @@ MessagingBufferComponent::Impl::pushNewBufferedMessage( { dbgTrace(D_MESSAGING_BUFFER) << "Pushing new message to buffer"; - message_metadata.setShouldBufferMessage(false); - if (!force_immediate_writing) { dbgDebug(D_MESSAGING_BUFFER) << "Holding message temporarily in memory"; memory_messages.emplace_back(body, method, uri, category, message_metadata); @@ -296,12 +294,14 @@ MessagingBufferComponent::Impl::sendMessage() HTTPStatusCode MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) const { + MessageMetadata message_metadata = message.getMessageMetadata(); + message_metadata.setShouldBufferMessage(false); auto res = messaging->sendSyncMessage( message.getMethod(), message.getURI(), message.getBody(), message.getCategory(), - message.getMessageMetadata() + message_metadata ); if (res.ok()) return HTTPStatusCode::HTTP_OK; @@ -316,7 +316,9 @@ MessagingBufferComponent::Impl::handleInMemoryMessages() memory_messages.reserve(32); for (const auto &message : messages) { - if (sendMessage(message) != HTTPStatusCode::HTTP_OK) writeToDisk(message); + if (sendMessage(message) != HTTPStatusCode::HTTP_OK) { + if (message.getMessageMetadata().shouldBufferMessage()) writeToDisk(message); + } mainloop->yield(); } } diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc index b3fbfa5..40323ba 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp_ut/messaging_buffer_comp_ut.cc @@ -233,17 +233,36 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory) string body_1 = "body1"; string body_2 = "body2"; string body_3 = "body3"; + string body_4 = "body4"; string uri_1 = "uri_1"; string uri_2 = "uri_2"; string uri_3 = "uri_3"; + string uri_4 = "uri_4"; MessageCategory category = MessageCategory::GENERIC; MessageMetadata message_metadata = MessageMetadata(); + MessageMetadata msg_2_message_metadata = MessageMetadata(); + msg_2_message_metadata.setShouldBufferMessage(true); HTTPMethod method = HTTPMethod::POST; buffer_provider->pushNewBufferedMessage(body_1, method, uri_1, category, message_metadata, false); - buffer_provider->pushNewBufferedMessage(body_2, method, uri_2, category, message_metadata, false); + buffer_provider->pushNewBufferedMessage( + body_2, + method, + uri_2, + category, + msg_2_message_metadata, + false + ); // should be buffered buffer_provider->pushNewBufferedMessage(body_3, method, uri_3, category, message_metadata, false); + buffer_provider->pushNewBufferedMessage( + body_4, + method, + uri_4, + category, + message_metadata, + false + ); // shouldn't be buffered HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); Maybe err = genError(res); @@ -251,6 +270,7 @@ TEST_F(TestMessagingBuffer, testRoutinInMemory) EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_1, body_1, _, _)).WillOnce(Return(res)); EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_2, body_2, _, _)).WillOnce(Return(err)); EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_3, body_3, _, _)).WillOnce(Return(res)); + EXPECT_CALL(mock_messaging, sendSyncMessage(method, uri_4, body_4, _, _)).WillOnce(Return(err)); memory_routine(); diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index 5ad5df9..6009075 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -136,10 +136,13 @@ MessagingComp::sendAsyncMessage( const string &uri, const string &body, MessageCategory category, - const MessageMetadata &message_metadata + const MessageMetadata &message_metadata, + bool force_buffering ) { - i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); + MessageMetadata new_message_metadata = message_metadata; + new_message_metadata.setShouldBufferMessage(force_buffering); + i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, new_message_metadata, false); } Maybe diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc index ec6e8d4..155e90c 100644 --- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -15,6 +15,7 @@ #include "mocks/mock_messaging_connection.h" #include "rest.h" #include "rest_server.h" +#include "dummy_socket.h" using namespace std; using namespace testing; @@ -48,11 +49,13 @@ class TestMessagingComp : public testing::Test public: TestMessagingComp() { + Debug::setUnitTestFlag(D_MESSAGING, Debug::DebugLevel::TRACE); EXPECT_CALL(mock_time_get, getMonotonicTime()).WillRepeatedly(Return(chrono::microseconds(0))); ON_CALL(mock_agent_details, getFogDomain()).WillByDefault(Return(Maybe(fog_addr))); ON_CALL(mock_agent_details, getFogPort()).WillByDefault(Return(Maybe(fog_port))); messaging_comp.init(); + dummy_socket.init(); } void @@ -70,8 +73,8 @@ public: EXPECT_CALL(mock_proxy_conf, getProxyAuthentication(_)).WillRepeatedly(Return(string("cred"))); } - const string fog_addr = "1.2.3.4"; - uint16_t fog_port = 80; + const string fog_addr = "127.0.0.1"; + int fog_port = 8080; CPTestTempfile agent_details_file; MessagingComp messaging_comp; ::Environment env; @@ -82,6 +85,7 @@ public: NiceMock mock_time_get; NiceMock mock_agent_details; NiceMock mock_proxy_conf; + DummySocket dummy_socket; }; TEST_F(TestMessagingComp, testInitComp) @@ -100,16 +104,19 @@ TEST_F(TestMessagingComp, testSendSyncMessage) HTTPMethod method = HTTPMethod::POST; string uri = "/test-uri"; MessageCategory category = MessageCategory::GENERIC; - MessageMetadata message_metadata; MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); - Connection conn(conn_key, message_metadata); + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true); + Connection conn(conn_key, conn_metadata); + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) .WillOnce(Return(conn)); HTTPResponse res(HTTPStatusCode::HTTP_OK, "response!!"); EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); - auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, message_metadata); + auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata); ASSERT_TRUE(sending_res.ok()); HTTPResponse http_res = sending_res.unpack(); EXPECT_EQ(http_res.getBody(), "response!!"); @@ -126,7 +133,7 @@ TEST_F(TestMessagingComp, testSendAsyncMessage) MessageMetadata message_metadata; EXPECT_CALL(mock_messaging_buffer, pushNewBufferedMessage(body, method, uri, category, _, _)).Times(1); - messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata); + messaging_comp.sendAsyncMessage(method, uri, body, category, message_metadata, true); } TEST_F(TestMessagingComp, testSendSyncMessageOnSuspendedConn) @@ -163,16 +170,18 @@ TEST_F(TestMessagingComp, testUploadFile) setAgentDetails(); string uri = "/test-uri"; MessageCategory category = MessageCategory::GENERIC; - MessageMetadata message_metadata; MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); - Connection conn(conn_key, message_metadata); + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true); + Connection conn(conn_key, conn_metadata); EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) .WillOnce(Return(conn)); HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); - auto upload_res = messaging_comp.uploadFile(uri, path, category, message_metadata); + auto upload_res = messaging_comp.uploadFile(uri, path, category, conn_metadata); ASSERT_TRUE(upload_res.ok()); EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); } @@ -185,16 +194,18 @@ TEST_F(TestMessagingComp, testDownloadFile) string uri = "/test-uri"; HTTPMethod method = HTTPMethod::GET; MessageCategory category = MessageCategory::GENERIC; - MessageMetadata message_metadata; MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); - Connection conn(conn_key, message_metadata); + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true); + Connection conn(conn_key, conn_metadata); EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) .WillOnce(Return(conn)); HTTPResponse res(HTTPStatusCode::HTTP_OK, ""); EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); - auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, message_metadata); + auto upload_res = messaging_comp.downloadFile(method, uri, "/tmp/test.txt", category, conn_metadata); ASSERT_TRUE(upload_res.ok()); EXPECT_EQ(upload_res.unpack(), HTTPStatusCode::HTTP_OK); } diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index b2233a8..004953a 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -65,7 +65,8 @@ GenericMetric::init( const ReportIS::IssuingEngine &_issuing_engine, chrono::seconds _report_interval, bool _reset, - Audience _audience + Audience _audience, + bool _force_buffering ) { i_mainloop = Singleton::Consume::by(); @@ -76,6 +77,7 @@ GenericMetric::init( team = _team; issuing_engine = _issuing_engine; audience = _audience; + force_buffering = _force_buffering; i_mainloop->addRecurringRoutine( I_MainLoop::RoutineType::System, @@ -233,7 +235,9 @@ GenericMetric::sendLog(const LogRest &metric_client_rest) const HTTPMethod::POST, fog_metric_uri, metric_client_rest, - MessageCategory::METRIC + MessageCategory::METRIC, + MessageMetadata(), + force_buffering ); } diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index e66f156..c71c27c 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -215,6 +215,7 @@ TEST_F(MetricTest, basicMetricTest) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -283,6 +284,7 @@ TEST_F(MetricTest, basicMetricTest) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -353,6 +355,7 @@ TEST_F(MetricTest, basicMetricTest) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -433,6 +436,7 @@ TEST_F(MetricTest, printMetricsTest) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).Times(AnyNumber()); @@ -504,6 +508,7 @@ TEST_F(MetricTest, metricTestWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -571,6 +576,7 @@ TEST_F(MetricTest, metricTestWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -638,6 +644,7 @@ TEST_F(MetricTest, metricTestWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -727,6 +734,7 @@ TEST_F(MetricTest, generateReportWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -801,6 +809,7 @@ TEST_F(MetricTest, generateReportWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -869,6 +878,7 @@ TEST_F(MetricTest, generateReportWithReset) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); @@ -1018,6 +1028,7 @@ TEST_F(MetricTest, testMapMetric) "/api/v1/agents/events", _, MessageCategory::METRIC, + _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); routine(); diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index a751fc0..783b415 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -133,7 +133,7 @@ RestServer::Impl::init() auto is_primary = Singleton::Consume::by()->get("Is Rest primary routine"); id = mainloop->addFileRoutine( - I_MainLoop::RoutineType::Offline, + I_MainLoop::RoutineType::RealTime, fd, [&] () { this->startNewConnection(); }, "REST server listener", @@ -174,7 +174,7 @@ RestServer::Impl::startNewConnection() const RestConn conn(new_socket, mainloop, this); mainloop->addFileRoutine( - I_MainLoop::RoutineType::Offline, + I_MainLoop::RoutineType::RealTime, new_socket, [conn] () { conn.parseConn(); }, "REST server connection handler" diff --git a/nodes/orchestration/package/local-default-policy.yaml b/nodes/orchestration/package/local-default-policy.yaml index e963ebf..efc0223 100644 --- a/nodes/orchestration/package/local-default-policy.yaml +++ b/nodes/orchestration/package/local-default-policy.yaml @@ -52,7 +52,7 @@ log-triggers: url-path: false url-query: false log-destination: - cloud: true + cloud: false stdout: format: json diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 6e5adec..326cdcc 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -49,6 +49,9 @@ NO_COLOR='\033[0m' pidof_cmd="pidof -x" is_alpine_release= +CURL_ERRORS="Operation timed out +Connection refused" + var_last_policy_modification_time=0 ls -l /etc/ | grep release > /dev/null 2>&1 @@ -333,22 +336,41 @@ get_profile_setting() # Initials - gps curl_func() # Initials - cf { cf_port=$1 - cf_data=$2 + cf_uri=$2 + cf_data=$3 if [ -z "$cf_data" ]; then cf_data="{}" fi if [ "${remove_curl_ld_path}" = "true" ]; then - echo "$(LD_LIBRARY_PATH="" ${curl_cmd} -sS --noproxy "*" --header "Content-Type: application/json" --request POST --data "$cf_data" http://127.0.0.1:"$cf_port" 2>&1)" + cf_response=$(LD_LIBRARY_PATH="" ${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data "$cf_data" http://127.0.0.1:"$cf_port/$cf_uri" 2>&1) else - echo "$(${curl_cmd} -sS --noproxy "*" --header "Content-Type: application/json" --request POST --data "$cf_data" http://127.0.0.1:"$cf_port" 2>&1)" + cf_response=$(${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data "$cf_data" http://127.0.0.1:"$cf_port/$cf_uri" 2>&1) fi + echo "$cf_response" +} + +curl_to_orchestration() # Initials - cto +{ + cto_uri=$1 + cto_data=$2 + if [ -z "$cto_data" ]; then + cto_data="{}" + fi + + cto_orchestration_port=$(extract_api_port orchestration) + cto_response=$(curl_func ${cto_orchestration_port} ${cto_uri} "${cto_data}") + cto_curl_error=$(echo "$cto_response" | grep -F -e "$CURL_ERRORS") + if [ -n "$cto_curl_error" ]; then + cto_secondary_orchestration_port=$(extract_default_api_port secondary_orchestration) + cto_response=$(curl_func ${cto_secondary_orchestration_port} ${cto_uri} "${cto_data}") + fi + echo "$cto_response" } get_registered_services_ports() # Initails - grsp { - grsp_orchestration_port=$1 - grsp_ports_and_services=$(curl_func "${grsp_orchestration_port}"/show-all-service-ports) + grsp_ports_and_services=$(curl_to_orchestration "show-all-service-ports") if echo "$grsp_ports_and_services" | grep -q "Connection refused"; then echo "Failed to reach orchestration" >&2 echo "" @@ -387,7 +409,7 @@ extract_api_port() # Initials - eap return fi - for pair in $(get_registered_services_ports "$eap_orchestration_port" | tr "," " "); do + for pair in $(get_registered_services_ports | tr "," " "); do eap_service="$(echo "$pair" | cut -d ':' -f1)" if [ "$(is_requested_service "$eap_service_name" "$eap_service")" = true ]; then echo "$pair" | cut -d ':' -f2 @@ -616,7 +638,7 @@ run_update_gradual_policy() # Initials - rugp rugp_service_api_port=$(extract_api_port "$rugp_service_name") # Load gradual policy configuration - rugp_errors=$(curl_func "${rugp_service_api_port}/set-gradual-deployment-policy" "${rugp_data}") + rugp_errors=$(curl_func "${rugp_service_api_port}" "set-gradual-deployment-policy" "${rugp_data}") sleep 1 if [ -n "$(echo "$rugp_errors" | sed "s/$(printf '\r')//g")" ]; then echo "Failed to set gradual policy. Error: $rugp_errors" @@ -624,7 +646,7 @@ run_update_gradual_policy() # Initials - rugp fi if [ "$rugp_service_name" = "access-control" ]; then # Load policy to kernel - rugp_errors=$(curl_func "${rugp_service_api_port}"/set-gradual-policy-to-kernel) + rugp_errors=$(curl_func "${rugp_service_api_port}" "set-gradual-policy-to-kernel") if [ -n "$(echo "$rugp_errors" | sed "s/$(printf '\r')//g")" ]; then echo "Failed to set gradual policy. Error: $rugp_errors" else @@ -760,7 +782,12 @@ print_metrics() # Initials - pm pm_service_name=$1 pm_port=$2 - pm_errors=$(curl_func "${pm_port}"/show-metrics) + if [ $pm_service_name = "Orchestration" ]; then + pm_errors=$(curl_to_orchestration "show-metrics") + else + pm_errors=$(curl_func "${pm_port}" "show-metrics") + fi + if [ -n "$(echo "$pm_errors" | sed "s/$(printf '\r')//g")" ]; then return fi @@ -780,17 +807,15 @@ run_print_metrics() # Initials - rpm fi fi - rpm_orchestration_port=$(extract_default_api_port orchestration) - if [ -z "$rpm_service_name" ]; then - print_metrics "Orchestration" "$rpm_orchestration_port" - rpm_list=$(get_registered_services_ports "$rpm_orchestration_port" | tr "," " ") + print_metrics "Orchestration" + rpm_list=$(get_registered_services_ports | tr "," " ") for pair in ${rpm_list}; do rpm_service=$(echo "$pair" | cut -d ':' -f1) print_metrics "$rpm_service" "$(echo "$pair" | cut -d ':' -f2)" done elif [ "$rpm_service_name" = "orchestration" ]; then - print_metrics "Orchestration" "$rpm_orchestration_port" + print_metrics "Orchestration" else rpm_port=$(extract_api_port "$rpm_service_name") print_metrics "$rpm_service_name" "$rpm_port" @@ -801,7 +826,7 @@ run_health_check() # Initials - rhc { rhc_orchestration_port=$(extract_default_api_port orchestration) - rhc_errors=$(curl_func "${rhc_orchestration_port}"/show-health-check-on-demand) + rhc_errors=$(curl_to_orchestration "show-health-check-on-demand") if [ -n "$(echo "$rhc_errors" | sed "s/$(printf '\r')//g")" ]; then return fi @@ -863,6 +888,8 @@ print_single_service_status() # Initials - psss psss_maybe_version=$(LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$USR_LIB_PATH/cpnano"$LD_LIBRARY_PATH_ADD" $psss_service_full_path --version 2>&1) if echo "$psss_maybe_version" | grep -q "error"; then echo "Version: Temporarily unavailable" + format_colored_status_line "Status: Not Available" + echo "" return fi echo "$psss_maybe_version" @@ -959,11 +986,7 @@ run_status() # Initials - rs format_colored_status_line "Status: Not running" fi - if [ "${remove_curl_ld_path}" = "true" ]; then - rs_orch_status=$(LD_LIBRARY_PATH="" ${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data {} http://127.0.0.1:"$(extract_api_port 'orchestration')"/show-orchestration-status 2>&1) - else - rs_orch_status=$(${curl_cmd} -sS -m 1 --noproxy "*" --header "Content-Type: application/json" --request POST --data {} http://127.0.0.1:"$(extract_api_port 'orchestration')"/show-orchestration-status 2>&1) - fi + rs_orch_status=$(curl_to_orchestration "show-orchestration-status") if echo "$rs_orch_status" | grep -q "update status"; then rs_line_count=$(echo "$rs_orch_status" | grep -c '^') @@ -1065,17 +1088,12 @@ set_proxy() # Initials - sp exit 1 fi - if [ "${remove_curl_ld_path}" = "true" ]; then - sp_curl_output=$(LD_LIBRARY_PATH="" ${curl_cmd} -w "%{http_code}\n" -sS -m 60 --noproxy "*" --header "Content-Type: application/json" --request POST --data '{"proxy":"'"$sp_proxy"'"}' http://127.0.0.1:"$(extract_api_port 'orchestration')"/add-proxy) - else - sp_curl_output=$(${curl_cmd} -w "%{http_code}\n" -sS -m 60 --noproxy "*" --header "Content-Type: application/json" --request POST --data '{"proxy":"'"$sp_proxy"'"}' http://127.0.0.1:"$(extract_api_port 'orchestration')"/add-proxy) - fi - if echo "$sp_curl_output" | grep -q "200"; then - echo "Proxy successfully changed to $sp_proxy" - else - echo "Failed to set proxy: Error code ${sp_curl_output}" + sp_curl_output=$(curl_to_orchestration "add-proxy" '{"proxy":"'"$sp_proxy"'"}') + if [ -n "$(echo "$sp_curl_output" | sed "s/$(printf '\r')//g")" ]; then + echo "Failed to set proxy. Error: ${sp_curl_output}" exit 1 fi + echo "Proxy successfully changed to $sp_proxy" } run_display_single_service_settings() # Initials - rdsss @@ -1277,7 +1295,7 @@ run_ai() # Initials - ra ra_tenant_id=$(printf "%s" "$ra_agent_details" | grep "Tenant ID" | cut -d '"' -f4) ra_agent_id=$(printf "%s" "$ra_agent_details" | grep "Agent ID" | cut -d '"' -f4) else - ra_orch_status=$(curl_func "$(extract_api_port orchestration)"/show-orchestration-status) + ra_orch_status=$(curl_to_orchestration "show-orchestration-status") if ! echo "$ra_orch_status" | grep -q "update status"; then [ -f ${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestrations_status.json ] && ra_orch_status=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestration_status.json) fi @@ -1302,7 +1320,7 @@ run_ai() # Initials - ra exit 1 fi if [ "$ra_upload_to_fog" = "true" ]; then - ra_token_data="$(curl_func "$(extract_api_port orchestration)"/show-access-token)" + ra_token_data=$(curl_to_orchestration "show-access-token") ra_token_hex=$(echo "$ra_token_data" | grep "token" | cut -d '"' -f4 | base64 -d | od -t x1 -An) ra_token_hex_formatted=$(echo $ra_token_hex | tr -d ' ') ra_token="$(xor_decrypt "${ra_token_hex_formatted}")" @@ -1543,7 +1561,7 @@ set_mode() # set mode sed -i "s/$cp_nano_mode/$mode/" ${FILESYSTEM_PATH}/orchestration/cp-nano-orchestration.cfg - ret=$(curl_func "$(extract_api_port orchestration)"/set-orchestration-mode) + ret=$(curl_to_orchestration "set-orchestration-mode") if [ "$mode" = "online_mode" ]; then time_sleep=2 diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index a31d605..d582f20 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -307,7 +307,7 @@ while true; do echo "Log files path: ${LOG_FILE_PATH}" elif [ "$1" = "--arm64_trustbox" ] || [ "$1" = "--arm64_linaro" ] || [ "$1" = "--arm32_rpi" ] || [ "$1" = "--gaia" ] || [ "$1" = "--smb_mrv_v1" ] || [ "$1" = "--smb_sve_v2" ] || [ "$1" = "--smb_thx_v3" ] || [ "$1" = "--x86" ] || [ "$1" = "./orchestration_package.sh" ]; then shift - continue + continue elif [ "$1" = "--skip_registration" ]; then var_skip_registration=true elif echo "$1" | grep -q ${FORCE_CLEAN_FLAG}; then @@ -536,7 +536,7 @@ install_watchdog() cp_copy service/x86/ubuntu16/nano_agent.service /etc/systemd/system/nano_agent.service echo "ExecStart=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/nano_agent.service echo "ExecStartPost=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" >> /etc/systemd/system/nano_agent.service - echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/nano_agent.service + echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/nano_agent.service cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" @@ -673,7 +673,7 @@ upgrade_conf_if_needed() [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && . "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') - if ! [ -z "$previous_mode" ]; then + if ! [ -z "$previous_mode" ]; then var_orchestration_mode=${previous_mode} fi @@ -719,9 +719,7 @@ copy_orchestration_executable() if [ -f /ext/appsec/local_policy.yaml ]; then cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml" else - if [ ! -f ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml ]; then - cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml - fi + cp_copy local-default-policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml fi } @@ -729,9 +727,9 @@ copy_k8s_executable() { if [ "$IS_K8S_ENV" = "true" ]; then cp -f k8s-check-update-listener.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh - cp -f k8s-check-update-trigger.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh - chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh + chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-listener.sh + cp -f k8s-check-update-trigger.sh ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh + chmod +x ${FILESYSTEM_PATH}/${SERVICE_PATH}/k8s-check-update-trigger.sh fi } @@ -950,18 +948,18 @@ install_orchestration() echo "MAX_FILE_SIZE=${WATCHDOG_MAX_FILE_SIZE}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} echo "MAX_ROTATION=${WATCHDOG_MAX_ROTATIONS}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - if [ -n "${FILESYSTEM_PATH}" ]; then - echo "CP_ENV_FILESYSTEM=${FILESYSTEM_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${LOG_FILE_PATH}" ]; then - echo "CP_ENV_LOG_FILE=${LOG_FILE_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${USR_LIB_PATH}" ]; then - echo "CP_USR_LIB_PATH=${USR_LIB_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi - if [ -n "${INIT_D_PATH}" ]; then - echo "CP_INIT_D_PATH=${INIT_D_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} - fi + if [ -n "${FILESYSTEM_PATH}" ]; then + echo "CP_ENV_FILESYSTEM=${FILESYSTEM_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${LOG_FILE_PATH}" ]; then + echo "CP_ENV_LOG_FILE=${LOG_FILE_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${USR_LIB_PATH}" ]; then + echo "CP_USR_LIB_PATH=${USR_LIB_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi + if [ -n "${INIT_D_PATH}" ]; then + echo "CP_INIT_D_PATH=${INIT_D_PATH}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + fi fi if [ -z "${var_token}" ] && [ ${var_hybrid_mode} = false ] && [ ${var_offline_mode} = false ] && [ -z ${EGG_MODE} ] && [ ${var_no_otp} = false ]; then