From da20943c09499297c64798b7c9a498eab4600d54 Mon Sep 17 00:00:00 2001 From: Ned Wright Date: Fri, 8 Aug 2025 11:06:28 +0000 Subject: [PATCH] sync code --- .../nginx_attachment/nginx_attachment.cc | 26 +- .../nginx_attachment_opaque.cc | 6 + .../nginx_attachment/nginx_parser.cc | 10 +- .../gradual_deployment/gradual_deployment.cc | 8 +- components/http_manager/http_manager.cc | 14 +- .../http_manager/http_manager_opaque.cc | 16 + components/http_manager/http_manager_opaque.h | 4 + .../generic_rulebase/triggers_config.h | 32 +- .../include/http_event_impl/filter_verdict.h | 13 +- .../http_event_impl/i_http_event_impl.h | 15 + components/include/http_transaction_data.h | 10 +- components/include/i_details_resolver.h | 1 + components/include/i_waap_telemetry.h | 4 +- components/include/ips_comp.h | 4 +- components/include/telemetry.h | 15 + .../http_geo_filter/http_geo_filter.cc | 22 +- .../ips/include/ips_signatures.h | 9 + components/security_apps/ips/ips_comp.cc | 16 + .../security_apps/ips/ips_signatures.cc | 23 + .../security_apps/ips/ips_ut/component_ut.cc | 2 + .../security_apps/ips/ips_ut/entry_ut.cc | 2 + .../security_apps/ips/ips_ut/resource_ut.cc | 4 + .../security_apps/ips/ips_ut/signatures_ut.cc | 9 +- .../layer_7_access_control.cc | 6 +- .../include/local_policy_common.h | 8 + .../local_policy_mgmt_gen/k8s_policy_utils.cc | 13 +- .../details_resolver/details_resolver.cc | 26 +- .../checkpoint_product_handlers.h | 61 +- .../details_resolver_impl.h | 51 +- .../orchestration/downloader/https_client.cc | 9 +- .../include/mock/mock_details_resolver.h | 7 +- .../manifest_diff_calculator.cc | 4 +- .../orchestration/orchestration_comp.cc | 7 +- .../orchestration_tools.cc | 3 +- .../orchestration_tools_ut.cc | 2 +- .../orchestration_multitenant_ut.cc | 1 + .../orchestration_ut/orchestration_ut.cc | 1 + .../service_controller/service_controller.cc | 1 + .../update_communication/fog_authenticator.cc | 54 +- .../security_apps/rate_limit/rate_limit.cc | 16 +- .../security_apps/waap/include/i_serialize.h | 36 +- .../waap/include/i_transaction.h | 5 + .../security_apps/waap/include/i_waapConfig.h | 1 - .../waap_clib/AutonomousSecurityDecision.cc | 9 - .../waap_clib/AutonomousSecurityDecision.h | 3 - .../waap/waap_clib/ConfidenceCalculator.cc | 859 +++++++++++++++--- .../waap/waap_clib/ConfidenceCalculator.h | 82 +- .../security_apps/waap/waap_clib/Csrf.cc | 15 + .../waap/waap_clib/DecisionType.h | 33 + .../waap/waap_clib/DeepParser.cc | 35 +- .../waap_clib/IndicatorsFiltersManager.cc | 1 + .../waap/waap_clib/KeywordIndicatorFilter.cc | 3 +- .../waap/waap_clib/KeywordIndicatorFilter.h | 1 + .../waap/waap_clib/LogGenWrapper.cc | 11 +- .../waap/waap_clib/ParserBinaryFile.cc | 6 +- .../waap/waap_clib/ParserUrlEncode.cc | 15 +- .../waap/waap_clib/ParserUrlEncode.h | 4 +- .../security_apps/waap/waap_clib/ParserXML.cc | 23 +- .../waap/waap_clib/RequestsMonitor.cc | 16 +- .../waap/waap_clib/RequestsMonitor.h | 1 + .../waap/waap_clib/Serializator.cc | 335 +++++-- .../waap/waap_clib/Signatures.cc | 2 +- .../waap/waap_clib/SingleDecision.cc | 41 +- .../waap/waap_clib/SingleDecision.h | 9 + .../security_apps/waap/waap_clib/Telemetry.cc | 48 + .../waap/waap_clib/TuningDecisions.h | 6 +- .../waap/waap_clib/TypeIndicatorsFilter.cc | 2 + .../waap/waap_clib/TypeIndicatorsFilter.h | 1 + .../waap/waap_clib/WaapAssetState.cc | 175 +++- .../waap/waap_clib/WaapConfigApplication.cc | 1 - .../waap/waap_clib/WaapConfigBase.cc | 37 +- .../waap/waap_clib/WaapOverride.cc | 39 +- .../waap/waap_clib/WaapOverride.h | 71 +- .../waap_clib/WaapResponseInspectReasons.cc | 16 +- .../waap_clib/WaapResponseInspectReasons.h | 2 - .../waap/waap_clib/WaapTrigger.cc | 48 + .../waap/waap_clib/WaapTrigger.h | 74 +- .../waap/waap_clib/WaapValueStatsAnalyzer.cc | 1 + .../waap/waap_clib/Waf2Engine.cc | 530 +++++------ .../security_apps/waap/waap_clib/Waf2Engine.h | 42 +- .../waap/waap_clib/Waf2EngineGetters.cc | 296 +++++- .../security_apps/waap/waap_clib/Waf2Util.cc | 55 +- .../security_apps/waap/waap_clib/Waf2Util.h | 6 +- .../security_apps/waap/waap_component_impl.cc | 44 +- .../security_apps/waap/waap_component_impl.h | 3 +- .../generic_rulebase/generic_rulebase.cc | 1 + .../utils/generic_rulebase/match_query.cc | 4 +- .../generic_rulebase/parameters_config.cc | 7 +- .../utils/generic_rulebase/triggers_config.cc | 6 + .../http_transaction_data.cc | 13 + .../http_transaction_data_ut.cc | 2 +- components/utils/ip_utilities/ip_utilities.cc | 4 +- .../nginx_conf_collector/CMakeLists.txt | 3 +- .../nginx_conf_collector.cc | 87 +- core/CMakeLists.txt | 1 + .../agent_core_utilities.cc | 8 + .../agent_core_utilities_ut.cc | 46 + core/agent_details/agent_details.cc | 21 +- core/buffers/buffer.cc | 2 +- core/config/config.cc | 59 +- core/include/attachments/compression_utils.h | 3 +- .../attachments/nginx_attachment_common.h | 5 +- .../interfaces/i_intelligence_is_v2.h | 8 +- .../intelligence_is_v2/asset_reply.h | 27 + .../intelligence_response.h | 10 + .../intelligence_is_v2/query_request_v2.h | 10 +- .../interfaces/messaging/interface_impl.h | 8 +- .../interfaces/messaging/messaging_enums.h | 1 + .../interfaces/messaging/messaging_metadata.h | 41 +- .../interfaces/mock/mock_intelligence.h | 17 +- .../resources/config/config_loader.h | 2 +- .../services_sdk/resources/debug_flags.h | 7 + .../services_sdk/resources/generic_metric.h | 2 +- .../resources/intelligence_invalidation.h | 79 +- .../services_sdk/resources/log_generator.h | 10 - .../resources/report/report_bulks.h | 3 + .../resources/report/report_enums.h | 1 + .../utilities/agent_core_utilities.h | 1 + core/intelligence_is_v2/asset_replay.cc | 22 + .../intelligence_comp_v2.cc | 79 +- .../intelligence_comp_v2_ut.cc | 78 +- .../intelligence_query_v2_ut.cc | 22 + .../query_request_v2_ut.cc | 3 +- .../query_response_v2_ut.cc | 85 +- .../intelligence_response.cc | 14 + core/intelligence_is_v2/invalidation.cc | 140 ++- core/intelligence_is_v2/query_request_v2.cc | 22 +- core/logging/fog_stream.cc | 1 + core/logging/log_generator.cc | 10 +- core/logging/log_streams.h | 2 +- core/logging/logging.cc | 22 +- core/logging/syslog_stream.cc | 4 +- core/mainloop/mainloop.cc | 25 +- core/messaging/connection/connection.cc | 94 +- core/messaging/connection/connection_comp.cc | 9 +- .../connection_ut/connection_comp_ut.cc | 48 + .../messaging_comp/messaging_comp.cc | 26 +- core/metric/generic_metric.cc | 56 +- core/metric/metric_ut/metric_ut.cc | 9 +- core/report/tag_and_enum_management.cc | 6 +- core/rest/rest_server.cc | 189 ++-- .../orchestration/package/open-appsec-ctl.sh | 5 + .../package/orchestration_package.sh | 61 +- .../package/cp-nano-prometheus-conf.json | 42 +- .../cp-nano-prometheus-debug-conf.json | 22 +- 145 files changed, 4157 insertions(+), 1016 deletions(-) diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index 3223ffe..3410146 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -36,6 +36,7 @@ #include "nginx_attachment_config.h" #include "nginx_attachment_opaque.h" +#include "generic_rulebase/evaluators/trigger_eval.h" #include "nginx_parser.h" #include "i_instance_awareness.h" #include "common.h" @@ -130,6 +131,7 @@ class NginxAttachment::Impl Singleton::Provide::From { static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + static constexpr auto LIMIT_RESPONSE_HEADERS = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS; static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; static constexpr auto INJECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; @@ -1146,10 +1148,18 @@ private: handleCustomWebResponse( SharedMemoryIPC *ipc, vector &verdict_data, - vector &verdict_data_sizes) + vector &verdict_data_sizes, + string web_user_response_id) { ngx_http_cp_web_response_data_t web_response_data; - + ScopedContext ctx; + if (web_user_response_id != "") { + dbgTrace(D_NGINX_ATTACHMENT) + << "web user response ID registered in contex: " + << web_user_response_id; + set triggers_set{web_user_response_id}; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); + } WebTriggerConf web_trigger_conf = getConfigurationWithDefault( WebTriggerConf::default_trigger_conf, "rulebase", @@ -1271,7 +1281,7 @@ private: if (verdict.getVerdict() == DROP) { nginx_attachment_event.addTrafficVerdictCounter(nginxAttachmentEvent::trafficVerdict::DROP); verdict_to_send.modification_count = 1; - return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes); + return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, verdict.getWebUserResponseID()); } if (verdict.getVerdict() == ACCEPT) { @@ -1497,11 +1507,17 @@ private: opaque.activateContext(); FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data, opaque); - bool is_header = *chunked_data_type == ChunkType::REQUEST_HEADER || *chunked_data_type == ChunkType::RESPONSE_HEADER || *chunked_data_type == ChunkType::CONTENT_LENGTH; + + if (verdict.getVerdict() == LIMIT_RESPONSE_HEADERS) { + handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header); + popData(attachment_ipc); + verdict = FilterVerdict(INSPECT); + } + handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header); bool is_final_verdict = verdict.getVerdict() == ACCEPT || @@ -1614,6 +1630,8 @@ private: return "INJECT"; case INSPECT: return "INSPECT"; + case LIMIT_RESPONSE_HEADERS: + return "LIMIT_RESPONSE_HEADERS"; case IRRELEVANT: return "IRRELEVANT"; case RECONF: diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc index a13869f..8f6263c 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc @@ -70,6 +70,12 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da ctx.registerValue(HttpTransactionData::uri_query_decoded, decoded_url.substr(question_mark_location + 1)); } ctx.registerValue(HttpTransactionData::uri_path_decoded, decoded_url); + + // Register waf_tag from transaction data if available + const std::string& waf_tag = transaction_data.getWafTag(); + if (!waf_tag.empty()) { + ctx.registerValue(HttpTransactionData::waf_tag_ctx, waf_tag); + } } NginxAttachmentOpaque::~NginxAttachmentOpaque() diff --git a/components/attachment-intakers/nginx_attachment/nginx_parser.cc b/components/attachment-intakers/nginx_attachment/nginx_parser.cc index db33918..f3d7eb9 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_parser.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_parser.cc @@ -28,7 +28,6 @@ USE_DEBUG_FLAG(D_NGINX_ATTACHMENT_PARSER); Buffer NginxParser::tenant_header_key = Buffer(); static const Buffer proxy_ip_header_key("X-Forwarded-For", 15, Buffer::MemoryType::STATIC); -static const Buffer waf_tag_key("x-waf-tag", 9, Buffer::MemoryType::STATIC); static const Buffer source_ip("sourceip", 8, Buffer::MemoryType::STATIC); bool is_keep_alive_ctx = getenv("SAAS_KEEP_ALIVE_HDR_NAME") != nullptr; @@ -244,8 +243,6 @@ NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set opaque.setSessionTenantAndProfile(active_tenant_and_profile[0], active_tenant_and_profile[1]); } else if (proxy_ip_header_key == header_key) { source_identifiers.setXFFValuesToOpaqueCtx(header, UsersAllIdentifiersConfig::ExtractType::PROXYIP); - } else if (waf_tag_key == header_key) { - source_identifiers.setWafTagValuesToOpaqueCtx(header); } } @@ -382,12 +379,15 @@ NginxParser::parseResponseBody(const Buffer &raw_response_body, CompressionStrea Maybe NginxParser::parseContentEncoding(const vector &headers) { - static const Buffer content_encoding_header_key("Content-Encoding"); + dbgFlow(D_NGINX_ATTACHMENT_PARSER) << "Parsing \"Content-Encoding\" header"; + static const Buffer content_encoding_header_key("content-encoding"); auto it = find_if( headers.begin(), headers.end(), - [&] (const HttpHeader &http_header) { return http_header.getKey() == content_encoding_header_key; } + [&] (const HttpHeader &http_header) { + return http_header.getKey().isEqualLowerCase(content_encoding_header_key); + } ); if (it == headers.end()) { dbgTrace(D_NGINX_ATTACHMENT_PARSER) diff --git a/components/gradual_deployment/gradual_deployment.cc b/components/gradual_deployment/gradual_deployment.cc index ca0c4b1..0bdfab7 100644 --- a/components/gradual_deployment/gradual_deployment.cc +++ b/components/gradual_deployment/gradual_deployment.cc @@ -142,7 +142,7 @@ private: if (temp_params_list.size() == 1) { Maybe maybe_ip = IPAddr::createIPAddr(temp_params_list[0]); if (!maybe_ip.ok()) return genError("Could not create IP address, " + maybe_ip.getErr()); - IpAddress addr = move(ConvertToIpAddress(maybe_ip.unpackMove())); + IpAddress addr = ConvertToIpAddress(maybe_ip.unpackMove()); return move(IPRange{.start = addr, .end = addr}); } @@ -157,11 +157,11 @@ private: IPAddr max_addr = maybe_ip_max.unpackMove(); if (min_addr > max_addr) return genError("Could not create ip range - start greater then end"); - IpAddress addr_min = move(ConvertToIpAddress(move(min_addr))); - IpAddress addr_max = move(ConvertToIpAddress(move(max_addr))); + IpAddress addr_min = ConvertToIpAddress(move(min_addr)); + IpAddress addr_max = ConvertToIpAddress(move(max_addr)); if (addr_max.ip_type != addr_min.ip_type) return genError("Range IP's type does not match"); - return move(IPRange{.start = move(addr_min), .end = move(addr_max)}); + return IPRange{.start = move(addr_min), .end = move(addr_max)}; } return genError("Illegal range received: " + range); diff --git a/components/http_manager/http_manager.cc b/components/http_manager/http_manager.cc index 30ce762..9fb2fba 100755 --- a/components/http_manager/http_manager.cc +++ b/components/http_manager/http_manager.cc @@ -37,6 +37,7 @@ operator<<(ostream &os, const EventVerdict &event) { switch (event.getVerdict()) { case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: return os << "Inspect"; + case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers"; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: return os << "Accept"; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: return os << "Drop"; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: return os << "Inject"; @@ -93,13 +94,14 @@ public: ctx.registerValue(app_sec_marker_key, i_transaction_table->keyToString(), EnvKeyAttr::LogSection::MARKER); HttpManagerOpaque &state = i_transaction_table->getState(); - string event_key = static_cast(event.getKey()); - - if (event_key == getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging")) { + + const auto &custom_header = getProfileAgentSettingWithDefault("", "agent.customHeaderValueLogging"); + + if (event.getKey().isEqualLowerCase(custom_header)) { string event_value = static_cast(event.getValue()); dbgTrace(D_HTTP_MANAGER) << "Found header key and value - (" - << event_key + << custom_header << ": " << event_value << ") that matched agent settings"; @@ -195,7 +197,6 @@ public: if (state.getUserDefinedValue().ok()) { ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA); } - return handleEvent(EndRequestEvent().performNamedQuery()); } @@ -323,8 +324,9 @@ private: << respond.second.getVerdict(); state.setApplicationVerdict(respond.first, respond.second.getVerdict()); + state.setApplicationWebResponse(respond.first, respond.second.getWebUserResponseByPractice()); } - FilterVerdict aggregated_verdict = state.getCurrVerdict(); + FilterVerdict aggregated_verdict(state.getCurrVerdict(), state.getCurrWebUserResponse()); if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) { SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify(); } diff --git a/components/http_manager/http_manager_opaque.cc b/components/http_manager/http_manager_opaque.cc index eea3ed4..06e0719 100644 --- a/components/http_manager/http_manager_opaque.cc +++ b/components/http_manager/http_manager_opaque.cc @@ -32,6 +32,13 @@ HttpManagerOpaque::setApplicationVerdict(const string &app_name, ngx_http_cp_ver applications_verdicts[app_name] = verdict; } +void +HttpManagerOpaque::setApplicationWebResponse(const string &app_name, string web_user_response_id) +{ + dbgTrace(D_HTTP_MANAGER) << "Security app: " << app_name << ", has web user response: " << web_user_response_id; + applications_web_user_response[app_name] = web_user_response_id; +} + ngx_http_cp_verdict_e HttpManagerOpaque::getApplicationsVerdict(const string &app_name) const { @@ -51,8 +58,12 @@ HttpManagerOpaque::getCurrVerdict() const for (const auto &app_verdic_pair : applications_verdicts) { switch (app_verdic_pair.second) { case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: + dbgTrace(D_HTTP_MANAGER) << "Verdict DROP for app: " << app_verdic_pair.first; + current_web_user_response = applications_web_user_response.at(app_verdic_pair.first); + dbgTrace(D_HTTP_MANAGER) << "current_web_user_response=" << current_web_user_response; return app_verdic_pair.second; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: + // Sent in ResponseHeaders and ResponseBody. verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT; break; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: @@ -60,11 +71,16 @@ HttpManagerOpaque::getCurrVerdict() const break; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: break; + case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: + // Sent in End Request. + verdict = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS; + break; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT: dbgTrace(D_HTTP_MANAGER) << "Verdict 'Irrelevant' is not yet supported. Returning Accept"; accepted_apps++; break; case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: + // Sent in Request Headers and Request Body. verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; break; default: diff --git a/components/http_manager/http_manager_opaque.h b/components/http_manager/http_manager_opaque.h index f9a3226..53e733e 100644 --- a/components/http_manager/http_manager_opaque.h +++ b/components/http_manager/http_manager_opaque.h @@ -28,10 +28,12 @@ public: HttpManagerOpaque(); void setApplicationVerdict(const std::string &app_name, ngx_http_cp_verdict_e verdict); + void setApplicationWebResponse(const std::string &app_name, std::string web_user_response_id); ngx_http_cp_verdict_e getApplicationsVerdict(const std::string &app_name) const; void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; } ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; } ngx_http_cp_verdict_e getCurrVerdict() const; + const std::string & getCurrWebUserResponse() const { return current_web_user_response; }; std::set getCurrentDropVerdictCausers() const; void saveCurrentDataToCache(const Buffer &full_data); void setUserDefinedValue(const std::string &value) { user_defined_value = value; } @@ -52,6 +54,8 @@ public: private: std::unordered_map applications_verdicts; + std::unordered_map applications_web_user_response; + mutable std::string current_web_user_response; ngx_http_cp_verdict_e manager_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; Buffer prev_data_cache; uint aggregated_payload_size = 0; diff --git a/components/include/generic_rulebase/triggers_config.h b/components/include/generic_rulebase/triggers_config.h index 13b5b6d..4dfb8b2 100755 --- a/components/include/generic_rulebase/triggers_config.h +++ b/components/include/generic_rulebase/triggers_config.h @@ -317,12 +317,12 @@ public: { return url_for_cef; } + Flags getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const; + Flags getEnrechments(SecurityType security_type) const; private: ReportIS::Severity getSeverity(bool is_action_drop_or_prevent) const; ReportIS::Priority getPriority(bool is_action_drop_or_prevent) const; - Flags getStreams(SecurityType security_type, bool is_action_drop_or_prevent) const; - Flags getEnrechments(SecurityType security_type) const; std::string name; std::string verbosity; @@ -339,4 +339,32 @@ private: bool should_format_output = false; }; +class ReportTriggerConf +{ +public: + /// \brief Default constructor for ReportTriggerConf. + ReportTriggerConf() {} + + /// \brief Preload function to register expected configuration. + static void + preload() + { + registerExpectedConfiguration("rulebase", "report"); + } + + /// \brief Load function to deserialize configuration from JSONInputArchive. + /// \param archive_in The JSON input archive. + void load(cereal::JSONInputArchive &archive_in); + + /// \brief Get the name. + /// \return The name. + const std::string & + getName() const + { + return name; + } +private: + std::string name; +}; + #endif //__TRIGGERS_CONFIG_H__ diff --git a/components/include/http_event_impl/filter_verdict.h b/components/include/http_event_impl/filter_verdict.h index 174ffd0..af81c64 100755 --- a/components/include/http_event_impl/filter_verdict.h +++ b/components/include/http_event_impl/filter_verdict.h @@ -27,9 +27,18 @@ public: verdict(_verdict) {} + FilterVerdict( + ngx_http_cp_verdict_e _verdict, + const std::string &_web_reponse_id) + : + verdict(_verdict), + web_user_response_id(_web_reponse_id) + {} + FilterVerdict(const EventVerdict &_verdict, ModifiedChunkIndex _event_idx = -1) : - verdict(_verdict.getVerdict()) + verdict(_verdict.getVerdict()), + web_user_response_id(_verdict.getWebUserResponseByPractice()) { if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) { addModifications(_verdict.getModifications(), _event_idx); @@ -59,10 +68,12 @@ public: uint getModificationsAmount() const { return total_modifications; } ngx_http_cp_verdict_e getVerdict() const { return verdict; } const std::vector & getModifications() const { return modifications; } + const std::string getWebUserResponseID() const { return web_user_response_id; } private: ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; std::vector modifications; + std::string web_user_response_id; uint total_modifications = 0; }; diff --git a/components/include/http_event_impl/i_http_event_impl.h b/components/include/http_event_impl/i_http_event_impl.h index 6e8608e..729e306 100755 --- a/components/include/http_event_impl/i_http_event_impl.h +++ b/components/include/http_event_impl/i_http_event_impl.h @@ -376,16 +376,31 @@ public: verdict(event_verdict) {} + EventVerdict( + const ModificationList &mods, + ngx_http_cp_verdict_e event_verdict, + std::string response_id) : + modifications(mods), + verdict(event_verdict), + webUserResponseByPractice(response_id) + {} + // LCOV_EXCL_START - sync functions, can only be tested once the sync module exists template void serialize(T &ar, uint) { ar(verdict); } // LCOV_EXCL_STOP const ModificationList & getModifications() const { return modifications; } ngx_http_cp_verdict_e getVerdict() const { return verdict; } + const std::string getWebUserResponseByPractice() const { return webUserResponseByPractice; } + void setWebUserResponseByPractice(const std::string id) { + dbgTrace(D_HTTP_MANAGER) << "current verdict web user response set to: " << id; + webUserResponseByPractice = id; + } private: ModificationList modifications; ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + std::string webUserResponseByPractice; }; #endif // __I_HTTP_EVENT_IMPL_H__ diff --git a/components/include/http_transaction_data.h b/components/include/http_transaction_data.h index 9ccc9e1..9b6633a 100755 --- a/components/include/http_transaction_data.h +++ b/components/include/http_transaction_data.h @@ -72,7 +72,8 @@ public: parsed_uri, client_ip, client_port, - response_content_encoding + response_content_encoding, + waf_tag ); } @@ -91,7 +92,8 @@ public: parsed_uri, client_ip, client_port, - response_content_encoding + response_content_encoding, + waf_tag ); } // LCOV_EXCL_STOP @@ -122,6 +124,9 @@ public: response_content_encoding = _response_content_encoding; } + const std::string & getWafTag() const { return waf_tag; } + void setWafTag(const std::string &_waf_tag) { waf_tag = _waf_tag; } + static const std::string http_proto_ctx; static const std::string method_ctx; static const std::string host_name_ctx; @@ -154,6 +159,7 @@ private: uint16_t client_port; bool is_request; CompressionType response_content_encoding; + std::string waf_tag; }; #endif // __HTTP_TRANSACTION_DATA_H__ diff --git a/components/include/i_details_resolver.h b/components/include/i_details_resolver.h index ae3b0aa..c4df0b9 100644 --- a/components/include/i_details_resolver.h +++ b/components/include/i_details_resolver.h @@ -26,6 +26,7 @@ public: virtual Maybe getArch() = 0; virtual std::string getAgentVersion() = 0; virtual bool isKernelVersion3OrHigher() = 0; + virtual bool isGw() = 0; virtual bool isGwNotVsx() = 0; virtual bool isVersionAboveR8110() = 0; virtual bool isReverseProxy() = 0; diff --git a/components/include/i_waap_telemetry.h b/components/include/i_waap_telemetry.h index 6b3e7d3..2490094 100755 --- a/components/include/i_waap_telemetry.h +++ b/components/include/i_waap_telemetry.h @@ -27,6 +27,7 @@ struct DecisionTelemetryData int responseCode; uint64_t elapsedTime; std::set attackTypes; + bool temperatureDetected; DecisionTelemetryData() : blockType(NOT_BLOCKING), @@ -38,7 +39,8 @@ struct DecisionTelemetryData method(POST), responseCode(0), elapsedTime(0), - attackTypes() + attackTypes(), + temperatureDetected(false) { } }; diff --git a/components/include/ips_comp.h b/components/include/ips_comp.h index 599a5b2..996b863 100644 --- a/components/include/ips_comp.h +++ b/components/include/ips_comp.h @@ -4,6 +4,7 @@ #include "singleton.h" #include "i_keywords_rule.h" #include "i_table.h" +#include "i_mainloop.h" #include "i_http_manager.h" #include "i_environment.h" #include "http_inspection_events.h" @@ -16,7 +17,8 @@ class IPSComp Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: IPSComp(); diff --git a/components/include/telemetry.h b/components/include/telemetry.h index 234921a..66d89e5 100755 --- a/components/include/telemetry.h +++ b/components/include/telemetry.h @@ -76,6 +76,20 @@ private: std::unordered_set sources_seen; }; +class WaapAdditionalTrafficTelemetrics : public WaapTelemetryBase +{ +public: + void updateMetrics(const std::string &asset_id, const DecisionTelemetryData &data); + void initMetrics(); + +private: + MetricCalculations::Counter requests{this, "reservedNgenA"}; + MetricCalculations::Counter sources{this, "reservedNgenB"}; + MetricCalculations::Counter blocked{this, "reservedNgenC"}; + MetricCalculations::Counter temperature_count{this, "reservedNgenD"}; + std::unordered_set sources_seen; +}; + class WaapTrafficTelemetrics : public WaapTelemetryBase { public: @@ -124,6 +138,7 @@ private: std::map> traffic_telemetries; std::map> attack_types; std::map> attack_types_telemetries; + std::map> additional_traffic_telemetries; template void initializeTelemetryData( diff --git a/components/security_apps/http_geo_filter/http_geo_filter.cc b/components/security_apps/http_geo_filter/http_geo_filter.cc index 4cca044..fb0f026 100644 --- a/components/security_apps/http_geo_filter/http_geo_filter.cc +++ b/components/security_apps/http_geo_filter/http_geo_filter.cc @@ -96,6 +96,7 @@ public: if (ignore_source_ip){ dbgDebug(D_GEO_FILTER) << "Geo protection ignoring source ip: " << source_ip; } else { + dbgTrace(D_GEO_FILTER) << "Geo protection source ip: " << source_ip; ip_set.insert(convertIpAddrToString(maybe_source_ip.unpack())); } @@ -335,6 +336,14 @@ private: ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT; I_GeoLocation *i_geo_location = Singleton::Consume::by(); EnumArray geo_location_data; + auto env = Singleton::Consume::by(); + string source_id; + auto maybe_source_id = env->get(HttpTransactionData::source_identifier); + if (!maybe_source_id.ok()) { + dbgTrace(D_GEO_FILTER) << "failed to get source identifier from env"; + } else { + source_id = maybe_source_id.unpack(); + } for (const std::string& source : sources) { @@ -366,11 +375,15 @@ private: << country_code << ", country name: " << country_name - << ", source ip address: " - << source; + << ", ip address: " + << source + << ", source identifier: " + << source_id; + unordered_map> exception_value_country_code = { - {"countryCode", {country_code}} + {"countryCode", {country_code}}, + {"sourceIdentifier", {source_id}} }; auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, geo_location_data); if (matched_behavior_maybe.ok()) { @@ -382,7 +395,8 @@ private: } unordered_map> exception_value_country_name = { - {"countryName", {country_name}} + {"countryName", {country_name}}, + {"sourceIdentifier", {source_id}} }; matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, geo_location_data); if (matched_behavior_maybe.ok()) { diff --git a/components/security_apps/ips/include/ips_signatures.h b/components/security_apps/ips/include/ips_signatures.h index 9987246..0ecee73 100644 --- a/components/security_apps/ips/include/ips_signatures.h +++ b/components/security_apps/ips/include/ips_signatures.h @@ -29,6 +29,8 @@ #include "pm_hook.h" #include "i_generic_rulebase.h" +#define DEFAULT_IPS_YIELD_COUNT 500 + /// \namespace IPSSignatureSubTypes /// \brief Namespace containing subtypes for IPS signatures. namespace IPSSignatureSubTypes @@ -342,10 +344,17 @@ public: return is_loaded; } + static void + setYieldCounter(int new_yield_cnt) + { + yield_on_load_cnt = new_yield_cnt; + } + private: IPSSignatureMetaData metadata; std::shared_ptr rule; bool is_loaded; + static int yield_on_load_cnt; }; /// \class SignatureAndAction diff --git a/components/security_apps/ips/ips_comp.cc b/components/security_apps/ips/ips_comp.cc index 125c143..2f44a2f 100644 --- a/components/security_apps/ips/ips_comp.cc +++ b/components/security_apps/ips/ips_comp.cc @@ -98,6 +98,7 @@ public: registerListener(); table = Singleton::Consume::by(); env = Singleton::Consume::by(); + updateSigsYieldCount(); } void @@ -307,6 +308,20 @@ public: EventVerdict respond (const EndTransactionEvent &) override { return ACCEPT; } + void + updateSigsYieldCount() + { + const char *ips_yield_env_str = getenv("CPNANO_IPS_LOAD_YIELD_CNT"); + int ips_yield_default = DEFAULT_IPS_YIELD_COUNT; + if (ips_yield_env_str != nullptr) { + dbgDebug(D_IPS) << "CPNANO_IPS_LOAD_YIELD_CNT env variable is set to " << ips_yield_env_str; + ips_yield_default = atoi(ips_yield_env_str); + } + int yield_limit = getProfileAgentSettingWithDefault(ips_yield_default, "ips.sigsYieldCnt"); + dbgDebug(D_IPS) << "Setting IPS yield count to " << yield_limit; + IPSSignatureSubTypes::CompleteSignature::setYieldCounter(yield_limit); + } + private: static void setDrop(IPSEntry &state) { state.setDrop(); } static bool isDrop(const IPSEntry &state) { return state.isDrop(); } @@ -373,6 +388,7 @@ IPSComp::preload() registerExpectedConfigFile("ips", Config::ConfigFileType::Policy); registerExpectedConfigFile("ips", Config::ConfigFileType::Data); registerExpectedConfigFile("snort", Config::ConfigFileType::Policy); + registerConfigLoadCb([this]() { pimpl->updateSigsYieldCount(); }); ParameterException::preload(); diff --git a/components/security_apps/ips/ips_signatures.cc b/components/security_apps/ips/ips_signatures.cc index df211ea..b4f40bd 100644 --- a/components/security_apps/ips/ips_signatures.cc +++ b/components/security_apps/ips/ips_signatures.cc @@ -45,6 +45,8 @@ static const map levels = { { "Very Low", IPSLevel::VERY_LOW } }; +int CompleteSignature::yield_on_load_cnt = DEFAULT_IPS_YIELD_COUNT; + static IPSLevel getLevel(const string &level_string, const string &attr_name) { @@ -219,6 +221,18 @@ IPSSignatureMetaData::getYear() const void CompleteSignature::load(cereal::JSONInputArchive &ar) { + static int sigs_load_counter = 0; + static I_Environment *env = Singleton::Consume::by(); + static bool post_init = false; + + if (!post_init) { + auto routine_id = Singleton::Consume::by()->getCurrentRoutineId(); + if (routine_id.ok()) { + post_init = true; + dbgInfo(D_IPS) << "Loading signatures post init, enabling yield with limit " << yield_on_load_cnt; + } + } + try { ar(cereal::make_nvp("protectionMetadata", metadata)); RuleDetection rule_detection(metadata.getName()); @@ -229,6 +243,15 @@ CompleteSignature::load(cereal::JSONInputArchive &ar) is_loaded = false; dbgWarning(D_IPS) << "Failed to load signature: " << e.what(); } + + if (post_init && (yield_on_load_cnt > 0) && (++sigs_load_counter == yield_on_load_cnt)) { + sigs_load_counter = 0; + auto maybe_is_async = env->get("Is Async Config Load"); + if (maybe_is_async.ok() && *maybe_is_async == true) { + dbgTrace(D_IPS) << "Yielding after " << yield_on_load_cnt << " signatures"; + Singleton::Consume::by()->yield(false); + } + } } MatchType diff --git a/components/security_apps/ips/ips_ut/component_ut.cc b/components/security_apps/ips/ips_ut/component_ut.cc index 70f64bf..3df1e43 100644 --- a/components/security_apps/ips/ips_ut/component_ut.cc +++ b/components/security_apps/ips/ips_ut/component_ut.cc @@ -29,6 +29,8 @@ public: { comp.preload(); comp.init(); + auto err = genError("not coroutine"); + EXPECT_CALL(mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe(err))); } ~ComponentTest() diff --git a/components/security_apps/ips/ips_ut/entry_ut.cc b/components/security_apps/ips/ips_ut/entry_ut.cc index f25691f..3b007be 100644 --- a/components/security_apps/ips/ips_ut/entry_ut.cc +++ b/components/security_apps/ips/ips_ut/entry_ut.cc @@ -41,6 +41,8 @@ public: EntryTest() { ON_CALL(table, getState(_)).WillByDefault(Return(ptr)); + auto err = genError("not coroutine"); + EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe(err))); } void diff --git a/components/security_apps/ips/ips_ut/resource_ut.cc b/components/security_apps/ips/ips_ut/resource_ut.cc index 60b380b..b7eca8c 100644 --- a/components/security_apps/ips/ips_ut/resource_ut.cc +++ b/components/security_apps/ips/ips_ut/resource_ut.cc @@ -2,6 +2,7 @@ #include "cptest.h" #include "environment.h" #include "config_component.h" +#include "mock/mock_mainloop.h" using namespace std; using namespace testing; @@ -61,6 +62,9 @@ TEST(resources, basic_resource) { ConfigComponent conf; ::Environment env; + NiceMock mock_mainloop; + auto err = genError("not coroutine"); + EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe(err))); conf.preload(); diff --git a/components/security_apps/ips/ips_ut/signatures_ut.cc b/components/security_apps/ips/ips_ut/signatures_ut.cc index c913c7d..b52eaba 100644 --- a/components/security_apps/ips/ips_ut/signatures_ut.cc +++ b/components/security_apps/ips/ips_ut/signatures_ut.cc @@ -60,7 +60,12 @@ public: { IPSHelper::has_deobfuscation = true; generic_rulebase.preload(); + env.preload(); + env.init(); + EXPECT_CALL(logs, getCurrentLogId()).Times(AnyNumber()); + auto err = genError("not coroutine"); + EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(Maybe(err))); ON_CALL(table, getState(_)).WillByDefault(Return(&ips_state)); { stringstream ss; @@ -123,9 +128,6 @@ public: void loadExceptions() { - env.preload(); - env.init(); - BasicRuleConfig::preload(); registerExpectedConfiguration("rulebase", "exception"); @@ -195,6 +197,7 @@ public: void load(const IPSSignaturesResource &policy, const string &severity, const string &confidence) { + Singleton::Consume::from(env)->registerValue("Is Async Config Load", false); setResource(policy, "IPS", "protections"); stringstream ss; ss << "{"; diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control.cc b/components/security_apps/layer_7_access_control/layer_7_access_control.cc index c999429..b17057f 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control.cc @@ -131,8 +131,12 @@ public: EventVerdict respond(const WaitTransactionEvent &) override { - dbgFlow(D_L7_ACCESS_CONTROL) << "Handling wait verdict"; + if (!isAppEnabled()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled"; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + dbgTrace(D_L7_ACCESS_CONTROL) << "Handling wait verdict"; return handleEvent(); } diff --git a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h index 02c4e6b..484108d 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h +++ b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h @@ -170,6 +170,7 @@ public: ss.str(modified_json); try { cereal::JSONInputArchive in_ar(ss); + in_ar(cereal::make_nvp("apiVersion", api_version)); in_ar(cereal::make_nvp("spec", spec)); in_ar(cereal::make_nvp("metadata", meta_data)); } catch (cereal::Exception &e) { @@ -191,11 +192,18 @@ public: return meta_data; } + const std::string & + getApiVersion() const + { + return api_version; + } + const T & getSpec() const { return spec; } private: T spec; AppsecSpecParserMetaData meta_data; + std::string api_version; }; #endif // __LOCAL_POLICY_COMMON_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc index 404b65d..4ce76ff 100644 --- a/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc @@ -515,17 +515,6 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds( } // LCOV_EXCL_STOP -bool -doesVersionExist(const map &annotations, const string &version) -{ - for (auto annotation : annotations) { - if(annotation.second.find(version) != std::string::npos) { - return true; - } - } - return false; -} - std::tuple, Maybe> K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &ingress_mode) const { @@ -534,7 +523,7 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i ); if (!maybe_appsec_policy_spec.ok() || - !doesVersionExist(maybe_appsec_policy_spec.unpack().getMetaData().getAnnotations(), "v1beta1") + maybe_appsec_policy_spec.unpack().getApiVersion().find("v1beta1") == std::string::npos ) { try { std::string v1beta1_error = diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index ece8423..9fd0ca4 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -41,6 +41,7 @@ public: string getAgentVersion() override; bool isKernelVersion3OrHigher() override; + bool isGw() override; bool isGwNotVsx() override; bool isVersionAboveR8110() override; bool isReverseProxy() override; @@ -167,6 +168,19 @@ DetailsResolver::Impl::isKernelVersion3OrHigher() return false; } +bool +DetailsResolver::Impl::isGw() +{ +#if defined(gaia) || defined(smb) + static const string is_gw_cmd = "cpprod_util FwIsFirewallModule"; + auto is_gw = DetailsResolvingHanlder::getCommandOutput(is_gw_cmd); + if (is_gw.ok() && !is_gw.unpack().empty()) { + return is_gw.unpack().front() == '1'; + } +#endif + return false; +} + bool DetailsResolver::Impl::isGwNotVsx() { @@ -238,15 +252,21 @@ DetailsResolver::Impl::parseNginxMetadata() "orchestration", "Nginx metadata temp file" ); + + const string &filesystem_path_config = getFilesystemPathConfig(); + const string srcipt_exe_cmd = - getFilesystemPathConfig() + + filesystem_path_config + "/scripts/cp-nano-makefile-generator.sh -f -o " + output_path; const string script_fresh_exe_cmd = - getFilesystemPathConfig() + + filesystem_path_config + "/scripts/cp-nano-makefile-generator-fresh.sh save --save-location " + - output_path; + output_path + + " --strings_bin_path " + + filesystem_path_config + + "/bin/strings"; dbgTrace(D_ORCHESTRATOR) << "Details resolver, srcipt exe cmd: " << srcipt_exe_cmd; if (isNoResponse("which nginx") && isNoResponse("which kong")) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 2b4ebdf..0560360 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -26,9 +26,7 @@ Maybe checkSAMLSupportedBlade(const string &command_output) { - // uncomment when vpn will support SAML authentication - // string supportedBlades[3] = {"identityServer", "vpn", "cvpn"}; - string supportedBlades[1] = {"identityServer"}; + string supportedBlades[3] = {"identityServer", "vpn", "cvpn"}; for(const string &blade : supportedBlades) { if (command_output.find(blade) != string::npos) { return string("true"); @@ -49,6 +47,17 @@ checkIDABlade(const string &command_output) return string("false"); } +Maybe +checkVPNBlade(const string &command_output) +{ + string vpnBlade = "vpn"; + if (command_output.find(vpnBlade) != string::npos) { + return string("true"); + } + + return string("false"); +} + Maybe checkSAMLPortal(const string &command_output) { @@ -60,9 +69,9 @@ checkSAMLPortal(const string &command_output) } Maybe -checkPepIdaIdnStatus(const string &command_output) +checkInfinityIdentityEnabled(const string &command_output) { - if (command_output.find("nac_pep_identity_next_enabled = 1") != string::npos) { + if (command_output.find("get_identities_from_infinity_identity (true)") != string::npos) { return string("true"); } return string("false"); @@ -90,9 +99,6 @@ checkIDP(shared_ptr file_stream) { string line; while (getline(*file_stream, line)) { - if (line.find("") != string::npos) { - return string("false"); - } if (line.find(" file_stream) return string("false"); } +Maybe +checkVPNCIDP(shared_ptr file_stream) +{ + string line; + while (getline(*file_stream, line)) { + if (line.find("") != string::npos) { + break; + } + } + } + } + + return string("false"); +} + #endif // gaia #if defined(gaia) || defined(smb) @@ -140,6 +166,17 @@ getIsAiopsRunning(const string &command_output) return command_output; } +Maybe +getInterfaceMgmtIp(const string &command_output) +{ + if (!command_output.empty()) { + return command_output; + } + + return genError("Eth Management IP was not found"); +} + + Maybe checkHasSDWan(const string &command_output) { @@ -451,6 +488,14 @@ extractManagements(const string &command_output) json_output += "]"; return json_output; } + +Maybe +checkQosLegacyBlade(const string &command_output) +{ + if (command_output == "true" || command_output == "false") return command_output; + + return string("false"); +} #endif // gaia || smb #if defined(gaia) diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index d2edbb0..1a541a7 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -79,6 +79,14 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] " SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] " "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh", getOtlpAgentGaiaOsRole) +SHELL_CMD_HANDLER("ETH_MGMT_IP", + "FS_PATH=;" + "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");" + "[ -z \"${VS_ID}\" ] && " + "(eth=\"$(grep 'management:interface' /config/active | awk '{print $2}')\" &&" + " ip addr show \"${eth}\" | grep inet | awk '{print $2}' | cut -d '/' -f1) || " + "(ip a | grep UP | grep -v lo | head -n 1 | cut -d ':' -f2 | tr -d ' ')", + getInterfaceMgmtIp) #endif #if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1) SHELL_CMD_HANDLER("GLOBAL_QUID", @@ -89,6 +97,8 @@ SHELL_CMD_HANDLER("QUID", "cat $FWDIR/database/myown.C " "| awk -F'[()]' '/:name/ { found=1; next } found && /:uuid/ { uid=tolower($2); print uid; exit }'", getQUID) + + SHELL_CMD_HANDLER("SMO_QUID", "echo ''", getQUID) SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID) SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole) @@ -114,12 +124,6 @@ SHELL_CMD_HANDLER( "jq -r .lsm_profile_uuid /tmp/cpsdwan_getdata_orch.json", checkLsmProfileUuid ) -SHELL_CMD_HANDLER( - "IP Address", - "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" " - "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)", - getGWIPAddress -) SHELL_CMD_HANDLER( "Version", "cat /etc/cp-release | grep -oE 'R[0-9]+(\\.[0-9]+)?'", @@ -138,13 +142,22 @@ SHELL_CMD_HANDLER( "fw ctl get int support_fec |& grep -sq \"support_fec =\";echo $?", getFecApplicable ) +SHELL_CMD_HANDLER("is_legacy_qos_blade_enabled", + "cpprod_util CPPROD_GetValue FG1 ProdActive 1 | grep -q '^1$' " + "&& (cpprod_util CPPROD_GetValue FG1 FgSDWAN 1 | grep -q '^1$' && echo false || echo true) || " + "echo false", + checkQosLegacyBlade) #endif //gaia || smb #if defined(gaia) SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade) SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade) +SHELL_CMD_HANDLER("hasVPNBlade", "enabled_blades", checkVPNBlade) SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal) -SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_identity_next_enabled", checkPepIdaIdnStatus) +SHELL_CMD_HANDLER("hasInfinityIdentityEnabled", + "cat $FWDIR/database/myself_objects.C | grep get_identities_from_infinity_identity", + checkInfinityIdentityEnabled +) SHELL_CMD_HANDLER("requiredNanoServices", "echo ida", getRequiredNanoServices) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtObjectName", @@ -209,6 +222,14 @@ SHELL_CMD_HANDLER( "echo 1", extractManagements ) +SHELL_CMD_HANDLER( + "IP Address", + "( [ $(cpprod_util FwIsHighAvail) -eq 1 ] && [ $(cpprod_util FwIsVSX) -eq 1 ]" + "&& (jq -r .cluster_main_ip /tmp/cpsdwan_getdata_orch.json) )" + "|| ( [ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" )" + "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)", + getGWIPAddress +) #endif //gaia #if defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1) @@ -270,6 +291,17 @@ SHELL_CMD_HANDLER( "echo 1", extractManagements ) +SHELL_CMD_HANDLER( + "IP Address", + "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" " + "|| (jq -r .main_ip /tmp/cpsdwan_getdata_orch.json)", + getGWIPAddress +) +SHELL_CMD_HANDLER( + "Hardware", + R"(ver | sed -E 's/^This is Check Point'\''s +([^ ]+).*$/\1/')", + getHardware +) #endif//smb SHELL_CMD_OUTPUT("kernel_version", "uname -r") @@ -287,6 +319,11 @@ FILE_CONTENT_HANDLER( (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml", checkIDP ) +FILE_CONTENT_HANDLER( + "hasVPNCidpConfigured", + (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml", + checkVPNCIDP +) #endif //gaia #if defined(alpine) diff --git a/components/security_apps/orchestration/downloader/https_client.cc b/components/security_apps/orchestration/downloader/https_client.cc index b0441c3..5f4abfd 100755 --- a/components/security_apps/orchestration/downloader/https_client.cc +++ b/components/security_apps/orchestration/downloader/https_client.cc @@ -41,8 +41,13 @@ HTTPSClient::getFile(const URLParser &url, const string &out_file, bool auth_req if (!url.isOverSSL()) return genError("URL is not over SSL."); - if (getFileSSLDirect(url, out_file, token).ok()) return Maybe(); - dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly."; + bool skip_direct_download = (url.getQuery().find("/resources/") != string::npos); + if (skip_direct_download) { + dbgWarning(D_ORCHESTRATOR) << "Resources path: " << url.getQuery() << ". Skipping direct download."; + } else { + if (getFileSSLDirect(url, out_file, token).ok()) return Maybe(); + dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly."; + } if (getFileSSL(url, out_file, token).ok()) return Maybe(); dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL)."; diff --git a/components/security_apps/orchestration/include/mock/mock_details_resolver.h b/components/security_apps/orchestration/include/mock/mock_details_resolver.h index c6a1dd7..9c5ebd8 100644 --- a/components/security_apps/orchestration/include/mock/mock_details_resolver.h +++ b/components/security_apps/orchestration/include/mock/mock_details_resolver.h @@ -42,13 +42,14 @@ public: MOCK_METHOD0(getPlatform, Maybe()); MOCK_METHOD0(getArch, Maybe()); MOCK_METHOD0(getAgentVersion, std::string()); - MOCK_METHOD0(isCloudStorageEnabled, bool()); + MOCK_METHOD0(isCloudStorageEnabled, bool()); MOCK_METHOD0(isReverseProxy, bool()); MOCK_METHOD0(isKernelVersion3OrHigher, bool()); + MOCK_METHOD0(isGw, bool()); MOCK_METHOD0(isGwNotVsx, bool()); MOCK_METHOD0(getResolvedDetails, std::map()); - MOCK_METHOD0(isVersionAboveR8110, bool()); - MOCK_METHOD0(parseNginxMetadata, Maybe>()); + MOCK_METHOD0(isVersionAboveR8110, bool()); + MOCK_METHOD0(parseNginxMetadata, Maybe>()); MOCK_METHOD0( readCloudMetadata, Maybe>()); }; diff --git a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc index dd5b996..bcc13e6 100755 --- a/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc +++ b/components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc @@ -115,9 +115,9 @@ ManifestDiffCalculator::buildRecInstallationQueue( const map ¤t_packages, const map &new_packages) { - const vector &requires = package.getRequire(); + const vector &requires_packages = package.getRequire(); - for (const auto &require : requires) { + for (const auto &require : requires_packages) { auto installed_package = current_packages.find(require); auto new_package = new_packages.find(require); diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 6a8b536..79c3891 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -1471,7 +1471,8 @@ private: string cc_opt; tie(config_opt, cc_opt, nginx_version, nginx_signature) = nginx_data.unpack(); agent_data_report - << make_pair("attachmentVersion", "Legacy") + << make_pair("configureOptStatus", "Enabled") + << make_pair("moduleSignatureStatus", "Enabled") << make_pair("nginxSignature", nginx_signature) << make_pair("nginxVersion", nginx_version) << make_pair("configureOpt", config_opt) @@ -1496,6 +1497,10 @@ private: agent_data_report << AgentReportFieldWithLabel("isKernelVersion3OrHigher", "true"); } + if (i_details_resolver->isGw()) { + agent_data_report << AgentReportFieldWithLabel("isGw", "true"); + } + if (i_details_resolver->isGwNotVsx()) { agent_data_report << AgentReportFieldWithLabel("isGwNotVsx", "true"); } diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc index db58383..46bbaff 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc @@ -150,7 +150,8 @@ getNamespaceDataFromCluster() string auth_header = "Authorization: Bearer " + token; string connection_header = "Connection: close"; string host = "https://kubernetes.default.svc:443/api/v1/namespaces/"; - string culr_cmd = "curl -s -k -H \"" + auth_header + "\" -H \"" + connection_header + "\" " + host + + string culr_cmd = + "LD_LIBRARY_PATH=\"\" curl -s -k -H \"" + auth_header + "\" -H \"" + connection_header + "\" " + host + " | /etc/cp/bin/cpnano_json"; auto output_res = Singleton::Consume::by()->getExecOutput(culr_cmd); diff --git a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc index 430317f..ca7f46a 100755 --- a/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc +++ b/components/security_apps/orchestration/orchestration_tools/orchestration_tools_ut/orchestration_tools_ut.cc @@ -86,7 +86,7 @@ TEST_F(OrchestrationToolsTest, setClusterId) EXPECT_CALL( mock_shell_cmd, getExecOutput( - "curl -s -k -H \"Authorization: Bearer 123\" -H \"Connection: close\" " + "LD_LIBRARY_PATH=\"\" curl -s -k -H \"Authorization: Bearer 123\" -H \"Connection: close\" " "https://kubernetes.default.svc:443/api/v1/namespaces/ | /etc/cp/bin/cpnano_json", 200, false diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc index 62e47ed..d0a15e4 100644 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc @@ -145,6 +145,7 @@ public: EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64"))); EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isGw()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 3e83ed3..383315a 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -174,6 +174,7 @@ public: EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isCloudStorageEnabled()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false)); + EXPECT_CALL(mock_details_resolver, isGw()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false)); EXPECT_CALL(mock_details_resolver, parseNginxMetadata()).WillRepeatedly(Return(no_nginx)); diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index b18a8d8..c4719ef 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -209,6 +209,7 @@ ServiceDetails::sendNewConfigurations(int configuration_id, const string &policy new_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); new_config_req_md.setSuspension(false); + new_config_req_md.setShouldSendAccessToken(false); auto res = messaging->sendSyncMessage( HTTPMethod::POST, "/set-new-configuration", diff --git a/components/security_apps/orchestration/update_communication/fog_authenticator.cc b/components/security_apps/orchestration/update_communication/fog_authenticator.cc index c58c96f..b5ffdf1 100755 --- a/components/security_apps/orchestration/update_communication/fog_authenticator.cc +++ b/components/security_apps/orchestration/update_communication/fog_authenticator.cc @@ -139,6 +139,25 @@ FogAuthenticator::RegistrationData::serialize(JSONOutputArchive &out_ar) const ); } +static string +getDeplymentType() +{ + auto deplyment_type = Singleton::Consume::by()->getEnvType(); + switch (deplyment_type) { + case EnvType::LINUX: return "Embedded"; + case EnvType::DOCKER: return "Docker"; + case EnvType::NON_CRD_K8S: + case EnvType::K8S: return "K8S"; + case EnvType::COUNT: break; + } + + dbgAssertOpt(false) + << AlertInfo(AlertTeam::CORE, "fog communication") + << "Failed to get a legitimate deployment type: " + << static_cast(deplyment_type); + return "Embedded"; +} + Maybe FogAuthenticator::registerAgent( const FogAuthenticator::RegistrationData ®_data, @@ -208,6 +227,13 @@ FogAuthenticator::registerAgent( request << make_pair("userEdition", getUserEdition()); + if (getDeplymentType() == "Docker" || getDeplymentType() == "K8S") { + const char *image_version_otp = getenv("IMAGE_VERSION"); + if (image_version_otp) { + request << make_pair("imageVersion", image_version_otp); + } + } + if (details_resolver->isReverseProxy()) { request << make_pair("reverse_proxy", "true"); } @@ -220,6 +246,10 @@ FogAuthenticator::registerAgent( request << make_pair("isKernelVersion3OrHigher", "true"); } + if (details_resolver->isGw()) { + request << make_pair("isGw", "true"); + } + if (details_resolver->isGwNotVsx()) { request << make_pair("isGwNotVsx", "true"); } @@ -283,11 +313,14 @@ FogAuthenticator::getAccessToken(const UserCredentials &user_credentials) const static const string grant_type_string = "/oauth/token?grant_type=client_credentials"; TokenRequest request = TokenRequest(); - MessageMetadata request_token_md; + MessageMetadata request_token_md(true); request_token_md.insertHeader( "Authorization", buildBasicAuthHeader(user_credentials.getClientId(), user_credentials.getSharedSecret()) ); + dbgInfo(D_ORCHESTRATOR) + << "Sending request for access token. Trace: " + << (request_token_md.getTraceId().ok() ? request_token_md.getTraceId().unpack() : "No trace id"); auto request_token_status = Singleton::Consume::by()->sendSyncMessage( HTTPMethod::POST, grant_type_string, @@ -461,25 +494,6 @@ FogAuthenticator::getCredentialsFromFile() const return orchestration_tools->jsonStringToObject(encrypted_cred.unpack()); } -static string -getDeplymentType() -{ - auto deplyment_type = Singleton::Consume::by()->getEnvType(); - switch (deplyment_type) { - case EnvType::LINUX: return "Embedded"; - case EnvType::DOCKER: return "Docker"; - case EnvType::NON_CRD_K8S: - case EnvType::K8S: return "K8S"; - case EnvType::COUNT: break; - } - - dbgAssertOpt(false) - << AlertInfo(AlertTeam::CORE, "fog communication") - << "Failed to get a legitimate deployment type: " - << static_cast(deplyment_type); - return "Embedded"; -} - Maybe FogAuthenticator::getCredentials() { diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index 31d63a8..dc410cb 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -250,13 +250,14 @@ public: fetchReplicaCount() { string curl_cmd = - "curl -H \"Authorization: Bearer " + kubernetes_token + "\" " + base_curl_cmd + " -H \"Authorization: Bearer " + kubernetes_token + "\" " "https://kubernetes.default.svc.cluster.local/apis/apps/v1/namespaces/" + kubernetes_namespace + - "/deployments/${AGENT_DEPLOYMENT_NAME} -k -s | jq .status.replicas"; + "/deployments/${AGENT_DEPLOYMENT_NAME} -k -s | jq .status.replicas"; auto maybe_replicas = i_shell_cmd->getExecOutput(curl_cmd); if (maybe_replicas.ok()) { try { replicas = std::stoi(maybe_replicas.unpack()); + dbgTrace(D_RATE_LIMIT) << "replicas is set to " << replicas; } catch (const std::exception &e) { dbgWarning(D_RATE_LIMIT) << "error while converting replicas: " << e.what(); } @@ -706,7 +707,9 @@ public: i_shell_cmd = Singleton::Consume::by(); i_env_details = Singleton::Consume::by(); env_type = i_env_details->getEnvType(); - if (env_type == EnvType::K8S) { + const char *nexus_env = getenv("KUBERNETES_METADATA"); + if (nexus_env == nullptr) return; + if (env_type == EnvType::K8S && string(nexus_env) == "true") { kubernetes_token = i_env_details->getToken(); kubernetes_namespace = i_env_details->getNameSpace(); fetchReplicaCount(); @@ -742,6 +745,13 @@ private: EnvType env_type; string kubernetes_namespace = ""; string kubernetes_token = ""; +#if defined(gaia) + const string base_curl_cmd = "curl_cli"; +#elif defined(alpine) + const string base_curl_cmd = "LD_LIBRARY_PATH=/usr/lib/:/usr/lib/cpnano curl"; +#else + const string base_curl_cmd = "curl"; +#endif }; RateLimit::RateLimit() : Component("RateLimit"), pimpl(make_unique()) {} diff --git a/components/security_apps/waap/include/i_serialize.h b/components/security_apps/waap/include/i_serialize.h index 31b7d8e..9d19353 100755 --- a/components/security_apps/waap/include/i_serialize.h +++ b/components/security_apps/waap/include/i_serialize.h @@ -23,7 +23,7 @@ static const uint max_send_obj_retries = 3; static const std::chrono::microseconds wait_next_attempt(5000000); -USE_DEBUG_FLAG(D_WAAP); +USE_DEBUG_FLAG(D_WAAP_SERIALIZE); class RestGetFile : public ClientRest { @@ -151,13 +151,14 @@ protected: I_Messaging *messaging = Singleton::Consume::by(); I_AgentDetails *agentDetails = Singleton::Consume::by(); if (agentDetails->getOrchestrationMode() == OrchestrationMode::OFFLINE) { - dbgDebug(D_WAAP) << "offline mode not sending object"; + dbgDebug(D_WAAP_SERIALIZE) << "offline mode not sending object"; return false; } if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); auto req_status = messaging->sendSyncMessage( method, uri, @@ -166,19 +167,22 @@ protected: req_md ); if (!req_status.ok()) { - dbgWarning(D_WAAP) << "failed to send request to uri: " << uri + dbgWarning(D_WAAP_SERIALIZE) << "failed to send request to uri: " << uri << ", error: " << req_status.getErr().toString(); } return req_status.ok(); } + MessageMetadata req_md; + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN); auto req_status = messaging->sendSyncMessage( method, uri, obj, - MessageCategory::GENERIC + MessageCategory::GENERIC, + req_md ); if (!req_status.ok()) { - dbgWarning(D_WAAP) << "failed to send request to uri: " << uri + dbgWarning(D_WAAP_SERIALIZE) << "failed to send request to uri: " << uri << ", error: " << req_status.getErr().toString(); } return req_status.ok(); @@ -192,14 +196,14 @@ protected: { if (sendObject(obj, method, uri)) { - dbgTrace(D_WAAP) << + dbgTrace(D_WAAP_SERIALIZE) << "object sent successfully after " << i << " retry attempts"; return true; } - dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i; + dbgInfo(D_WAAP_SERIALIZE) << "Failed to send object. Attempt: " << i; mainloop->yield(wait_next_attempt); } - dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << + dbgWarning(D_WAAP_SERIALIZE) << "Failed to send object to " << uri << ", reached maximum attempts: " << max_send_obj_retries; return false; } @@ -210,13 +214,14 @@ protected: I_Messaging *messaging = Singleton::Consume::by(); I_AgentDetails *agentDetails = Singleton::Consume::by(); if (agentDetails->getOrchestrationMode() == OrchestrationMode::OFFLINE) { - dbgDebug(D_WAAP) << "offline mode not sending object"; + dbgDebug(D_WAAP_SERIALIZE) << "offline mode not sending object"; return false; } if (agentDetails->getOrchestrationMode() == OrchestrationMode::HYBRID) { MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); return messaging->sendSyncMessageWithoutResponse( method, uri, @@ -225,11 +230,14 @@ protected: req_md ); } + MessageMetadata req_md; + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN); return messaging->sendSyncMessageWithoutResponse( method, uri, obj, - MessageCategory::GENERIC + MessageCategory::GENERIC, + req_md ); } @@ -241,14 +249,14 @@ protected: { if (sendNoReplyObject(obj, method, uri)) { - dbgTrace(D_WAAP) << + dbgTrace(D_WAAP_SERIALIZE) << "object sent successfully after " << i << " retry attempts"; return true; } - dbgInfo(D_WAAP) << "Failed to send object. Attempt: " << i; + dbgInfo(D_WAAP_SERIALIZE) << "Failed to send object. Attempt: " << i; mainloop->yield(wait_next_attempt); } - dbgWarning(D_WAAP) << "Failed to send object to " << uri << ", reached maximum attempts: " << + dbgWarning(D_WAAP_SERIALIZE) << "Failed to send object to " << uri << ", reached maximum attempts: " << max_send_obj_retries; return false; } @@ -257,6 +265,7 @@ protected: std::chrono::seconds m_interval; std::string m_owner; const std::string m_assetId; + bool m_remoteSyncEnabled; private: bool localSyncAndProcess(); @@ -272,7 +281,6 @@ private: size_t m_daysCount; size_t m_windowsCount; size_t m_intervalsCounter; - bool m_remoteSyncEnabled; const bool m_isAssetIdUuid; std::string m_type; std::string m_lastProcessedModified; diff --git a/components/security_apps/waap/include/i_transaction.h b/components/security_apps/waap/include/i_transaction.h index a2f3da8..b1bb897 100755 --- a/components/security_apps/waap/include/i_transaction.h +++ b/components/security_apps/waap/include/i_transaction.h @@ -84,6 +84,7 @@ public: virtual const std::string getUri() const = 0; virtual const std::string getUriStr() const = 0; virtual const std::string& getSourceIdentifier() const = 0; + virtual const std::string getCurrentWebUserResponse() = 0; virtual double getScore() const = 0; virtual double getOtherModelScore() const = 0; virtual const std::vector getScoreArray() const = 0; @@ -130,6 +131,7 @@ public: virtual void add_request_body_chunk(const char* data, int data_len) = 0; virtual void end_request_body() = 0; virtual void end_request() = 0; + virtual bool shouldLimitResponseHeadersInspection() = 0; // Response virtual void start_response(int response_status, int http_version) = 0; virtual void start_response_hdrs() = 0; @@ -145,4 +147,7 @@ public: virtual ReportIS::Severity computeEventSeverityFromDecision() const = 0; virtual void finish() = 0; virtual Waf2TransactionFlags &getTransactionFlags() = 0; + + virtual void setTemperatureDetected(bool detected) = 0; + virtual bool wasTemperatureDetected() const = 0; }; diff --git a/components/security_apps/waap/include/i_waapConfig.h b/components/security_apps/waap/include/i_waapConfig.h index f44ac23..5acbe9e 100755 --- a/components/security_apps/waap/include/i_waapConfig.h +++ b/components/security_apps/waap/include/i_waapConfig.h @@ -26,7 +26,6 @@ #include "../waap_clib/SecurityHeadersPolicy.h" #include - enum class BlockingLevel { NO_BLOCKING = 0, LOW_BLOCKING_LEVEL, diff --git a/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.cc b/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.cc index 47cd9a1..d510fa6 100755 --- a/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.cc +++ b/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.cc @@ -19,7 +19,6 @@ AutonomousSecurityDecision::AutonomousSecurityDecision(DecisionType type) : m_fpMitigationScore(0.0f), m_finalScore(0.0f), m_threatLevel(NO_THREAT), - m_overridesLog(false), m_relativeReputationMean(0.0), m_variance(0.0) {} @@ -52,10 +51,6 @@ void AutonomousSecurityDecision::setThreatLevel(ThreatLevel threatLevel) m_threatLevel = threatLevel; } -void AutonomousSecurityDecision::setOverridesLog(bool overridesLog) -{ - m_overridesLog = overridesLog; -} void AutonomousSecurityDecision::setRelativeReputationMean(double relativeReputationMean) { m_relativeReputationMean = relativeReputationMean; @@ -80,10 +75,6 @@ ThreatLevel AutonomousSecurityDecision::getThreatLevel() const { return m_threatLevel; } -bool AutonomousSecurityDecision::getOverridesLog() const -{ - return m_overridesLog; -} double AutonomousSecurityDecision::getRelativeReputationMean() const { return m_relativeReputationMean; diff --git a/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.h b/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.h index 7d2f5be..6ea23ba 100755 --- a/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.h +++ b/components/security_apps/waap/waap_clib/AutonomousSecurityDecision.h @@ -30,14 +30,12 @@ public: void setFpMitigationScore(double fpMitigationScore); void setFinalScore(double finalScore); void setThreatLevel(ThreatLevel threatLevel); - void setOverridesLog(bool overridesLog); void setRelativeReputationMean(double relativeReputationMean); void setVariance(double variance); double getRelativeReputation() const; double getFpMitigationScore() const; double getFinalScore() const; ThreatLevel getThreatLevel() const; - bool getOverridesLog() const; double getRelativeReputationMean() const; double getVariance() const; @@ -46,7 +44,6 @@ private: double m_fpMitigationScore; double m_finalScore; ThreatLevel m_threatLevel; - bool m_overridesLog; double m_relativeReputationMean; double m_variance; }; diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc index 78fa7a2..e90cc7f 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.cc @@ -13,33 +13,39 @@ #include "ConfidenceCalculator.h" #include +#include #include "waap.h" #include "ConfidenceFile.h" #include "i_agent_details.h" #include "i_mainloop.h" +#include #include +#include // For DIR, opendir, readdir, closedir +#include // For isdigit + +using namespace std; USE_DEBUG_FLAG(D_WAAP); -#define SYNC_WAIT_TIME std::chrono::seconds(300) // 5 minutes in seconds +#define SYNC_WAIT_TIME chrono::seconds(300) // 5 minutes in seconds #define SCORE_THRESHOLD 100.0 -#define BUSY_WAIT_TIME std::chrono::microseconds(100000) // 0.1 seconds +#define BUSY_WAIT_TIME chrono::microseconds(100000) // 0.1 seconds #define WAIT_LIMIT 10 #define BENIGN_PARAM_FACTOR 2 double logn(double x, double n) { - return std::log(x) / std::log(n); + return log(x) / log(n); } ConfidenceCalculator::ConfidenceCalculator(size_t minSources, size_t minIntervals, - std::chrono::minutes intervalDuration, + chrono::minutes intervalDuration, double ratioThreshold, const Val &nullObj, - const std::string &backupPath, - const std::string &remotePath, - const std::string &assetId, + const string &backupPath, + const string &remotePath, + const string &assetId, TuningDecision* tuning, I_IgnoreSources* ignoreSrc) : SerializeToLocalAndRemoteSyncBase(intervalDuration, @@ -48,36 +54,66 @@ ConfidenceCalculator::ConfidenceCalculator(size_t minSources, (remotePath == "") ? remotePath : remotePath + "/Confidence", assetId, "ConfidenceCalculator"), - m_params({ minSources, minIntervals, intervalDuration, ratioThreshold, true }), + m_params({ minSources, minIntervals, intervalDuration, ratioThreshold, true, defaultConfidenceMemUsage}), m_null_obj(nullObj), - m_time_window_logger(), + m_time_window_logger(make_shared()), + m_time_window_logger_backup(nullptr), m_confident_sets(), m_confidence_level(), m_last_indicators_update(0), + m_latest_index(0), m_ignoreSources(ignoreSrc), - m_tuning(tuning) + m_tuning(tuning), + m_estimated_memory_usage(0), + m_mainLoop(Singleton::Consume::by()), + m_routineId(0), + m_filesToRemove() { restore(); + + // Start asynchronous deletion of existing carry-on data files + garbageCollector(); } ConfidenceCalculator::~ConfidenceCalculator() { - m_time_window_logger.clear(); + m_time_window_logger->clear(); + m_time_window_logger.reset(); m_confident_sets.clear(); + if (!m_path_to_backup.empty()) { + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; + } + if (m_time_window_logger_backup) { + m_time_window_logger_backup->clear(); + m_time_window_logger_backup.reset(); + } } void ConfidenceCalculator::hardReset() { - m_time_window_logger.clear(); + if (m_time_window_logger) { + m_time_window_logger->clear(); + m_time_window_logger = make_shared(); + } + if (m_time_window_logger_backup) { + m_time_window_logger_backup->clear(); + m_time_window_logger_backup.reset(); + } + m_estimated_memory_usage = 0; m_confidence_level.clear(); m_confident_sets.clear(); - std::remove(m_filePath.c_str()); + remove(m_filePath.c_str()); } void ConfidenceCalculator::reset() { - m_time_window_logger.clear(); + if (m_time_window_logger) { + m_time_window_logger->clear(); + m_time_window_logger = make_shared(); + } + m_estimated_memory_usage = 0; if (!m_params.learnPermanently) { hardReset(); @@ -107,6 +143,12 @@ public: { } + ~WindowLogPost() + { + window_logger.get().clear(); + window_logger.get().rehash(0); + } + private: C2S_PARAM(ConfidenceCalculator::KeyValSourcesLogger, window_logger) }; @@ -127,15 +169,275 @@ private: S2C_PARAM(ConfidenceCalculator::KeyValSourcesLogger, window_logger) }; + +// Function to handle compression +void compressDataWrapper(const string& uncompressed_data, size_t chunk_size, vector& compressed_data) { + auto compression_stream = initCompressionStream(); + size_t offset = 0; + + while (offset < uncompressed_data.size()) { + size_t current_chunk_size = std::min(chunk_size, uncompressed_data.size() - offset); + bool is_last = (offset + current_chunk_size >= uncompressed_data.size()); + CompressionResult chunk_res = compressData( + compression_stream, + CompressionType::GZIP, + static_cast(current_chunk_size), + reinterpret_cast(uncompressed_data.c_str() + offset), + is_last ? 1 : 0 + ); + + if (!chunk_res.ok) { + finiCompressionStream(compression_stream); + throw runtime_error("Compression failed"); + } + + if (chunk_res.output && chunk_res.num_output_bytes > 0) { + compressed_data.insert( + compressed_data.end(), + chunk_res.output, + chunk_res.output + chunk_res.num_output_bytes + ); + free(chunk_res.output); + } + + offset += current_chunk_size; + } + + finiCompressionStream(compression_stream); +} + + +Maybe ConfidenceCalculator::writeToFile(const string& path, const vector& data) +{ + ofstream file(path, ios::binary); + if (!file.is_open()) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << path + << ", errno: " << errno << ", strerror: " << strerror(errno); + return genError("Failed to open file"); + } + + // Write compressed data to file in chunks to avoid large memory usage + const uint CHUNK_SIZE = getProfileAgentSettingWithDefault( + 64 * 1024, // 64 KiB + "appsecLearningSettings.writeChunkSize" + ); + size_t offset = 0; + while (offset < data.size()) { + size_t current_chunk_size = min(static_cast(CHUNK_SIZE), data.size() - offset); + file.write(reinterpret_cast(data.data()) + offset, current_chunk_size); + offset += current_chunk_size; + m_mainLoop->yield(false); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Write progress: " << offset << "/" << data.size() + << " bytes (" << (offset * 100 / data.size()) << "%) - yielded"; + } + file.close(); + return Maybe(); +} + +void ConfidenceCalculator::saveTimeWindowLogger() +{ + if (m_path_to_backup != "") // remove old file from exceed memory cap flow + { + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; + m_mainLoop->yield(false); + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Yielded after removing old backup file"; + } + // create temp file with random name + char temp_filename[] = "/tmp/waap_confidence_XXXXXX.gz"; + int fd = mkstemps(temp_filename, 3); + if (fd == -1) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to create temporary file. errono: " << errno; + m_time_window_logger_backup = m_time_window_logger; + return; + } + close(fd); + m_mainLoop->yield(false); + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Yielded after creating temp file: " << temp_filename; + + m_path_to_backup = temp_filename; + + stringstream ss; + { + cereal::JSONOutputArchive archive(ss); + archive(cereal::make_nvp("logger", *m_time_window_logger)); + } + + m_mainLoop->yield(false); + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "JSON serialized, size: " << ss.str().size() << " bytes"; + + string data = ss.str(); + + const uint COMPRESSED_CHUNK_SIZE = getProfileAgentSettingWithDefault( + 16 * 1024, // 16KB + "appsecLearningSettings.compressionChunkSize" + ); + auto compression_stream = initCompressionStream(); + size_t offset = 0; + vector compressed_data; + bool ok = true; + + while (offset < data.size()) { + size_t chunk_size = min(static_cast(COMPRESSED_CHUNK_SIZE), data.size() - offset); + bool is_last = (offset + chunk_size >= data.size()); + CompressionResult chunk_res = compressData( + compression_stream, + CompressionType::GZIP, + static_cast(chunk_size), + reinterpret_cast(data.c_str() + offset), + is_last ? 1 : 0 + ); + if (!chunk_res.ok) { + ok = false; + break; + } + if (chunk_res.output && chunk_res.num_output_bytes > 0) { + compressed_data.insert( + compressed_data.end(), + chunk_res.output, + chunk_res.output + chunk_res.num_output_bytes + ); + free(chunk_res.output); + } + offset += chunk_size; + m_mainLoop->yield(false); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Compression progress: " << offset << "/" << data.size() + << " bytes processed (" << (offset * 100 / data.size()) << "%) - yielded"; + } + finiCompressionStream(compression_stream); + + if (!ok) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress data"; + m_time_window_logger_backup = m_time_window_logger; + return; + } + + auto maybeError = writeToFile(m_path_to_backup, compressed_data); + if (!maybeError.ok()) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to write the backup file: " << m_path_to_backup; + m_time_window_logger_backup = m_time_window_logger; + m_path_to_backup = ""; + } else { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Finished writing the backup file: " << m_path_to_backup; + } +} + +shared_ptr ConfidenceCalculator::loadTimeWindowLogger() +{ + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Loading the time window logger from: " << m_path_to_backup; + if (m_path_to_backup.empty()) { + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "No backup file path set, cannot load logger"; + return nullptr; + } + + ifstream file(m_path_to_backup, ios::binary); + if (!file.is_open()) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << m_path_to_backup + << ", errno: " << errno << ", strerror: " << strerror(errno); + return nullptr; + } + + stringstream buffer; + // Read the file in chunks to avoid large memory usage + const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault( + 16 * 1024, + "appsecLearningSettings.readChunkSize"); // 16 KiB + vector chunk(READ_CHUNK_SIZE); + size_t chunk_size = static_cast(READ_CHUNK_SIZE); + size_t total_bytes_read = 0; + size_t chunks_read = 0; + + while (file.peek() != EOF) { + file.read(chunk.data(), chunk_size); + streamsize bytesRead = file.gcount(); + if (bytesRead > 0) { + buffer.write(chunk.data(), bytesRead); + total_bytes_read += bytesRead; + chunks_read++; + } + m_mainLoop->yield(false); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Read chunk " << chunks_read + << " (" << total_bytes_read << " bytes total) - yielded"; + } + file.close(); + + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; + + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Completed reading " << total_bytes_read + << " bytes in " << chunks_read << " chunks"; + + string compressed_data = buffer.str(); + auto compression_stream = initCompressionStream(); + + chunk_size = static_cast( + getProfileAgentSettingWithDefault( + 32 * 1024, // 32KiB + "appsecLearningSettings.compressionChunkSize" + ) + ); + size_t offset = 0; + string decompressed_data; + + while (offset < compressed_data.size()) { + size_t current_chunk_size = min(chunk_size, compressed_data.size() - offset); + DecompressionResult res = decompressData( + compression_stream, + current_chunk_size, + reinterpret_cast(compressed_data.c_str() + offset) + ); + + if (!res.ok) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress data from file: " << m_path_to_backup; + finiCompressionStream(compression_stream); + return nullptr; + } + + decompressed_data.append(reinterpret_cast(res.output), res.num_output_bytes); + free(res.output); + res.output = nullptr; + res.num_output_bytes = 0; + + offset += current_chunk_size; + + // Yield control after processing each chunk + m_mainLoop->yield(false); + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompression progress: " << offset << "/" << compressed_data.size() + << " bytes (" << (offset * 100 / compressed_data.size()) << "%) - yielded"; + } + + finiCompressionStream(compression_stream); + + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompressed data size: " << decompressed_data.size() + << " bytes (compression ratio: " << (float)decompressed_data.size() / compressed_data.size() << "x)"; + + stringstream decompressed_stream(decompressed_data); + auto window_logger = make_shared(); + + try { + cereal::JSONInputArchive archive(decompressed_stream); + archive(cereal::make_nvp("logger", *window_logger)); + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Successfully deserialized logger from JSON"; + } catch (cereal::Exception& e) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the time window logger: " << e.what(); + return nullptr; + } + + return window_logger; +} + bool ConfidenceCalculator::postData() { - m_time_window_logger_backup = m_time_window_logger; - m_time_window_logger.clear(); - std::string url = getPostDataUrl(); + saveTimeWindowLogger(); + m_mainLoop->yield(false); + WindowLogPost currentWindow(*m_time_window_logger); + m_mainLoop->yield(false); + m_time_window_logger = make_shared(); + string url = getPostDataUrl() + to_string(m_post_index++); + m_estimated_memory_usage = 0; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url; - WindowLogPost currentWindow(m_time_window_logger_backup); bool ok = sendNoReplyObjectWithRetry(currentWindow, HTTPMethod::PUT, url); @@ -145,14 +447,23 @@ bool ConfidenceCalculator::postData() return ok; } -void ConfidenceCalculator::pullData(const std::vector& files) +void ConfidenceCalculator::pullData(const vector& files) { if (getIntervalsCount() == m_params.minIntervals) { mergeProcessedFromRemote(); } - std::string url = getPostDataUrl(); - std::string sentFile = url.erase(0, strlen("/storage/waap/")); + if (m_time_window_logger_backup == nullptr) + { + m_time_window_logger_backup = loadTimeWindowLogger(); + if (m_time_window_logger_backup == nullptr) + { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the time window logger"; + return; + } + } + string url = getPostDataUrl(); + string sentFile = url.erase(0, strlen("/storage/waap/")); dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "pulling files, skipping: " << sentFile; for (auto file : files) { @@ -174,13 +485,13 @@ void ConfidenceCalculator::pullData(const std::vector& files) KeyValSourcesLogger remoteLogger = getWindow.getWindowLogger().unpack(); for (auto& log : remoteLogger) { - std::string key = log.first; + const string & key = log.first; for (auto& entry : log.second) { - std::string value = entry.first; - for (auto& source : entry.second) + const string & value = entry.first; + for (auto & source : entry.second) { - m_time_window_logger_backup[key][value].insert(source); + (*m_time_window_logger_backup)[key][value].insert(source); } } } @@ -190,22 +501,55 @@ void ConfidenceCalculator::pullData(const std::vector& files) void ConfidenceCalculator::processData() { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " - processing the confidence data"; - if (m_time_window_logger_backup.empty()) + m_post_index = 0; + if (m_time_window_logger_backup == nullptr || m_time_window_logger_backup->empty()) { - m_time_window_logger_backup = m_time_window_logger; - m_time_window_logger.clear(); + if (m_path_to_backup != "") + { + m_time_window_logger_backup = loadTimeWindowLogger(); + m_mainLoop->yield(false); + if (m_time_window_logger_backup == nullptr) + { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the time window logger"; + return; + } + } else { + m_time_window_logger_backup = m_time_window_logger; + m_time_window_logger = make_shared(); + m_estimated_memory_usage = 0; + } } calculateInterval(); + // clear temp data + m_time_window_logger_backup->clear(); + m_time_window_logger_backup.reset(); + if (m_path_to_backup != "") + { + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; + } } -void ConfidenceCalculator::updateState(const std::vector& files) +void ConfidenceCalculator::updateState(const vector& files) { pullProcessedData(files); + // clear temp data + if (m_time_window_logger_backup) + { + m_time_window_logger_backup->clear(); + m_time_window_logger_backup.reset(); + } + if (m_path_to_backup != "") + { + remove(m_path_to_backup.c_str()); + m_path_to_backup = ""; + } } -void ConfidenceCalculator::pullProcessedData(const std::vector& files) +void ConfidenceCalculator::pullProcessedData(const vector& files) { dbgTrace(D_WAAP) << "Fetching the confidence set object"; + m_post_index = 0; bool is_first_pull = true; bool is_ok = false; for (auto file : files) @@ -217,12 +561,13 @@ void ConfidenceCalculator::pullProcessedData(const std::vector& fil is_ok |= res; if (res && getConfFile.getConfidenceSet().ok()) { - mergeFromRemote(getConfFile.getConfidenceSet().unpack(), is_first_pull); + mergeFromRemote(getConfFile.getConfidenceSet().unpackMove(), is_first_pull); is_first_pull = false; } if (res && getConfFile.getConfidenceLevels().ok()) { - m_confidence_level = getConfFile.getConfidenceLevels().unpackMove(); + // write to disk the confidence levels + saveConfidenceLevels(getConfFile.getConfidenceLevels()); } } // is_ok = false -> no file was downloaded and merged @@ -238,7 +583,7 @@ void ConfidenceCalculator::postProcessedData() dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Not sending the processed data - not enough windows"; return; } - std::string postUrl = getUri() + "/" + m_remotePath + "/processed/confidence.data"; + string postUrl = getUri() + "/" + m_remotePath + "/processed/confidence.data"; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Posting the confidence set object to: " << postUrl; ConfidenceFileEncryptor postConfFile(m_confident_sets, m_confidence_level); sendNoReplyObjectWithRetry(postConfFile, @@ -246,7 +591,7 @@ void ConfidenceCalculator::postProcessedData() postUrl); } -void ConfidenceCalculator::serialize(std::ostream& stream) +void ConfidenceCalculator::serialize(ostream& stream) { cereal::JSONOutputArchive archive(stream); @@ -255,10 +600,11 @@ void ConfidenceCalculator::serialize(std::ostream& stream) cereal::make_nvp("params", m_params), cereal::make_nvp("last_indicators_update", m_last_indicators_update), cereal::make_nvp("confidence_levels", m_confidence_level), - cereal::make_nvp("confident_sets", m_confident_sets) + cereal::make_nvp("confident_sets", m_confident_sets), + cereal::make_nvp("latest_index", m_latest_index + getIntervalsCount()) ); } -void ConfidenceCalculator::deserialize(std::istream& stream) +void ConfidenceCalculator::deserialize(istream& stream) { size_t version; cereal::JSONInputArchive archive(stream); @@ -266,7 +612,7 @@ void ConfidenceCalculator::deserialize(std::istream& stream) { archive(cereal::make_nvp("version", version)); } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); version = 0; dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << @@ -297,18 +643,18 @@ void ConfidenceCalculator::loadVer0(cereal::JSONInputArchive& archive) { if (!tryParseVersionBasedOnNames( archive, - std::string("params"), - std::string("last_indicators_update"), - std::string("windows_summary_list"), - std::string("confident_sets") + string("params"), + string("last_indicators_update"), + string("windows_summary_list"), + string("confident_sets") )) { tryParseVersionBasedOnNames( archive, - std::string("m_params"), - std::string("m_lastIndicatorsUpdate"), - std::string("m_windows_summary_list"), - std::string("m_confident_sets") + string("m_params"), + string("m_lastIndicatorsUpdate"), + string("m_windows_summary_list"), + string("m_confident_sets") ); } @@ -322,7 +668,7 @@ void ConfidenceCalculator::convertWindowSummaryToConfidenceLevel(const WindowsCo { for (const auto& value : window) { - m_confidence_level[windowKey.first][value] += std::ceil(SCORE_THRESHOLD / m_params.minIntervals); + m_confidence_level[windowKey.first][value] += ceil(SCORE_THRESHOLD / m_params.minIntervals); } } } @@ -339,6 +685,7 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) cereal::make_nvp("confidence_levels", confidenceLevels), cereal::make_nvp("confident_sets", confidenceSets) ); + params.maxMemoryUsage = defaultConfidenceMemUsage; reset(params); for (auto& confidentSet : confidenceSets) { @@ -346,7 +693,7 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) } for (auto& confidenceLevel : confidenceLevels) { - std::string normParam = normalize_param(confidenceLevel.first); + string normParam = normalize_param(confidenceLevel.first); if (m_confidence_level.find(normParam) == m_confidence_level.end()) { m_confidence_level[normParam] = confidenceLevel.second; @@ -361,7 +708,7 @@ void ConfidenceCalculator::loadVer2(cereal::JSONInputArchive& archive) } else { - double maxScore = std::max(m_confidence_level[normParam][valueLevelItr.first], + double maxScore = max(m_confidence_level[normParam][valueLevelItr.first], valueLevelItr.second); m_confidence_level[normParam][valueLevelItr.first] = maxScore; } @@ -379,6 +726,21 @@ void ConfidenceCalculator::loadVer3(cereal::JSONInputArchive& archive) cereal::make_nvp("confidence_levels", m_confidence_level), cereal::make_nvp("confident_sets", m_confident_sets) ); + try { + archive(cereal::make_nvp("latest_index", m_latest_index)); + } + catch (runtime_error & e) { + m_latest_index = 0; + archive.setNextName(nullptr); + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) + << "Owner: " << m_owner << + ", failed to load the latest index from the learned data file: " + << e.what(); + } + if (params.maxMemoryUsage == 0) + { + params.maxMemoryUsage = defaultConfidenceMemUsage; + } reset(params); } @@ -394,6 +756,8 @@ void ConfidenceCalculator::loadVer1(cereal::JSONInputArchive& archive) cereal::make_nvp("windows_summary_list", windows_summary_list), cereal::make_nvp("confident_sets", m_confident_sets) ); + params.maxMemoryUsage = defaultConfidenceMemUsage; + reset(params); convertWindowSummaryToConfidenceLevel(windows_summary_list); @@ -401,10 +765,10 @@ void ConfidenceCalculator::loadVer1(cereal::JSONInputArchive& archive) bool ConfidenceCalculator::tryParseVersionBasedOnNames( cereal::JSONInputArchive& archive, - const std::string ¶ms_field_name, - const std::string &indicators_update_field_name, - const std::string &windows_summary_field_name, - const std::string &confident_sets_field_name) + const string ¶ms_field_name, + const string &indicators_update_field_name, + const string &windows_summary_field_name, + const string &confident_sets_field_name) { bool result = true; try @@ -414,7 +778,7 @@ bool ConfidenceCalculator::tryParseVersionBasedOnNames( reset(temp_params); m_params = temp_params; } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " failed to load configuration of WAAP parameters from the learned data file: " @@ -428,7 +792,7 @@ bool ConfidenceCalculator::tryParseVersionBasedOnNames( archive(cereal::make_nvp(indicators_update_field_name, temp_last_indicator_update)); m_last_indicators_update = temp_last_indicator_update; } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " failed to load the update date for indicators from the learned data file: " @@ -442,7 +806,7 @@ bool ConfidenceCalculator::tryParseVersionBasedOnNames( archive(cereal::make_nvp(windows_summary_field_name, temp_windows_summary_list)); convertWindowSummaryToConfidenceLevel(temp_windows_summary_list); } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " failed to load windows summary list from the learned data file: " << e.what(); @@ -451,16 +815,18 @@ bool ConfidenceCalculator::tryParseVersionBasedOnNames( try { - std::unordered_map temp_confident_sets; + unordered_map temp_confident_sets; archive(cereal::make_nvp(confident_sets_field_name, temp_confident_sets)); - size_t current_time = std::chrono::duration_cast( - Singleton::Consume::by()->getWalltime()).count(); + size_t current_time = chrono::duration_cast( + Singleton::Consume::by()->getWalltime() + ).count(); + for (auto setItr : temp_confident_sets) { - m_confident_sets[setItr.first] = std::pair(setItr.second, current_time); + m_confident_sets[setItr.first] = pair(setItr.second, current_time); } } - catch (std::runtime_error & e) { + catch (runtime_error & e) { archive.setNextName(nullptr); dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " failed to load confident sets from the learned data file: " << e.what(); @@ -486,8 +852,8 @@ void ConfidenceCalculator::mergeConfidenceSets( confidence_set[normalize_param(set.first)].first.insert(value); } - confidence_set[set.first].second = std::max(confidence_set[set.first].second, set.second.second); - last_indicators_update = std::max(last_indicators_update, confidence_set[set.first].second); + confidence_set[set.first].second = max(confidence_set[set.first].second, set.second.second); + last_indicators_update = max(last_indicators_update, confidence_set[set.first].second); } }; @@ -523,7 +889,7 @@ bool ConfidenceCalculator::is_confident(const Key &key, const Val &value) const void ConfidenceCalculator::calcConfidentValues() { - std::unordered_map confidenceSetCopy = m_confident_sets; + unordered_map confidenceSetCopy = m_confident_sets; if (!m_params.learnPermanently) { m_confident_sets.clear(); @@ -543,10 +909,10 @@ void ConfidenceCalculator::calcConfidentValues() m_confident_sets[key].first.insert(value); if (m_confident_sets[key].first.size() > confidenceValuesSize) { - std::chrono::seconds current_time = std::chrono::duration_cast( + chrono::seconds current_time = chrono::duration_cast( Singleton::Consume::by()->getWalltime()); m_confident_sets[key].second = current_time.count(); - m_last_indicators_update = std::chrono::duration_cast(current_time).count(); + m_last_indicators_update = chrono::duration_cast(current_time).count(); } } } @@ -571,35 +937,57 @@ size_t ConfidenceCalculator::getLastConfidenceUpdate() return m_last_indicators_update; } -void ConfidenceCalculator::log(const Key &key, const Val &value, const std::string &source) +void ConfidenceCalculator::log(const Key &key, const Val &value, const string &source) { - m_time_window_logger[key][value].insert(source); + auto& sources_set = (*m_time_window_logger)[key][value]; + auto result = sources_set.insert(source); + if (result.second) { + // New entry added, update memory usage + if ((*m_time_window_logger)[key][value].size() == 1) { + // first source for this value - means new value + m_estimated_memory_usage += sizeof(value) + value.capacity(); + m_estimated_memory_usage += sizeof(SourcesSet); + + if ((*m_time_window_logger)[key].size() == 1) { + // first value for this key - means new key + m_estimated_memory_usage += sizeof(key) + key.capacity(); + m_estimated_memory_usage += sizeof(SourcesCounters); + } + } + m_estimated_memory_usage += sizeof(source) + source.capacity(); + } + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "memory usage: " << m_estimated_memory_usage << + "/" << m_params.maxMemoryUsage; + if (value != m_null_obj) { logSourceHit(key, source); + return; + } + + // if estimated memory usage is too high, send to the server + if (m_remoteSyncEnabled && m_estimated_memory_usage > m_params.maxMemoryUsage) + { + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "sending data to the server, memory usage: " + << m_estimated_memory_usage; + // run a onetime routine to send the data to the server + I_MainLoop *mainLoop = Singleton::Consume::by(); + mainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::Offline, + [this]() { + postData(); + }, + "ConfidenceCalculator post data offsync" + ); + m_estimated_memory_usage = 0; } } -void ConfidenceCalculator::logSourceHit(const Key &key, const std::string &source) +void ConfidenceCalculator::logSourceHit(const Key &key, const string &source) { log(key, m_null_obj, source); } -void ConfidenceCalculator::mergeSourcesCounter(const Key& key, const SourcesCounters& counters) -{ - if (key.rfind("url#", 0) == 0 && m_owner == "TypeIndicatorFilter") - { - return; - } - SourcesCounters& currentCounters = m_time_window_logger[key]; - for (auto& counter : counters) - { - SourcesSet& srcSet = currentCounters[counter.first]; - srcSet.insert(counter.second.begin(), counter.second.end()); - } -} - -void ConfidenceCalculator::removeBadSources(SourcesSet& sources, const std::vector* badSources) +void ConfidenceCalculator::removeBadSources(SourcesSet& sources, const vector* badSources) { if (badSources == nullptr) { @@ -629,15 +1017,154 @@ size_t ConfidenceCalculator::sumSourcesWeight(const SourcesSet& sources) return sourcesWeights; } +void ConfidenceCalculator::loadConfidenceLevels() +{ + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - loading the confidence levels from disk, latest index: " << m_latest_index << + ", intervals count: " << getIntervalsCount(); + + string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount() - 1) % 2) + ".gz"; + ifstream file(file_path, ios::binary); + if (!file.is_open()) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open the file: " << file_path; + return; + } + + stringstream buffer; + // Read the file in chunks to avoid large memory usage + const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault( + 16 * 1024, + "appsecLearningSettings.readChunkSize"); // 16 KiB + vector chunk(READ_CHUNK_SIZE); + size_t chunk_size = static_cast(READ_CHUNK_SIZE); + while (file.peek() != EOF) { + file.read(chunk.data(), chunk_size); + streamsize bytesRead = file.gcount(); + if (bytesRead > 0) { + buffer.write(chunk.data(), bytesRead); + } + m_mainLoop->yield(false); + } + file.close(); + + string compressed_data = buffer.str(); + + auto compression_stream = initCompressionStream(); + DecompressionResult res; + res.ok = true; + res.output = nullptr; + res.num_output_bytes = 0; + size_t offset = 0; + const size_t CHUNK_SIZE = static_cast( + getProfileAgentSettingWithDefault( + 16 * 1024, // 16KiB + "appsecLearningSettings.compressionChunkSize" + ) + ); + vector decompressed_data_vec; + while (offset < compressed_data.size()) { + size_t current_chunk_size = min(CHUNK_SIZE, compressed_data.size() - offset); + DecompressionResult chunk_res = decompressData( + compression_stream, + current_chunk_size, + reinterpret_cast(compressed_data.c_str() + offset) + ); + if (!chunk_res.ok) { + res.ok = false; + break; + } + if (chunk_res.output && chunk_res.num_output_bytes > 0) { + // Append directly to the vector to avoid extra copies + size_t old_size = decompressed_data_vec.size(); + decompressed_data_vec.resize(old_size + chunk_res.num_output_bytes); + memcpy(decompressed_data_vec.data() + old_size, chunk_res.output, chunk_res.num_output_bytes); + free(chunk_res.output); + } + offset += current_chunk_size; + m_mainLoop->yield(false); + } + finiCompressionStream(compression_stream); + + if (!res.ok) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress the confidence levels data."; + return; + } + + string decompressed_data(decompressed_data_vec.begin(), decompressed_data_vec.end()); + stringstream decompressed_stream(decompressed_data); + + try { + cereal::JSONInputArchive archive(decompressed_stream); + archive(cereal::make_nvp("confidence_levels", m_confidence_level)); + } catch (runtime_error &e) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk: " << e.what(); + } + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - loaded the confidence levels from disk, latest index: " << m_latest_index << + ", intervals count: " << getIntervalsCount(); + m_mainLoop->yield(false); + if (m_confidence_level.empty()) + { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk"; + } +} + +void ConfidenceCalculator::saveConfidenceLevels() +{ + Maybe confidenceLevels(genError("not available")); + saveConfidenceLevels(confidenceLevels); +} + +void ConfidenceCalculator::saveConfidenceLevels(Maybe confidenceLevels) +{ + string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount()) % 2) + ".gz"; + stringstream serialized_data; + + try { + { + cereal::JSONOutputArchive archive(serialized_data); + if (confidenceLevels.ok()) { + archive(cereal::make_nvp("confidence_levels", confidenceLevels.unpackMove())); + } else { + archive(cereal::make_nvp("confidence_levels", m_confidence_level)); + } + } + } catch (runtime_error &e) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize the confidence levels: " << e.what(); + return; + } + m_mainLoop->yield(false); + + string uncompressed_data = serialized_data.str(); + const size_t CHUNK_SIZE = 16 * 1024; // 16 KiB + vector compressed_data; + + try { + compressDataWrapper(uncompressed_data, CHUNK_SIZE, compressed_data); + writeToFile(file_path, compressed_data); + } catch (const runtime_error& e) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress the confidence levels data: " << e.what(); + } + + m_confidence_level.clear(); + + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - saved the confidence levels to disk, latest index: " << m_latest_index << + ", intervals count: " << getIntervalsCount(); +} + void ConfidenceCalculator::calculateInterval() { - std::vector* sourcesToIgnore = nullptr; + // load confidence levels from the disk + loadConfidenceLevels(); + + vector* sourcesToIgnore = nullptr; if (m_ignoreSources != nullptr) { int waitItr = 0; while (!m_ignoreSources->ready() && waitItr < WAIT_LIMIT) { - Singleton::Consume::by()->yield(BUSY_WAIT_TIME); + m_mainLoop->yield(BUSY_WAIT_TIME); waitItr++; } if (waitItr == WAIT_LIMIT && !m_ignoreSources->ready()) @@ -648,17 +1175,23 @@ void ConfidenceCalculator::calculateInterval() sourcesToIgnore = m_ignoreSources->getSourcesToIgnore(); } - for (auto sourcesCtrItr : m_time_window_logger_backup) + int itr = 0; + + for (auto sourcesCtrItr : *m_time_window_logger_backup) { + if (++itr % 20 == 0) { + // yield every 20 iterations + m_mainLoop->yield(false); + } SourcesCounters& srcCtrs = sourcesCtrItr.second; Key key = sourcesCtrItr.first; ValuesSet summary; double factor = 1.0; if (m_tuning != nullptr) { - std::string param_name = key; + string param_name = key; auto param_name_pos = key.find("#"); - if (param_name_pos != std::string::npos && (param_name_pos + 1) <= key.size()) { + if (param_name_pos != string::npos && (param_name_pos + 1) <= key.size()) { param_name = key.substr(param_name_pos + 1); // not always accurate but good enough } if (m_tuning->getDecision(param_name, PARAM_NAME) == BENIGN) @@ -674,16 +1207,15 @@ void ConfidenceCalculator::calculateInterval() removeBadSources(sourcesUnion, sourcesToIgnore); size_t numOfSources = sumSourcesWeight(sourcesUnion); - m_windows_counter[key]++; if (numOfSources < m_params.minSources) { // not enough sources to learn from dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " -" << " not enough sources to learn for " << key << " - needed: " << m_params.minSources << ", have: " << sourcesUnion.size(); - mergeSourcesCounter(key, srcCtrs); continue; } + for (auto srcSets : srcCtrs) { // log the ratio of unique sources from all sources for each value @@ -698,28 +1230,27 @@ void ConfidenceCalculator::calculateInterval() auto& confidenceLevel = m_confidence_level[key][value]; if (currentSourcesCount == 0) { - confidenceLevel -= std::ceil(SCORE_THRESHOLD / m_params.minIntervals); + confidenceLevel -= ceil(SCORE_THRESHOLD / m_params.minIntervals); continue; } double ratio = ((double)currentSourcesCount / numOfSources); - double diff = std::ceil(SCORE_THRESHOLD / m_params.minIntervals) * (ratio / m_params.ratioThreshold) * + double diff = ceil(SCORE_THRESHOLD / m_params.minIntervals) * (ratio / m_params.ratioThreshold) * logn(currentSourcesCount, m_params.minSources) * factor; confidenceLevel += diff; dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << " - key: " << key << - " value: " << value << "confidence level: " << confidenceLevel; + " value: " << value << " confidence level: " << confidenceLevel; } - m_windows_counter[key] = 0; } for (auto& keyMap : m_confidence_level) { for (auto& valMap : keyMap.second) { - if (m_time_window_logger_backup.find(keyMap.first) != m_time_window_logger_backup.end() && - m_time_window_logger_backup[keyMap.first].find(valMap.first) == - m_time_window_logger_backup[keyMap.first].end()) + if (m_time_window_logger_backup->find(keyMap.first) != m_time_window_logger_backup->end() && + (*m_time_window_logger_backup)[keyMap.first].find(valMap.first) == + (*m_time_window_logger_backup)[keyMap.first].end()) { // reduce confidence when value do not appear valMap.second *= m_params.ratioThreshold; @@ -727,11 +1258,12 @@ void ConfidenceCalculator::calculateInterval() } } - m_time_window_logger_backup.clear(); calcConfidentValues(); + // save confidence levels to the disk + saveConfidenceLevels(); } -void ConfidenceCalculator::setOwner(const std::string& owner) +void ConfidenceCalculator::setOwner(const string& owner) { m_owner = owner + "/ConfidenceCalculator"; } @@ -742,15 +1274,138 @@ bool ConfidenceCalculatorParams::operator==(const ConfidenceCalculatorParams& ot minIntervals == other.minIntervals && intervalDuration == other.intervalDuration && ratioThreshold == other.ratioThreshold && - learnPermanently == other.learnPermanently); + learnPermanently == other.learnPermanently && + maxMemoryUsage == other.maxMemoryUsage); } -std::ostream& operator<<(std::ostream& os, const ConfidenceCalculatorParams& ccp) +ostream& operator<<(ostream& os, const ConfidenceCalculatorParams& ccp) { os << "min sources: " << ccp.minSources << " min intervals: " << ccp.minIntervals << " interval duration(minutes): " << ccp.intervalDuration.count() << " ratio threshold: " << ccp.ratioThreshold << - " should keep indicators permanently: " << ccp.learnPermanently; + " should keep indicators permanently: " << ccp.learnPermanently << + " max memory usage: " << ccp.maxMemoryUsage; return os; } + +void ConfidenceCalculator::garbageCollector() +{ + dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - starting asynchronous garbage collection of carry-on data files"; + + I_MainLoop *mainLoop = Singleton::Consume::by(); + mainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::Offline, + [this, mainLoop]() { + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - running garbage collection of carry-on data files"; + // Extract the base filename from m_filePath (without directory) + string baseFileName; + size_t lastSlash = m_filePath.find_last_of('/'); + if (lastSlash != string::npos) { + baseFileName = m_filePath.substr(lastSlash + 1); + } else { + baseFileName = m_filePath; + } + + // Cache the directory path where we'll be looking for .data files + string dirPath = m_filePath.substr(0, m_filePath.find_last_of('/')); + if (dirPath.empty()) { + dirPath = "."; + } + + DIR* dir = opendir(dirPath.c_str()); + if (!dir) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - failed to open directory for garbage collection: " << dirPath; + return; + } + + struct dirent* entry; + int filesDeleted = 0; + int fileCount = 0; + + // Iterate through directory entries + while ((entry = readdir(dir)) != nullptr) { + string filename = entry->d_name; + + // Check if this is a carry-on data file matching the pattern: + // m_filePath + "." + to_string(keyHash) + ".data" + if (filename.length() > baseFileName.length() + 7 && + filename.compare(filename.length() - 5, 5, ".data") == 0 && + filename.compare(0, baseFileName.length(), baseFileName) == 0) { + + // Extract the part between baseFileName and .data (should be .{number}) + string middlePart = filename.substr( + baseFileName.length(), + filename.length() - baseFileName.length() - 5 + ); + + // Verify middle part starts with a dot followed by numeric characters + if (middlePart.length() > 1 && middlePart[0] == '.') { + bool isNumeric = true; + for (size_t i = 1; i < middlePart.length(); i++) { + if (!isdigit(middlePart[i])) { + isNumeric = false; + break; + } + } + + if (isNumeric) { + string fullPath = dirPath + "/" + filename; + // yield to prevent blocking - use all allocated time + mainLoop->yield(false); + // Delete the file + if (remove(fullPath.c_str()) == 0) { + filesDeleted++; + } else { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - failed to delete carry-on data file: " << fullPath << + " errono: " << errno; + } + } + } + if (fileCount - filesDeleted > 5) { + dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - failed to delete many files. Files not deleted: " << fileCount - filesDeleted; + } + } + } + + closedir(dir); + + // Additional logic to delete temporary files under /tmp/ matching waap_confidence_XXXXXX + const string tmpDir = getProfileAgentSettingWithDefault( + "/tmp/", + "appsecLearningSettings.tmpDir" + ); + dir = opendir(tmpDir.c_str()); + if (!dir) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - failed to open /tmp/ directory for garbage collection."; + return; + } + + while ((entry = readdir(dir)) != nullptr) { + string filename = entry->d_name; + + // Check if the file matches the pattern waap_confidence_XXXXXX + if (filename.find("waap_confidence_") == 0) { + string fullPath = tmpDir + filename; + if (remove(fullPath.c_str()) == 0) { + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Deleted temporary file: " << fullPath; + } else { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to delete temporary file: " << fullPath; + } + mainLoop->yield(false); + } + } + + closedir(dir); + + dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + " - finished garbage collection."; + }, + "ConfidenceCalculator garbage collection" + ); +} diff --git a/components/security_apps/waap/waap_clib/ConfidenceCalculator.h b/components/security_apps/waap/waap_clib/ConfidenceCalculator.h index 75d7146..bb591d3 100755 --- a/components/security_apps/waap/waap_clib/ConfidenceCalculator.h +++ b/components/security_apps/waap/waap_clib/ConfidenceCalculator.h @@ -28,6 +28,8 @@ #include "i_ignoreSources.h" #include "TuningDecisions.h" +static constexpr size_t defaultConfidenceMemUsage = 40 * 1024 * 1024; // 40MB + USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR); class WaapComponent; @@ -39,9 +41,10 @@ struct ConfidenceCalculatorParams std::chrono::minutes intervalDuration; double ratioThreshold; bool learnPermanently; + size_t maxMemoryUsage; template - void serialize(Archive& ar) + void serialize(Archive &ar) { size_t duration = intervalDuration.count(); ar(cereal::make_nvp("minSources", minSources), @@ -50,10 +53,17 @@ struct ConfidenceCalculatorParams cereal::make_nvp("ratioThreshold", ratioThreshold), cereal::make_nvp("learnPermanently", learnPermanently)); intervalDuration = std::chrono::minutes(duration); + try { + ar(cereal::make_nvp("maxMemoryUsage", maxMemoryUsage)); + } catch (cereal::Exception &e) { + maxMemoryUsage = defaultConfidenceMemUsage; + dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "maxMemoryUsage not found in serialized data"; + ar.setNextName(nullptr); + } } - bool operator==(const ConfidenceCalculatorParams& other); - friend std::ostream& operator<<(std::ostream& os, const ConfidenceCalculatorParams& ccp); + bool operator==(const ConfidenceCalculatorParams &other); + friend std::ostream & operator<<(std::ostream &os, const ConfidenceCalculatorParams &ccp); }; class ConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase @@ -74,7 +84,6 @@ public: typedef std::list ValuesList; typedef UMap WindowsConfidentValuesList; typedef UMap> ConfidenceLevels; - typedef UMap WindowsCounter; typedef UMap ConfidenceSet; ConfidenceCalculator(size_t minSources, @@ -82,19 +91,19 @@ public: std::chrono::minutes intervalDuration, double ratioThreshold, const Val &nullObj, - const std::string& backupPath, - const std::string& remotePath, - const std::string& assetId, + const std::string &backupPath, + const std::string &remotePath, + const std::string &assetId, TuningDecision* tuning = nullptr, I_IgnoreSources* ignoreSrc = nullptr); ~ConfidenceCalculator(); - void setOwner(const std::string& owner); + void setOwner(const std::string &owner); void hardReset(); void reset(); - bool reset(ConfidenceCalculatorParams& params); + bool reset(ConfidenceCalculatorParams ¶ms); virtual bool postData(); virtual void pullData(const std::vector& files); @@ -103,10 +112,12 @@ public: virtual void pullProcessedData(const std::vector& files); virtual void updateState(const std::vector& files); - virtual void serialize(std::ostream& stream); - virtual void deserialize(std::istream& stream); + virtual void serialize(std::ostream &stream); + virtual void deserialize(std::istream &stream); - void mergeFromRemote(const ConfidenceSet& remote_confidence_set, bool is_first_pull); + Maybe writeToFile(const std::string& path, const std::vector& data); + + void mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull); bool is_confident(const Key &key, const Val &value) const; @@ -121,35 +132,50 @@ public: void calculateInterval(); - static void mergeConfidenceSets(ConfidenceSet& confidence_set, - const ConfidenceSet& confidence_set_to_merge, - size_t& last_indicators_update); + static void mergeConfidenceSets(ConfidenceSet &confidence_set, + const ConfidenceSet &confidence_set_to_merge, + size_t &last_indicators_update); private: - void loadVer0(cereal::JSONInputArchive& archive); - void loadVer1(cereal::JSONInputArchive& archive); - void loadVer2(cereal::JSONInputArchive& archive); - void loadVer3(cereal::JSONInputArchive& archive); + void loadVer0(cereal::JSONInputArchive &archive); + void loadVer1(cereal::JSONInputArchive &archive); + void loadVer2(cereal::JSONInputArchive &archive); + void loadVer3(cereal::JSONInputArchive &archive); bool tryParseVersionBasedOnNames( - cereal::JSONInputArchive& archive, + cereal::JSONInputArchive &archive, const std::string ¶ms_field_name, const std::string &indicators_update_field_name, const std::string &windows_summary_field_name, const std::string &confident_sets_field_name); - void convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList& windows); + void convertWindowSummaryToConfidenceLevel(const WindowsConfidentValuesList &windows); - std::string getParamName(const Key& key); - size_t sumSourcesWeight(const SourcesSet& sources); - void mergeSourcesCounter(const Key& key, const SourcesCounters& counters); - void removeBadSources(SourcesSet& sources, const std::vector* badSources); + void loadConfidenceLevels(); + void saveConfidenceLevels(Maybe confidenceLevels); + void saveConfidenceLevels(); + + void saveTimeWindowLogger(); + std::shared_ptr loadTimeWindowLogger(); + + std::string getParamName(const Key &key); + size_t sumSourcesWeight(const SourcesSet &sources); + void removeBadSources(SourcesSet &sources, const std::vector* badSources); + + // Delete existing carry-on data files asynchronously with yields + void garbageCollector(); ConfidenceCalculatorParams m_params; Val m_null_obj; - KeyValSourcesLogger m_time_window_logger; - KeyValSourcesLogger m_time_window_logger_backup; + std::shared_ptr m_time_window_logger; + std::shared_ptr m_time_window_logger_backup; + std::string m_path_to_backup; ConfidenceSet m_confident_sets; ConfidenceLevels m_confidence_level; - WindowsCounter m_windows_counter; size_t m_last_indicators_update; + size_t m_latest_index; I_IgnoreSources* m_ignoreSources; TuningDecision* m_tuning; + size_t m_estimated_memory_usage; // Variable to track estimated memory usage + size_t m_post_index; + I_MainLoop *m_mainLoop; + I_MainLoop::RoutineID m_routineId; + std::vector m_filesToRemove; }; diff --git a/components/security_apps/waap/waap_clib/Csrf.cc b/components/security_apps/waap/waap_clib/Csrf.cc index d98258e..d46a043 100644 --- a/components/security_apps/waap/waap_clib/Csrf.cc +++ b/components/security_apps/waap/waap_clib/Csrf.cc @@ -45,6 +45,21 @@ State::decide } auto csrfDecision = decision.getDecision(CSRF_DECISION); + auto autonomousDecision = decision.getDecision(AUTONOMOUS_SECURITY_DECISION); + if (autonomousDecision->shouldForceBlock()) + { + dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): Autonomous decision force should block."; + csrfDecision->setBlock(true); + csrfDecision->setForceBlock(true); + return true; + } + if (autonomousDecision->shouldForceAllow()) + { + dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): Autonomous decision force should allow."; + csrfDecision->setBlock(false); + csrfDecision->setForceAllow(true); + return false; + } if (csrf_token.empty()) { dbgTrace(D_WAAP) << "Waap::CSRF::State::decide(): missing token."; diff --git a/components/security_apps/waap/waap_clib/DecisionType.h b/components/security_apps/waap/waap_clib/DecisionType.h index 8808654..41edc54 100755 --- a/components/security_apps/waap/waap_clib/DecisionType.h +++ b/components/security_apps/waap/waap_clib/DecisionType.h @@ -14,6 +14,8 @@ #ifndef __DECISION_TYPE_H__ #define __DECISION_TYPE_H__ +#include + enum DecisionType { // This order determines the priority of the decisions sent to management @@ -28,4 +30,35 @@ enum DecisionType // Must be kept last NO_WAAP_DECISION }; + +inline const char * +decisionTypeToString(DecisionType type) +{ + switch (type) { + case DecisionType::AUTONOMOUS_SECURITY_DECISION: + return "AUTONOMOUS_SECURITY_DECISION"; + case DecisionType::CSRF_DECISION: + return "CSRF_DECISION"; + case DecisionType::OPEN_REDIRECT_DECISION: + return "OPEN_REDIRECT_DECISION"; + case DecisionType::ERROR_DISCLOSURE_DECISION: + return "ERROR_DISCLOSURE_DECISION"; + case DecisionType::ERROR_LIMITING_DECISION: + return "ERROR_LIMITING_DECISION"; + case DecisionType::USER_LIMITS_DECISION: + return "USER_LIMITS_DECISION"; + case DecisionType::RATE_LIMITING_DECISION: + return "RATE_LIMITING_DECISION"; + case DecisionType::NO_WAAP_DECISION: + return "NO_WAAP_DECISION"; + default: + return "INVALID_DECISION_TYPE"; + } +} + +inline std::ostream & operator<<(std::ostream& os, const DecisionType& type) +{ + return os << decisionTypeToString(type); +} + #endif diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index 7a5dbf9..06d2c54 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -36,6 +36,7 @@ #include "debug.h" #include "i_transaction.h" #include "agent_core_utilities.h" +#include USE_DEBUG_FLAG(D_WAAP_DEEP_PARSER); USE_DEBUG_FLAG(D_WAAP_ULIMITS); @@ -93,6 +94,12 @@ DeepParser::depth() const return m_depth; } +static bool err = false; +static const SingleRegex temperature_value_re( + "^\\s*([0-9](?:\\.\\d+)?)\\s*$", + err, + "temperature_value"); + // Called when another key/value pair is ready int DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) @@ -195,6 +202,14 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f bool isBodyPayload = (m_key.first().size() == 4 && m_key.first() == "body"); + if (isBodyPayload && v_len < 32 && k_len == 11 && + boost::to_lower_copy(std::string(k, k_len)) == "temperature" && + temperature_value_re.hasMatch(std::string(v, v_len))) { + m_pTransaction->setTemperatureDetected(true); + dbgTrace(D_WAAP_DEEP_PARSER) << "temperature detected, value: " << std::string(v, v_len); + } + + // If csrf/antibot cookie - send to Waf2Transaction for collection of cookie value. if (m_depth == 1 && isCookiePayload && (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) { std::string cur_val = std::string(v, v_len); @@ -288,6 +303,11 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f dbgTrace(D_WAAP_DEEP_PARSER) << "removing leading '/' from URL param value"; base64_offset = 1; } + if (m_depth == 1 && (isUrlParamPayload || isRefererParamPayload) && + k_len != 0 && (v_len == 0 || (v[0] == '=' && v_len == 1))) { + // if the value is empty or starts with '=' - replace it with key + cur_val = std::string(k, k_len); + } std::string decoded_val, decoded_key; base64_variants base64_status = Waap::Util::b64Test( cur_val, @@ -477,6 +497,19 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f } } + // If this is url_paran and key is match to nosql_key_evasion_detector_re and this is 1st and last buffer + // than add to beginning of cur_val "=" where key is the key + if (flags == BUFFERED_RECEIVER_F_BOTH) { + std::string key = std::string(k, k_len); + if (Waap::Util::testNoSQLKeySuspect(key)) { + cur_val = key + "=" + cur_val; + dbgTrace(D_WAAP_DEEP_PARSER) + << "DeepParser::onKv(): found: key = " + << key + << " is a candidate for NoSQL key evasion - sending to updated string for scanning."; + } + } + // If there's a parser in parsers stack, push the value to the top parser if (!m_parsersDeque.empty() && offset >= 0 @@ -1326,7 +1359,7 @@ DeepParser::createInternalParser( } else if (b64FileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE) { dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a known binary file, base64 encoded"; m_parsersDeque.push_back( - std::make_shared>(*this, parser_depth + 1, true, b64FileType) + std::make_shared>(*this, parser_depth + 1, false, b64FileType) ); offset = 0; } diff --git a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc index 27b0acc..f93a4ae 100755 --- a/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc +++ b/components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc @@ -89,6 +89,7 @@ bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const shouldFilter |= m_keywordsFreqFilter->shouldFilterKeyword(type, keyword); } } + if (m_matchedOverrideKeywords.size() > 0 && m_matchedOverrideKeywords.find(keyword) != m_matchedOverrideKeywords.end()) { diff --git a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc index faa86fe..daed6f4 100755 --- a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc +++ b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc @@ -86,10 +86,11 @@ bool KeywordIndicatorFilter::loadParams(std::shared_ptrgetParamVal("learnIndicators.learnPermanently", "true"); params.learnPermanently = !boost::iequals(learnPermanentlyStr.c_str(), "false"); + params.maxMemoryUsage = std::stoul(pParams->getParamVal("learnIndicators.maxMemoryUsage", + std::to_string(CONFIDENCE_MAX_MEMORY_USAGE))); std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true"); bool syncEnabled = !boost::iequals(remoteSyncStr, "false"); - dbgTrace(D_WAAP) << params << " remote sync: " << remoteSyncStr; m_confidence_calc.setRemoteSyncEnabled(syncEnabled); diff --git a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h index 8e7ccc4..1fc7177 100755 --- a/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h +++ b/components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h @@ -21,6 +21,7 @@ #define CONFIDENCE_MIN_INTERVALS 5 #define CONFIDENCE_THRESHOLD 0.8 #define CONFIDENCE_WINDOW_INTERVAL std::chrono::minutes(120) +#define CONFIDENCE_MAX_MEMORY_USAGE (40 * 1024 * 1024) // 40MB class KeywordIndicatorFilter : public IndicatorFilterBase diff --git a/components/security_apps/waap/waap_clib/LogGenWrapper.cc b/components/security_apps/waap/waap_clib/LogGenWrapper.cc index 3c09611..a3fb9c5 100644 --- a/components/security_apps/waap/waap_clib/LogGenWrapper.cc +++ b/components/security_apps/waap/waap_clib/LogGenWrapper.cc @@ -41,15 +41,16 @@ LogGenWrapper::LogGenWrapper( } else { m_log_gen = std::make_unique( - maybe_trigger.unpack(), title, - security_type, + ReportIS::Level::LOG, + ReportIS::Audience::SECURITY, severity, priority, - is_action_drop_or_prevent, ReportIS::Tags::WAF, - ReportIS::Tags::THREAT_PREVENTION - ); + ReportIS::Tags::THREAT_PREVENTION, + maybe_trigger.unpack().getStreams(security_type, is_action_drop_or_prevent), + maybe_trigger.unpack().getEnrechments(security_type) + ); } } diff --git a/components/security_apps/waap/waap_clib/ParserBinaryFile.cc b/components/security_apps/waap/waap_clib/ParserBinaryFile.cc index 6379785..cbc7bca 100644 --- a/components/security_apps/waap/waap_clib/ParserBinaryFile.cc +++ b/components/security_apps/waap/waap_clib/ParserBinaryFile.cc @@ -78,7 +78,6 @@ ParserBinaryFile::detectBinaryFileHeader(const string &buf) return BinaryFileType::FILE_TYPE_NONE; } - size_t ParserBinaryFile::push(const char *buf, size_t len) { @@ -151,7 +150,10 @@ ParserBinaryFile::push(const char *buf, size_t len) } else { dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail; size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0; - c = strstr(buf + tail_lookup_offset, tail.c_str()); + c = static_cast(memmem(buf + tail_lookup_offset, + len - tail_lookup_offset, + tail.c_str(), + tail.size())); dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c; if (c) { m_state = s_end; diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc index 4b77428..3c7d59d 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc @@ -21,7 +21,11 @@ USE_DEBUG_FLAG(D_WAAP); const std::string ParserUrlEncode::m_parserName = "ParserUrlEncode"; ParserUrlEncode::ParserUrlEncode( - IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar, bool should_decode_per + IParserStreamReceiver &receiver, + size_t parser_depth, + char separatorChar, + bool should_decode_per, + bool should_decode_plus ) : m_receiver(receiver), m_state(s_start), @@ -29,13 +33,16 @@ ParserUrlEncode::ParserUrlEncode( m_separatorChar(separatorChar), m_escapedCharCandidate(0), should_decode_percent(should_decode_per), + m_should_decode_plus(should_decode_plus), m_parser_depth(parser_depth) { dbgTrace(D_WAAP) << "should_decode_percent=" << should_decode_per << "parser_depth=" - << parser_depth; + << parser_depth + << "m_should_decode_plus=" + << m_should_decode_plus; // TODO:: is there a need for this? memset(m_escaped, 0, sizeof(m_escaped)); @@ -124,7 +131,7 @@ ParserUrlEncode::push(const char *buf, size_t len) } m_state = s_key_escaped1; break; - } else if (c == '+') { + } else if (c == '+' && m_should_decode_plus) { // convert plus character to space if (i - mark > 0) { if (m_receiver.onKey(buf + mark, i - mark) != 0) { @@ -281,7 +288,7 @@ ParserUrlEncode::push(const char *buf, size_t len) } m_state = s_value_escaped1; break; - } else if (c == '+') { + } else if (c == '+' && m_should_decode_plus) { // convert plus character to space if (i - mark > 0) { if (m_receiver.onValue(buf + mark, i - mark) != 0) { diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.h b/components/security_apps/waap/waap_clib/ParserUrlEncode.h index 7b1cbc0..4141475 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.h +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.h @@ -25,7 +25,8 @@ public: IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar = '&', - bool should_decode_per = true); + bool should_decode_per = true, + bool should_decode_plus = true); virtual ~ParserUrlEncode(); size_t push(const char *data, size_t data_len); void finish(); @@ -55,6 +56,7 @@ private: char m_separatorChar; char m_escapedCharCandidate; bool should_decode_percent; + bool m_should_decode_plus; static const std::string m_parserName; size_t m_parser_depth; }; diff --git a/components/security_apps/waap/waap_clib/ParserXML.cc b/components/security_apps/waap/waap_clib/ParserXML.cc index 84da857..e6d5466 100755 --- a/components/security_apps/waap/waap_clib/ParserXML.cc +++ b/components/security_apps/waap/waap_clib/ParserXML.cc @@ -170,19 +170,22 @@ ParserXML::onEntityDeclaration( { dbgTrace(D_WAAP_PARSER_XML) << "ENTITY FOUND WITH VALUE: '" << (content ? (const char*)content : "null") << "'"; - ParserXML* p = (ParserXML*)ctx; - std::string kw = "08a80340-06d3-11ea-9f87-0242ac11000f"; + if (systmeid != nullptr) { + dbgTrace(D_WAAP_PARSER_XML) << "ENTITY FOUND WITH SYSTEM ID: '" << (const char*)systmeid << "'"; + ParserXML* p = (ParserXML*)ctx; + std::string kw = "08a80340-06d3-11ea-9f87-0242ac11000f"; - if (p->m_receiver.onKey(p->m_key.c_str(), p->m_key.size()) != 0) { - p->m_state = s_error; - } + if (p->m_receiver.onKey(p->m_key.c_str(), p->m_key.size()) != 0) { + p->m_state = s_error; + } - if (p->m_receiver.onValue(kw.data(), kw.size()) != 0) { - p->m_state = s_error; - } + if (p->m_receiver.onValue(kw.data(), kw.size()) != 0) { + p->m_state = s_error; + } - if (p->m_receiver.onKvDone() != 0) { - p->m_state = s_error; // error + if (p->m_receiver.onKvDone() != 0) { + p->m_state = s_error; // error + } } } diff --git a/components/security_apps/waap/waap_clib/RequestsMonitor.cc b/components/security_apps/waap/waap_clib/RequestsMonitor.cc index af156c1..a328d17 100644 --- a/components/security_apps/waap/waap_clib/RequestsMonitor.cc +++ b/components/security_apps/waap/waap_clib/RequestsMonitor.cc @@ -18,9 +18,11 @@ SourcesRequestMonitor::SourcesRequestMonitor( filePath, remotePath != "" ? remotePath + "/Monitor" : remotePath, assetId, - owner - ), m_sourcesRequests() + owner), + m_sourcesRequests(), + m_enabled(false) { + m_enabled = getProfileAgentSettingWithDefault(false, "appsec.sourceRequestsMonitor.enabled"); } SourcesRequestMonitor::~SourcesRequestMonitor() @@ -35,17 +37,18 @@ void SourcesRequestMonitor::syncWorker() OrchestrationMode mode = Singleton::exists() ? Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; - bool enabled = getProfileAgentSettingWithDefault(false, "appsec.sourceRequestsMonitor.enabled"); + m_enabled = getProfileAgentSettingWithDefault(false, "appsec.sourceRequestsMonitor.enabled"); - if (mode == OrchestrationMode::OFFLINE || !enabled || isBase() || !postData()) { + if (mode == OrchestrationMode::OFFLINE || !m_enabled || isBase() || !postData()) { dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Did not report data. for asset: " << m_assetId << " Remote URL: " << m_remotePath << " is enabled: " - << to_string(enabled) + << to_string(m_enabled) << ", mode: " << int(mode); + m_sourcesRequests.clear(); return; } @@ -72,6 +75,9 @@ void SourcesRequestMonitor::syncWorker() void SourcesRequestMonitor::logSourceHit(const string& source) { + if (!m_enabled) { + return; + } m_sourcesRequests[chrono::duration_cast( Singleton::Consume::by()->getWalltime() ).count()][source]++; diff --git a/components/security_apps/waap/waap_clib/RequestsMonitor.h b/components/security_apps/waap/waap_clib/RequestsMonitor.h index f0298eb..79daab5 100644 --- a/components/security_apps/waap/waap_clib/RequestsMonitor.h +++ b/components/security_apps/waap/waap_clib/RequestsMonitor.h @@ -28,6 +28,7 @@ protected: private: // map of sources and their requests per minute (UNIX) MonitorData m_sourcesRequests; + bool m_enabled; }; #endif // __REQUESTS_MONITOR_H__ diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index f770342..578a6a2 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -29,7 +29,7 @@ #include "compression_utils.h" #include "config.h" -USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR); +USE_DEBUG_FLAG(D_WAAP_SERIALIZE); namespace ch = std::chrono; using namespace std; @@ -52,6 +52,22 @@ isGZipped(const string &stream) return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b; } +void yieldIfPossible(const string& func, int line) +{ + // Check if we are in the main loop + if (Singleton::exists() && + Singleton::Consume::by()->getCurrentRoutineId().ok()) + { + // If we are not in the main loop, yield to allow other routines to run + // This is important for the main loop to be able to process other events + // and avoid blocking the entire system. + dbgTrace(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line; + Singleton::Consume::by()->yield(false); + } +} + +#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__) + bool RestGetFile::loadJson(const string& json) { string json_str; @@ -61,6 +77,9 @@ bool RestGetFile::loadJson(const string& json) { return ClientRest::loadJson(json_str); } + YIELD_IF_POSSIBLE(); + dbgTrace(D_WAAP_SERIALIZE) << "before decompression in loadJson, data size: " + << json_str.size() << " bytes"; auto compression_stream = initCompressionStream(); DecompressionResult res = decompressData( compression_stream, @@ -75,33 +94,81 @@ bool RestGetFile::loadJson(const string& json) } finiCompressionStream(compression_stream); + YIELD_IF_POSSIBLE(); + dbgTrace(D_WAAP_SERIALIZE) << "Yielded after decompression in loadJson, decompressed size: " + << json_str.size() << " bytes"; + return ClientRest::loadJson(json_str); } Maybe RestGetFile::genJson() const { Maybe json = ClientRest::genJson(); + YIELD_IF_POSSIBLE(); + if (json.ok()) { string data = json.unpack(); + + // Get chunk size from profile settings for compression chunks + const size_t COMPRESSED_CHUNK_SIZE = static_cast( + getProfileAgentSettingWithDefault(64 * 1024, "appsecLearningSettings.compressionChunkSize")); + auto compression_stream = initCompressionStream(); - CompressionResult res = compressData( - compression_stream, - CompressionType::GZIP, - data.size(), - reinterpret_cast(data.c_str()), - true); + size_t offset = 0; + std::vector compressed_data; + bool ok = true; + size_t chunk_count = 0; + + // Process data in chunks for compression + while (offset < data.size()) { + size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset); + bool is_last = (offset + chunk_size >= data.size()); + CompressionResult chunk_res = compressData( + compression_stream, + CompressionType::GZIP, + static_cast(chunk_size), + reinterpret_cast(data.c_str() + offset), + is_last ? 1 : 0 + ); + + if (!chunk_res.ok) { + ok = false; + break; + } + + if (chunk_res.output && chunk_res.num_output_bytes > 0) { + compressed_data.insert( + compressed_data.end(), + chunk_res.output, + chunk_res.output + chunk_res.num_output_bytes + ); + free(chunk_res.output); + chunk_res.output = nullptr; + } + + offset += chunk_size; + chunk_count++; + YIELD_IF_POSSIBLE(); + dbgTrace(D_WAAP_SERIALIZE) << "Processed compression chunk " << chunk_count + << ", progress: " << offset << "/" << data.size() << " bytes (" + << (offset * 100 / data.size()) << "%) - yielded"; + } + finiCompressionStream(compression_stream); - if (!res.ok) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data"; + dbgDebug(D_WAAP_SERIALIZE) << "Yielded after finalizing compression stream. " + << "Total chunks: " << chunk_count << ", Compression ratio: " + << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x"; + + if (!ok) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to gzip data"; return genError("Failed to compress data"); } - data = string((const char *)res.output, res.num_output_bytes); - json = data; - if (res.output) free(res.output); - res.output = nullptr; - res.num_output_bytes = 0; + // Create string from compressed data + string compressed_str(reinterpret_cast(compressed_data.data()), compressed_data.size()); + + json = compressed_str; } return json; } @@ -128,16 +195,16 @@ void SerializeToFilePeriodically::backupWorker() I_TimeGet* timer = Singleton::Consume::by(); auto currentTime = timer->getMonotonicTime(); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: current time: " << currentTime.count(); + dbgTrace(D_WAAP_SERIALIZE) << "backup worker: current time: " << currentTime.count(); if (currentTime - m_lastSerialization >= m_interval) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: backing up data"; + dbgTrace(D_WAAP_SERIALIZE) << "backup worker: backing up data"; m_lastSerialization = currentTime; // save data saveData(); - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "backup worker: data is backed up"; + dbgTrace(D_WAAP_SERIALIZE) << "backup worker: data is backed up"; } } @@ -153,7 +220,7 @@ void SerializeToFilePeriodically::setInterval(ch::seconds newInterval) SerializeToFileBase::SerializeToFileBase(string fileName) : m_filePath(fileName) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath + dbgTrace(D_WAAP_SERIALIZE) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath << "'"; } @@ -165,52 +232,119 @@ SerializeToFileBase::~SerializeToFileBase() void SerializeToFileBase::saveData() { fstream filestream; - - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "saving to file: " << m_filePath; + auto maybe_routine = Singleton::Consume::by()->getCurrentRoutineId(); + dbgTrace(D_WAAP_SERIALIZE) << "saving to file: " << m_filePath; filestream.open(m_filePath, fstream::out); stringstream ss; if (filestream.is_open() == false) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to open file: " << m_filePath << " Error: " + dbgWarning(D_WAAP_SERIALIZE) << "failed to open file: " << m_filePath << " Error: " << strerror(errno); return; } - + if (maybe_routine.ok()) { + Singleton::Consume::by()->yield(false); + } serialize(ss); + if (maybe_routine.ok()) { + Singleton::Consume::by()->yield(false); + } string data = ss.str(); + dbgDebug(D_WAAP_SERIALIZE) << "Serialized data size: " << data.size() << " bytes"; + + // Get chunk size from profile settings, with default of 16 MiB for compression chunks + const size_t CHUNK_SIZE = static_cast( + getProfileAgentSettingWithDefault(16 * 1024 * 1024, "appsecLearningSettings.writeChunkSize")); + // Get chunk size for writing compressed data, with default of 16 MiB + const size_t COMPRESSED_CHUNK_SIZE = static_cast( + getProfileAgentSettingWithDefault(16 * 1024 * 1024, "appsecLearningSettings.compressionChunkSize")); auto compression_stream = initCompressionStream(); - CompressionResult res = compressData( - compression_stream, - CompressionType::GZIP, - data.size(), - reinterpret_cast(data.c_str()), - true - ); - finiCompressionStream(compression_stream); - if (!res.ok) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data"; - } else { - ss.str(string((const char *)res.output, res.num_output_bytes)); - // free the memory allocated by compressData - if (res.output) free(res.output); - res.output = nullptr; - res.num_output_bytes = 0; + size_t offset = 0; + std::vector compressed_data; + bool ok = true; + size_t chunk_count = 0; + + // Process data in chunks for compression + while (offset < data.size()) { + size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset); + bool is_last = (offset + chunk_size >= data.size()); + CompressionResult chunk_res = compressData( + compression_stream, + CompressionType::GZIP, + static_cast(chunk_size), + reinterpret_cast(data.c_str() + offset), + is_last ? 1 : 0 + ); + + if (!chunk_res.ok) { + ok = false; + break; + } + + if (chunk_res.output && chunk_res.num_output_bytes > 0) { + compressed_data.insert( + compressed_data.end(), + chunk_res.output, + chunk_res.output + chunk_res.num_output_bytes + ); + free(chunk_res.output); + chunk_res.output = nullptr; + } + + offset += chunk_size; + chunk_count++; + if (maybe_routine.ok()) { + Singleton::Consume::by()->yield(false); + dbgTrace(D_WAAP_SERIALIZE) << "Compression chunk " << chunk_count + << " processed (" << offset << "/" << data.size() << " bytes, " + << (offset * 100 / data.size()) << "%) - yielded"; + } } - if (res.output) free(res.output); - res.output = nullptr; - res.num_output_bytes = 0; + finiCompressionStream(compression_stream); + dbgDebug(D_WAAP_SERIALIZE) << "Finished compression stream. " + << "Total chunks: " << chunk_count << ", Compression ratio: " + << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x"; + + if (!ok) { + dbgWarning(D_WAAP_SERIALIZE) << "Failed to compress data"; + filestream.close(); + return; + } + + dbgDebug(D_WAAP_SERIALIZE) << "Compression complete: " << data.size() << " bytes -> " + << compressed_data.size() << " bytes (ratio: " + << (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x)"; - filestream << ss.str(); + // Use compressed data directly + string data_to_write(reinterpret_cast(compressed_data.data()), compressed_data.size()); + + // Write data to file in chunks with yield points + offset = 0; + size_t write_chunks = 0; + + while (offset < data_to_write.size()) { + size_t current_chunk_size = std::min(CHUNK_SIZE, data_to_write.size() - offset); + filestream.write(data_to_write.c_str() + offset, current_chunk_size); + offset += current_chunk_size; + write_chunks++; + Singleton::Consume::by()->yield(false); + dbgTrace(D_WAAP_SERIALIZE) << "Write chunk " << write_chunks + << " complete: " << offset << "/" << data_to_write.size() << " bytes (" + << (offset * 100 / data_to_write.size()) << "%) - yielded"; + } + filestream.close(); + dbgDebug(D_WAAP_SERIALIZE) << "Finished writing backup file: " << m_filePath + << " (" << data_to_write.size() << " bytes in " << write_chunks << " chunks)"; } string decompress(string fileContent) { if (!isGZipped(fileContent)) { - dbgTrace(D_WAAP) << "file note zipped"; + dbgTrace(D_WAAP_SERIALIZE) << "file note zipped"; return fileContent; } auto compression_stream = initCompressionStream(); @@ -236,13 +370,13 @@ string decompress(string fileContent) { void SerializeToFileBase::loadFromFile(string filePath) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loadFromFile() file: " << filePath; + dbgTrace(D_WAAP_SERIALIZE) << "loadFromFile() file: " << filePath; fstream filestream; filestream.open(filePath, fstream::in); if (filestream.is_open() == false) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to open file: " << filePath << " Error: " << + dbgTrace(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " << strerror(errno); if (!Singleton::exists() || errno != ENOENT) { @@ -262,18 +396,18 @@ void SerializeToFileBase::loadFromFile(string filePath) if (idPosition != string::npos) { filePath.erase(idPosition, idStr.length() - 1); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "retry to load file from : " << filePath; + dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << filePath; loadFromFile(filePath); } return; } - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loading from file: " << filePath; + dbgTrace(D_WAAP_SERIALIZE) << "loading from file: " << filePath; int length; filestream.seekg(0, ios::end); // go to the end length = filestream.tellg(); // report location (this is the length) - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "file length: " << length; + dbgTrace(D_WAAP_SERIALIZE) << "file length: " << length; assert(length >= 0); // length -1 really happens if filePath is a directory (!) char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension filestream.seekg(0, ios::beg); // go back to the beginning @@ -281,7 +415,7 @@ void SerializeToFileBase::loadFromFile(string filePath) { filestream.close(); delete[] buffer; - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to read file, file: " << filePath; + dbgWarning(D_WAAP_SERIALIZE) << "Failed to read file, file: " << filePath; return; } filestream.close(); @@ -298,7 +432,7 @@ void SerializeToFileBase::loadFromFile(string filePath) deserialize(ss); } catch (runtime_error & e) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to deserialize file: " << m_filePath << ", error: " << + dbgWarning(D_WAAP_SERIALIZE) << "failed to deserialize file: " << m_filePath << ", error: " << e.what(); } } @@ -318,11 +452,11 @@ RemoteFilesList::RemoteFilesList() : files(), filesPathsList() bool RemoteFilesList::loadJson(const string& xml) { xmlDocPtr doc; // the resulting document tree - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "XML input: " << xml; + dbgTrace(D_WAAP_SERIALIZE) << "XML input: " << xml; doc = xmlParseMemory(xml.c_str(), xml.length()); if (doc == NULL) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to parse " << xml; + dbgWarning(D_WAAP_SERIALIZE) << "Failed to parse " << xml; return false; } @@ -343,7 +477,7 @@ bool RemoteFilesList::loadJson(const string& xml) { if (xmlStrEqual(contents_name, node->name) == 1) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the Contents element"; + dbgTrace(D_WAAP_SERIALIZE) << "Found the Contents element"; xmlNodePtr contents_node = node->children; string file; string lastModified; @@ -351,21 +485,21 @@ bool RemoteFilesList::loadJson(const string& xml) { if (xmlStrEqual(key_name, contents_node->name) == 1) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the Key element"; + dbgTrace(D_WAAP_SERIALIZE) << "Found the Key element"; xmlChar* xml_file = xmlNodeGetContent(contents_node); file = string(reinterpret_cast(xml_file)); xmlFree(xml_file); } if (xmlStrEqual(last_modified_name, contents_node->name) == 1) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Found the LastModified element"; + dbgTrace(D_WAAP_SERIALIZE) << "Found the LastModified element"; xmlChar* xml_file = xmlNodeGetContent(contents_node); lastModified = string(reinterpret_cast(xml_file)); xmlFree(xml_file); } if (!file.empty() && !lastModified.empty()) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Adding the file: " << file << + dbgTrace(D_WAAP_SERIALIZE) << "Adding the file: " << file << " last modified: " << lastModified; break; } @@ -408,18 +542,18 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( m_interval(0), m_owner(owner), m_assetId(replaceAllCopy(assetId, "/", "")), + m_remoteSyncEnabled(true), m_pMainLoop(nullptr), m_waitForSync(waitForSync), m_workerRoutineId(0), m_daysCount(0), m_windowsCount(0), m_intervalsCounter(0), - m_remoteSyncEnabled(true), m_isAssetIdUuid(Waap::Util::isUuid(assetId)), m_shared_storage_host(genError("not set")), m_learning_host(genError("not set")) { - dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId << + dbgInfo(D_WAAP_SERIALIZE) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId << "', owner='" << m_owner << "'"; if (Singleton::exists() && @@ -429,7 +563,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( if (sharedStorageHost != NULL) { m_shared_storage_host = string(sharedStorageHost); } else { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << + dbgWarning(D_WAAP_SERIALIZE) << "shared storage host name(" << SHARED_STORAGE_HOST_ENV_NAME << ") is not set"; @@ -438,7 +572,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase( if (learningHost != NULL) { m_learning_host = string(learningHost); } else { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << + dbgWarning(D_WAAP_SERIALIZE) << "learning host name(" << SHARED_STORAGE_HOST_ENV_NAME << ") is not set"; @@ -515,7 +649,7 @@ string SerializeToLocalAndRemoteSyncBase::getPostDataUrl() { I_InstanceAwareness* instance = Singleton::Consume::by(); Maybe uniqueId = instance->getUniqueID(); - if (uniqueId.ok()) + if (uniqueId.ok() && !uniqueId.unpack().empty()) { agentId += "/" + uniqueId.unpack(); } @@ -530,7 +664,7 @@ void SerializeToLocalAndRemoteSyncBase::setRemoteSyncEnabled(bool enabled) void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "setInterval: from " << m_interval.count() << " to " << + dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " << newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'"; if (newInterval == m_interval) @@ -571,7 +705,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) if (remainingTime > m_interval) { // on load between trigger and offset remaining time is larger than the interval itself remainingTime -= m_interval; - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "adjusting remaining time: " << remainingTime.count(); + dbgDebug(D_WAAP_SERIALIZE) << "adjusting remaining time: " << remainingTime.count(); if (timeBeforeSyncWorker.count() != 0) { auto updateTime = timeBeforeSyncWorker - m_interval; @@ -585,13 +719,13 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) if (remainingTime < ch::seconds(0)) { // syncWorker execution time was so large the remaining time became negative remainingTime = ch::seconds(0); - dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "syncWorker execution time (owner='" << m_owner << + dbgError(D_WAAP_SERIALIZE) << "syncWorker execution time (owner='" << m_owner << "', assetId='" << m_assetId << "') is " << ch::duration_cast(timeAfterSyncWorker - timeBeforeSyncWorker).count() << " seconds, too long to cause negative remainingTime. Waiting 0 seconds..."; } - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "current time: " << timeBeforeSyncWorker.count() << " \u00b5s" << + dbgDebug(D_WAAP_SERIALIZE) << "current time: " << timeBeforeSyncWorker.count() << " \u00b5s" << ": assetId='" << m_assetId << "'" << ", owner='" << m_owner << "'" << ", daysCount=" << m_daysCount << @@ -604,7 +738,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) m_pMainLoop->yield(remainingTime); timeBeforeSyncWorker = timer->getWalltime(); - syncWorker(); + m_pMainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::System, [this]() {syncWorker();}, "Sync worker"); timeAfterSyncWorker = timer->getWalltime(); } }; @@ -618,11 +752,11 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval) bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess() { bool isBackupSyncEnabled = getProfileAgentSettingWithDefault( - true, + false, "appsecLearningSettings.backupLocalSync"); if (!isBackupSyncEnabled) { - dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Local sync is disabled"; + dbgInfo(D_WAAP_SERIALIZE) << "Local sync is disabled"; processData(); saveData(); return true; @@ -630,7 +764,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess() RemoteFilesList rawDataFiles; - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Getting files of all agents"; + dbgTrace(D_WAAP_SERIALIZE) << "Getting files of all agents"; bool isSuccessful = sendObjectWithRetry(rawDataFiles, HTTPMethod::GET, @@ -638,7 +772,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess() if (!isSuccessful) { - dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files"; + dbgError(D_WAAP_SERIALIZE) << "Failed to get the list of files"; return false; } @@ -662,7 +796,7 @@ void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService() RemoteFilesList remoteFiles = getRemoteProcessedFilesList(); if (remoteFiles.getFilesMetadataList().empty()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "no files generated by the remote service were found"; + dbgWarning(D_WAAP_SERIALIZE) << "no files generated by the remote service were found"; continue; } string lastModified = remoteFiles.getFilesMetadataList().begin()->modified; @@ -670,26 +804,26 @@ void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService() { m_lastProcessedModified = lastModified; updateState(remoteFiles.getFilesList()); - dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner << + dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner << ". updated state generated by remote at " << m_lastProcessedModified; return; } } - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "polling for update state timeout. for assetId='" + dbgWarning(D_WAAP_SERIALIZE) << "polling for update state timeout. for assetId='" << m_assetId << "', owner='" << m_owner; localSyncAndProcess(); } void SerializeToLocalAndRemoteSyncBase::syncWorker() { - dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" << + dbgInfo(D_WAAP_SERIALIZE) << "Running the sync worker for assetId='" << m_assetId << "', owner='" << m_owner << "'" << " last modified state: " << m_lastProcessedModified; incrementIntervalsCount(); OrchestrationMode mode = Singleton::exists() ? Singleton::Consume::by()->getOrchestrationMode() : OrchestrationMode::ONLINE; if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() || !postData()) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) + dbgDebug(D_WAAP_SERIALIZE) << "Did not synchronize the data. for asset: " << m_assetId << " Remote URL: " @@ -702,17 +836,17 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() return; } - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data"; + dbgTrace(D_WAAP_SERIALIZE) << "Waiting for all agents to post their data"; waitSync(); // check if learning service is operational if (m_lastProcessedModified == "") { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "check if remote service is operational"; + dbgTrace(D_WAAP_SERIALIZE) << "check if remote service is operational"; RemoteFilesList remoteFiles = getRemoteProcessedFilesList(); if (!remoteFiles.getFilesMetadataList().empty()) { m_lastProcessedModified = remoteFiles.getFilesMetadataList()[0].modified; - dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "First sync by remote service: " << m_lastProcessedModified; + dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service: " << m_lastProcessedModified; } } @@ -721,15 +855,15 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() true, "appsecLearningSettings.remoteServiceEnabled"); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "using remote service: " << isRemoteServiceEnabled; + dbgDebug(D_WAAP_SERIALIZE) << "using remote service: " << isRemoteServiceEnabled; if ((m_lastProcessedModified == "" || !isRemoteServiceEnabled) && !localSyncAndProcess()) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "local sync and process failed"; + dbgWarning(D_WAAP_SERIALIZE) << "local sync and process failed"; return; } if (mode == OrchestrationMode::HYBRID) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "detected running in standalone mode"; + dbgDebug(D_WAAP_SERIALIZE) << "detected running in standalone mode"; I_AgentDetails *agentDetails = Singleton::Consume::by(); I_Messaging *messaging = Singleton::Consume::by(); @@ -738,6 +872,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() MessageMetadata req_md(getLearningHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); bool ok = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/api/sync", @@ -745,14 +880,14 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() MessageCategory::GENERIC, req_md ); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sent learning sync notification ok: " << ok; + dbgDebug(D_WAAP_SERIALIZE) << "sent learning sync notification ok: " << ok; if (!ok) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to send learning notification"; + dbgWarning(D_WAAP_SERIALIZE) << "failed to send learning notification"; } } else { SyncLearningNotificationObject syncNotification(m_assetId, m_type, getWindowId()); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sending sync notification: " << syncNotification; + dbgDebug(D_WAAP_SERIALIZE) << "sending sync notification: " << syncNotification; ReportMessaging( "sync notification for '" + m_assetId + "'", @@ -766,6 +901,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker() if (m_lastProcessedModified != "" && isRemoteServiceEnabled) { + // wait for remote service to process the data + waitSync(); updateStateFromRemoteService(); } } @@ -775,7 +912,7 @@ void SerializeToLocalAndRemoteSyncBase::restore() SerializeToFileBase::restore(); if (!isBase()) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "merge state from remote service"; + dbgTrace(D_WAAP_SERIALIZE) << "merge state from remote service"; mergeProcessedFromRemote(); } } @@ -789,7 +926,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList() if (!isRemoteServiceEnabled) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "remote service is disabled"; + dbgDebug(D_WAAP_SERIALIZE) << "remote service is disabled"; return remoteFiles; } @@ -800,7 +937,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList() if (!isSuccessful) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files"; + dbgWarning(D_WAAP_SERIALIZE) << "Failed to get the list of files"; } return remoteFiles; } @@ -814,12 +951,12 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() { const vector& filesMD = processedFilesList.getFilesMetadataList(); if (filesMD.size() > 1) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "got more than 1 expected processed file"; + dbgWarning(D_WAAP_SERIALIZE) << "got more than 1 expected processed file"; } if (!filesMD.empty()) { m_lastProcessedModified = filesMD[0].modified; } - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "found " << filesMD.size() << " remote service state files. " + dbgTrace(D_WAAP_SERIALIZE) << "found " << filesMD.size() << " remote service state files. " "last modified: " << m_lastProcessedModified; return processedFilesList; @@ -833,11 +970,11 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() if (!isSuccessful) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files"; + dbgDebug(D_WAAP_SERIALIZE) << "Failed to get the list of files"; } else if (!processedFilesList.getFilesList().empty()) { - dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "found state files"; + dbgTrace(D_WAAP_SERIALIZE) << "found state files"; return processedFilesList; } // backward compatibility - try to get backup file with the buggy prefix tenantID/assetID/instanceID/ @@ -846,7 +983,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() pos = bcRemotePath.find('/', pos + 1); if (!Singleton::exists()) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "missing instance of instance awareness," + dbgDebug(D_WAAP_SERIALIZE) << "missing instance of instance awareness," " can't check backward compatibility"; return processedFilesList; } @@ -854,13 +991,13 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() Maybe id = instanceAwareness->getUniqueID(); if (!id.ok()) { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to get instance id err: " << id.getErr() << + dbgDebug(D_WAAP_SERIALIZE) << "failed to get instance id err: " << id.getErr() << ". can't check backward compatibility"; return processedFilesList; } string idStr = id.unpack(); bcRemotePath.insert(pos + 1, idStr + "/"); - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "List of files is empty - trying to get the file from " << + dbgDebug(D_WAAP_SERIALIZE) << "List of files is empty - trying to get the file from " << bcRemotePath; isSuccessful = sendObject( @@ -870,16 +1007,16 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList() if (!isSuccessful) { - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the list of files"; + dbgWarning(D_WAAP_SERIALIZE) << "Failed to get the list of files"; } - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "backwards computability: got " + dbgDebug(D_WAAP_SERIALIZE) << "backwards computability: got " << processedFilesList.getFilesList().size() << " state files"; return processedFilesList; } void SerializeToLocalAndRemoteSyncBase::mergeProcessedFromRemote() { - dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Merging processed data from remote. assetId='" << m_assetId << + dbgDebug(D_WAAP_SERIALIZE) << "Merging processed data from remote. assetId='" << m_assetId << "', owner='" << m_owner << "'"; m_pMainLoop->addOneTimeRoutine( I_MainLoop::RoutineType::Offline, @@ -903,7 +1040,7 @@ SerializeToLocalAndRemoteSyncBase::getLearningHost() m_learning_host = string(learningHost); return learningHost; } - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "learning host is not set. using default"; + dbgWarning(D_WAAP_SERIALIZE) << "learning host is not set. using default"; } return defaultLearningHost; } @@ -919,7 +1056,7 @@ SerializeToLocalAndRemoteSyncBase::getSharedStorageHost() m_shared_storage_host = string(sharedStorageHost); return sharedStorageHost; } - dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "shared storage host is not set. using default"; + dbgWarning(D_WAAP_SERIALIZE) << "shared storage host is not set. using default"; } return defaultSharedStorageHost; } diff --git a/components/security_apps/waap/waap_clib/Signatures.cc b/components/security_apps/waap/waap_clib/Signatures.cc index 82a05fc..030b50e 100755 --- a/components/security_apps/waap/waap_clib/Signatures.cc +++ b/components/security_apps/waap/waap_clib/Signatures.cc @@ -116,7 +116,7 @@ Signatures::Signatures(const std::string& filepath) : ), allowed_text_re(sigsSource["allowed_text_re"].get(), error, "allowed_text_re"), pipe_split_re( - "([\\w\\=\\-\\_\\.\\,\\(\\)\\[\\]\\/\\%\\s]+?)\\||([\\w\\=\\-\\_\\.\\,\\(\\)\\[\\]\\/\\%\\s]+)|\\|()", + "([^|]*)\\||([^|]+)|\\|()", error, "pipe_decode"), semicolon_split_re("([\\w\\=\\-\\_\\.\\,\\(\\)\\%]+?);|([\\w\\=\\-\\_\\.\\,\\(\\)\\%]+)|;()", error, "sem_decode"), diff --git a/components/security_apps/waap/waap_clib/SingleDecision.cc b/components/security_apps/waap/waap_clib/SingleDecision.cc index b7a38d1..4e91ebc 100755 --- a/components/security_apps/waap/waap_clib/SingleDecision.cc +++ b/components/security_apps/waap/waap_clib/SingleDecision.cc @@ -19,7 +19,10 @@ USE_DEBUG_FLAG(D_WAAP); SingleDecision::SingleDecision(DecisionType type): m_type(type), m_log(false), - m_block(false) + m_block(false), + m_ForceLog(false), + m_forceAllow(false), + m_forceBlock(false) {} SingleDecision::~SingleDecision() @@ -35,11 +38,28 @@ bool SingleDecision::shouldLog() const return m_log; } +bool SingleDecision::shouldForceLog() const +{ + return m_ForceLog; +} + bool SingleDecision::shouldBlock() const { return m_block; } +bool SingleDecision::shouldForceAllow() const +{ + dbgTrace(D_WAAP) << "should force allow: " << m_forceAllow; + return m_forceAllow; +} + +bool SingleDecision::shouldForceBlock() const +{ + dbgTrace(D_WAAP) << "should force block: " << m_forceBlock; + return m_forceBlock; +} + void SingleDecision::setLog(bool log) { dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes should log from " << m_log << " to " << log; @@ -51,3 +71,22 @@ void SingleDecision::setBlock(bool block) dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes should block from " << m_block << " to " << block; m_block = block; } + +void SingleDecision::setForceLog(bool overridesLog) +{ + dbgTrace(D_WAAP) << "Decision "<< getTypeStr() << + " changes overrides log from " << m_ForceLog << " to " << overridesLog; + m_ForceLog = overridesLog; +} + +void SingleDecision::setForceAllow(bool allow) +{ + dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes force allow from " << m_forceAllow << " to " << allow; + m_forceAllow = allow; +} + +void SingleDecision::setForceBlock(bool block) +{ + dbgTrace(D_WAAP) << "Decision " << getTypeStr() << " changes force block from " << m_forceBlock << " to " << block; + m_forceBlock = block; +} diff --git a/components/security_apps/waap/waap_clib/SingleDecision.h b/components/security_apps/waap/waap_clib/SingleDecision.h index da39f33..145bb57 100755 --- a/components/security_apps/waap/waap_clib/SingleDecision.h +++ b/components/security_apps/waap/waap_clib/SingleDecision.h @@ -25,15 +25,24 @@ public: void setLog(bool log); void setBlock(bool block); + void setForceLog(bool overridesLog); + void setForceAllow(bool allow); + void setForceBlock(bool block); DecisionType getType() const; bool shouldLog() const; bool shouldBlock() const; + bool shouldForceLog() const; + bool shouldForceAllow() const; + bool shouldForceBlock() const; virtual std::string getTypeStr() const = 0; protected: DecisionType m_type; bool m_log; bool m_block; + bool m_ForceLog; + bool m_forceAllow; + bool m_forceBlock; }; #endif diff --git a/components/security_apps/waap/waap_clib/Telemetry.cc b/components/security_apps/waap/waap_clib/Telemetry.cc index 08105f5..dc6c3e5 100755 --- a/components/security_apps/waap/waap_clib/Telemetry.cc +++ b/components/security_apps/waap/waap_clib/Telemetry.cc @@ -76,6 +76,7 @@ WaapTelemetrics::initMetrics() waf_blocked.report(0); force_and_block_exceptions.report(0); } + void WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data) { @@ -243,6 +244,46 @@ WaapAttackTypesMetrics::updateMetrics(const string &asset_id, const DecisionTele } } +void +WaapAdditionalTrafficTelemetrics::initMetrics() +{ + requests.report(0); + sources.report(0); + blocked.report(0); + temperature_count.report(0); + sources_seen.clear(); +} + +void +WaapAdditionalTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data) +{ + initMetrics(); + + auto is_keep_alive_ctx = Singleton::Consume::by()->get( + "keep_alive_request_ctx" + ); + if (!is_keep_alive_ctx.ok() || !*is_keep_alive_ctx) { + requests.report(1); + } else { + dbgTrace(D_WAAP) << "Not increasing the number of requests due to keep alive"; + } + + if (!data.source.empty()) { + if (sources_seen.find(data.source) == sources_seen.end()) { + sources_seen.insert(data.source); + sources.report(1); + } + } + + if (data.blockType == WAF_BLOCK) { + blocked.report(1); + } + + if (data.temperatureDetected) { + temperature_count.report(1); + } +} + void WaapMetricWrapper::upon(const WaapTelemetryEvent &event) { @@ -268,10 +309,17 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event) attack_types_telemetries ); initializeTelemetryData(asset_id, data, "WAAP traffic telemetry", traffic_telemetries); + initializeTelemetryData( + asset_id, + data, + "WAAP Additional Traffic Telemetry", + additional_traffic_telemetries + ); telemetries[asset_id]->updateMetrics(asset_id, data); attack_types_telemetries[asset_id]->updateMetrics(asset_id, data); traffic_telemetries[asset_id]->updateMetrics(asset_id, data); + additional_traffic_telemetries[asset_id]->updateMetrics(asset_id, data); auto agent_mode = Singleton::Consume::by()->getOrchestrationMode(); string tenant_id = Singleton::Consume::by()->getTenantId(); diff --git a/components/security_apps/waap/waap_clib/TuningDecisions.h b/components/security_apps/waap/waap_clib/TuningDecisions.h index a6f9ee4..cb30afe 100755 --- a/components/security_apps/waap/waap_clib/TuningDecisions.h +++ b/components/security_apps/waap/waap_clib/TuningDecisions.h @@ -60,6 +60,7 @@ private: MessageMetadata req_md(getSharedStorageHost(), 80); req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); auto req_status = messaging->sendSyncMessage( method, uri, @@ -69,11 +70,14 @@ private: ); return req_status.ok(); } + MessageMetadata req_md; + req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN); auto req_status = messaging->sendSyncMessage( method, uri, obj, - MessageCategory::GENERIC + MessageCategory::GENERIC, + req_md ); return req_status.ok(); } diff --git a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc index 273878c..8c4372b 100755 --- a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc +++ b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc @@ -128,6 +128,8 @@ void TypeIndicatorFilter::loadParams(std::shared_ptrgetParamVal("typeIndicators.learnPermanently", "true"); params.learnPermanently = !boost::iequals(learnPermanentlyStr, "false"); + params.maxMemoryUsage = std::stoul(pParams->getParamVal("typeIndicators.maxMemoryUsage", + std::to_string(TYPE_FILTER_CONFIDENCE_MAX_MEMORY_USAGE))); std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true"); bool syncEnabled = !boost::iequals(remoteSyncStr, "false"); diff --git a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h index 79ff770..1a8676b 100755 --- a/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h +++ b/components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h @@ -24,6 +24,7 @@ #define TYPE_FILTER_CONFIDENCE_MIN_INTERVALS 5 #define TYPE_FILTER_CONFIDENCE_THRESHOLD 0.8 #define TYPE_FILTER_INTERVAL_DURATION std::chrono::minutes(60) +#define TYPE_FILTER_CONFIDENCE_MAX_MEMORY_USAGE (40 * 1024 * 1024) // 40MB class TypeIndicatorFilter : public IndicatorFilterBase { diff --git a/components/security_apps/waap/waap_clib/WaapAssetState.cc b/components/security_apps/waap/waap_clib/WaapAssetState.cc index 536ca19..f360b3f 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetState.cc +++ b/components/security_apps/waap/waap_clib/WaapAssetState.cc @@ -446,13 +446,14 @@ WaapAssetState::WaapAssetState(std::shared_ptr signatures, // update orig_size and orig_capacity after string copy orig_size = text.size(); orig_capacity = text.capacity(); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (1) (after filterUTF7) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (1) (after filterUTF7) '" << text << + "' size: " << text.size(); // 2. Replace %xx sequences by their single-character equivalents. // Also replaces '+' symbol by space character. // Python equivalent: text = urllib.unquote_plus(text) text.erase(unquote_plus(text.begin(), text.end()), text.end()); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (2) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (2) '" << text << "' size: " << text.size(); fixBreakingSpace(text); @@ -460,38 +461,38 @@ WaapAssetState::WaapAssetState(std::shared_ptr signatures, // remove all characters whose ASCII code is >=128. // Python equivalent: text.encode('ascii',errors='ignore') filterUnicode(text); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (3) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (3) '" << text << "' size: " << text.size(); // 4. oh shi?... should I handle unicode html entities (python's htmlentitydefs module)??? // Python equivalent: text = HTMLParser.HTMLParser().unescape(text) text.erase(escape_html(text.begin(), text.end()), text.end()); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (4) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (4) '" << text << "' size: " << text.size(); // 5. Apply backslash escaping (like in C) // Python equivalent: text = text.decode('string_escape') text.erase(escape_backslashes(text.begin(), text.end()), text.end()); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (5) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (5) '" << text << "' size: " << text.size(); // 6. remove all unicode characters from string. Basically, // remove all characters whose ASCII code is >=128. // Python equivalent: text.encode('ascii',errors='ignore') filterUnicode(text); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (6) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (6) '" << text << "' size: " << text.size(); // 7. Replace %xx sequences by their single-character equivalents. // Also replaces '+' symbol by space character. // Python equivalent: text = urllib.unquote_plus(text) text.erase(unquote_plus(text.begin(), text.end()), text.end()); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (7) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (7) '" << text << "' size: " << text.size(); unescapeUnicode(text); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "after unescapeUnicode '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "after unescapeUnicode '" << text << "' size: " << text.size(); // 8. remove all unicode characters from string. Basically, // remove all characters whose ASCII code is >=128. // Python equivalent: text.encode('ascii',errors='ignore') filterUnicode(text); - dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (8) '" << text << "'"; + dbgTrace(D_WAAP_SAMPLE_PREPROCESS) << "unescape: (8) '" << text << "' size: " << text.size(); // 9. ??? // @@ -530,7 +531,7 @@ WaapAssetState::WaapAssetState(std::shared_ptr signatures, << AlertInfo(AlertTeam::CORE, "WAAP sample processing") << "unescape: original size=" << orig_size << " capacity=" << orig_capacity << " new size=" << text.size() << " capacity=" << text.capacity() - << " text='" << text << "'"; + << " text='" << text << "'" << " orig text='" << s << "'"; return text; } @@ -1098,6 +1099,9 @@ WaapAssetState::apply( // Detect long text spans, and also any-length spans that end with file extensions such as ".jpg" bool longTextFound = m_Signatures->longtext_re.hasMatch(res.unescaped_line); + // When this flag remains false until the last evasion handling, a second unescape is performed + bool evasion_detected = false; + if (longTextFound) { dbgTrace(D_WAAP_SAMPLE_SCAN) << "longtext found"; } @@ -1297,6 +1301,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1326,6 +1331,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1409,6 +1415,42 @@ WaapAssetState::apply( dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res); } + bool tab_evasion = (res.unescaped_line.find('\t') != std::string::npos); + + if (tab_evasion) { + dbgTrace(D_WAAP_EVASIONS) << "Tab character evasion detected"; + + // Create normalized version with tabs removed + std::string unescaped = res.unescaped_line; + + // Remove all tab characters to normalize the string + std::string::iterator end_pos = std::remove(unescaped.begin(), unescaped.end(), '\t'); + unescaped.erase(end_pos, unescaped.end()); + + size_t kwCount = res.keyword_matches.size(); + + if (res.unescaped_line != unescaped) { + SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); + checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, + res.found_patterns, longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, + longTextFound, binaryDataFound); + } + + if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + // If new keywords were found, add a specific indication + res.keyword_matches.push_back("tab_character_evasion"); + + // Recalculate repetition and/or probing indicators + unsigned int newWordsCount = 0; + calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, + newWordsCount); + // Take minimal words count because empirically it means evasion was probably successfully decoded + wordsCount = std::min(wordsCount, newWordsCount); + } + } bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys( "quotes_space_ev_fast_reg", @@ -1471,6 +1513,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1499,7 +1542,7 @@ WaapAssetState::apply( longTextFound, binaryDataFound); } - + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1613,6 +1656,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1644,6 +1688,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1675,6 +1720,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1706,6 +1752,7 @@ WaapAssetState::apply( } if (kwCount != res.keyword_matches.size() && !binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1715,6 +1762,74 @@ WaapAssetState::apply( } } + // Detect evasion for command injection vulnerability using encoded characters (such as ^). + // example: "c^m^d /c d^i^r" -> "cmd /c dir" + // if '^' is found in the line, remove it and rescan + // search in scan result for any keyword with "os", "cmd", "exec" "command" or "shell" in it + // if not found, we don't want this evasion + // if found, we want to use its results + // Note: this is not a perfect solution, but it is better than nothing + if (line.find('^') != std::string::npos) { + dbgTrace(D_WAAP_EVASIONS) << "Windows command injection evasion suspected"; + + std::string unescaped = line; + bool is_win_cmd_evasion = false; + // remove all occurances of '^' character + unescaped.erase(std::remove(unescaped.begin(), unescaped.end(), '^'), unescaped.end()); + dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'"; + if (!unescaped.empty()) { + std::vector evKeywordMatches(res.keyword_matches); + std::vector evRegexMatches(res.regex_matches); + Waap::Util::map_of_stringlists_t evFoundPatterns(res.found_patterns); + bool evLongTextFound = longTextFound; + bool evBinaryDataFound = binaryDataFound; + + if (line != unescaped) { + SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); + checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, evKeywordMatches, + res.found_patterns, evLongTextFound, evBinaryDataFound); + checkRegex(unescapedSample, m_Signatures->words_regex, evKeywordMatches, evFoundPatterns, + evLongTextFound, evBinaryDataFound); + checkRegex(unescapedSample, m_Signatures->pattern_regex, evRegexMatches, evFoundPatterns, + evLongTextFound, evBinaryDataFound); + } + + if (evKeywordMatches.size() != res.keyword_matches.size()) { + if (Waap::Util::find_in_map_of_stringlists_keys("os_commands", evFoundPatterns)) { + is_win_cmd_evasion = true; + } else { + for (const auto &kw : evKeywordMatches) { + if (kw.size() < 2 || + str_contains(kw, "cmd") || + str_contains(kw, "command") || + str_contains(kw, "os_") || + str_contains(kw, "exec")) { + is_win_cmd_evasion = true; + break; + } + } + } + } + + if (is_win_cmd_evasion) { + dbgTrace(D_WAAP_EVASIONS) << "Evasion and relevant matches found, updating results"; + // found relevant keywords after unescaping, set to new matches + res.keyword_matches = evKeywordMatches; + res.regex_matches = evRegexMatches; + res.found_patterns = evFoundPatterns; + longTextFound = evLongTextFound; + binaryDataFound = evBinaryDataFound; + // Recalculate repetition and/or probing indicators + unsigned int newWordsCount = 0; + calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, + newWordsCount); + // Take minimal words count because empirically it means evasion was probably successfully decoded + wordsCount = std::min(wordsCount, newWordsCount); + } else { + dbgTrace(D_WAAP_EVASIONS) << "Evasion not relevant, will not apply to results"; + } + } + } // python: escape ='hi_acur_fast_reg_evasion' in found_patterns bool escape = Waap::Util::find_in_map_of_stringlists_keys("evasion", res.found_patterns); @@ -1757,6 +1872,7 @@ WaapAssetState::apply( escape = false; } else if (!binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1833,6 +1949,7 @@ WaapAssetState::apply( escape = false; } else if (!binaryDataFound) { + evasion_detected = true; // Recalculate repetition and/or probing indicators unsigned int newWordsCount = 0; calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, @@ -1841,6 +1958,42 @@ WaapAssetState::apply( wordsCount = std::min(wordsCount, newWordsCount); } } + + // Handle second URL encoding evasion + if (!evasion_detected && + !binaryDataFound && + !longTextFound && + Waap::Util::containsPercentEncoding(res.unescaped_line)) { + dbgTrace(D_WAAP_EVASIONS) << "Second URL encoding evasion detected"; + + std::string unescaped = unescape(res.unescaped_line); + size_t kwCount = res.keyword_matches.size(); + + if (res.unescaped_line != unescaped) { + SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); + checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, + res.found_patterns, longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, + longTextFound, binaryDataFound); + } + + if (kwCount == res.keyword_matches.size()) { + // Remove the evasion keyword if no real evasion found + keywordsToRemove.push_back("evasion"); + } + else if (!binaryDataFound) { + evasion_detected = true; + // Recalculate repetition and/or probing indicators + unsigned int newWordsCount = 0; + calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, + newWordsCount); + // Take minimal words count because empirically it means evasion was probably succesfully decoded + wordsCount = std::min(wordsCount, newWordsCount); + } + } + dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res); // Remove evasion keywords that should not be reported because there's no real evasion found if (!keywordsToRemove.empty()) { diff --git a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc index 2f98558..2b5167d 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc @@ -83,7 +83,6 @@ void WaapConfigApplication::load(cereal::JSONInputArchive& ar) assets_ids_aggregation.insert(m_assetId); } - bool WaapConfigApplication::operator==(const WaapConfigApplication& other) const { const WaapConfigBase* configBase = this; diff --git a/components/security_apps/waap/waap_clib/WaapConfigBase.cc b/components/security_apps/waap/waap_clib/WaapConfigBase.cc index f38ef1a..1e059dd 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigBase.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigBase.cc @@ -132,7 +132,7 @@ void WaapConfigBase::loadOverridePolicy(cereal::JSONInputArchive& ar) try { m_overridePolicy = std::make_shared(ar); } - catch (std::runtime_error& e) { + catch (const cereal::Exception &e) { ar.setNextName(nullptr); dbgWarning(D_WAAP) << failMessage << e.what(); m_overridePolicy = nullptr; @@ -329,37 +329,38 @@ const std::string& WaapConfigBase::get_AssetName() const return m_assetName; } -const std::string& WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const +const string & +WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const { - + dbgTrace(D_WAAP) << "get practice ID of decision type: " << practiceType; switch (practiceType) { - case DecisionType::AUTONOMOUS_SECURITY_DECISION: - return m_practiceId; + case DecisionType::AUTONOMOUS_SECURITY_DECISION: break; default: - dbgError(D_WAAP) - << "Can't find practice type for practice ID by practice: " - << practiceType - << ", return web app practice ID"; - return m_practiceId; + dbgDebug(D_WAAP) + << "Can't find practice type for practice ID by practice: " + << practiceType + << ", return web app practice ID"; } + return m_practiceId; } -const std::string& WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const +const string & +WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const { + dbgTrace(D_WAAP) << "get practice name of decision type: " << practiceType; switch (practiceType) { - case DecisionType::AUTONOMOUS_SECURITY_DECISION: - return m_practiceName; + case DecisionType::AUTONOMOUS_SECURITY_DECISION: break; default: - dbgError(D_WAAP) - << "Can't find practice type for practice name by practice: " - << practiceType - << ", return web app practice name"; - return m_practiceName; + dbgDebug(D_WAAP) + << "Can't find practice type for practice name by practice: " + << practiceType + << ", return web app practice name"; } + return m_practiceName; } const std::string& WaapConfigBase::get_RuleId() const diff --git a/components/security_apps/waap/waap_clib/WaapOverride.cc b/components/security_apps/waap/waap_clib/WaapOverride.cc index 9f1d670..7111c3d 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.cc +++ b/components/security_apps/waap/waap_clib/WaapOverride.cc @@ -82,11 +82,48 @@ State::State() : forceBlockIds(), bForceException(false), forceExceptionIds(), - bIgnoreLog(false), + bSupressLog(false), bSourceIdentifierOverride(false), sSourceIdentifierMatch("") { } +bool ExceptionsByPractice::operator==(const ExceptionsByPractice &other) const +{ + return m_web_app_ids == other.m_web_app_ids && + m_api_protect_ids == other.m_api_protect_ids && + m_anti_bot_ids == other.m_anti_bot_ids; +} + +const std::vector& ExceptionsByPractice::getExceptionsOfPractice(DecisionType practiceType) const +{ + switch (practiceType) + { + + case DecisionType::AUTONOMOUS_SECURITY_DECISION: + return m_web_app_ids; + default: + dbgError(D_WAAP) << + "Can't find practice type for exceptions by practice: " << + practiceType << + ", return web app exceptions"; + return m_web_app_ids; + } +} + +const std::set& ExceptionsByPractice::getAllExceptions() const +{ + return m_all_ids; +} + +bool ExceptionsByPractice::isIDInWebApp(const std::string &id) const +{ + auto it = std::find(m_web_app_ids.begin(), m_web_app_ids.end(), id); + if (it != m_web_app_ids.end()) { + dbgTrace(D_WAAP) << "rule id is in web application exceptions by practice: " << id; + return true; + } + return false; +} } } diff --git a/components/security_apps/waap/waap_clib/WaapOverride.h b/components/security_apps/waap/waap_clib/WaapOverride.h index 77c76ae..977f4f9 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.h +++ b/components/security_apps/waap/waap_clib/WaapOverride.h @@ -21,6 +21,7 @@ #include #include "debug.h" #include "CidrMatch.h" +#include "DecisionType.h" #include "RegexComparator.h" USE_DEBUG_FLAG(D_WAAP_OVERRIDE); @@ -264,6 +265,15 @@ public: template void serialize(_A &ar) { + try { + ar(cereal::make_nvp("parsedMatch", m_match)); + } + catch(const cereal::Exception &e) + { + dbgDebug(D_WAAP_OVERRIDE) << "An override rule was not loaded, parsedMatch error:" << e.what(); + isValid = false; + } + try { ar(cereal::make_nvp("id", m_id)); } @@ -272,7 +282,6 @@ public: dbgDebug(D_WAAP_OVERRIDE) << "An override rule has no id."; m_id.clear(); } - ar(cereal::make_nvp("parsedMatch", m_match)); if (!m_match.isValidMatch()) { dbgDebug(D_WAAP_OVERRIDE) << "An override rule was not load"; isValid = false; @@ -342,14 +351,50 @@ private: bool isValid; }; +class ExceptionsByPractice +{ +public: + template + void serialize(_A& ar) + { + ar( + cereal::make_nvp("WebApplicationExceptions", m_web_app_ids), + cereal::make_nvp("APIProtectionExceptions", m_api_protect_ids), + cereal::make_nvp("AntiBotExceptions", m_anti_bot_ids) + ); + m_all_ids.insert(m_web_app_ids.begin(), m_web_app_ids.end()); + m_all_ids.insert(m_api_protect_ids.begin(), m_api_protect_ids.end()); + m_all_ids.insert(m_anti_bot_ids.begin(), m_anti_bot_ids.end()); + } + + bool operator==(const ExceptionsByPractice &other) const; + const std::vector& getExceptionsOfPractice(DecisionType practiceType) const; + const std::set& getAllExceptions() const; + bool isIDInWebApp(const std::string &id) const; +private: + std::vector m_web_app_ids; + std::vector m_api_protect_ids; + std::vector m_anti_bot_ids; + std::set m_all_ids; +}; + class Policy { public: template Policy(_A &ar) { + try { + ar( + cereal::make_nvp("exceptionsPerPractice", m_exceptionsByPractice) + ); + } + catch (std::runtime_error & e) { + ar.setNextName(nullptr); + dbgInfo(D_WAAP_OVERRIDE) << "Failed to load exceptions per practice, error: ", e.what(); + m_exceptionsByPractice = ExceptionsByPractice(); + } std::vector rules; ar(cereal::make_nvp("overrides", rules)); m_isOverrideResponse = false; - for (std::vector::const_iterator it = rules.begin(); it != rules.end(); ++it) { const Waap::Override::Rule& rule = *it; if (!rule.isValidRule()) { @@ -379,6 +424,14 @@ public: const std::vector& rules = requestOverrides ? m_RequestOverrides : m_ResponseOverrides; dbgTrace(D_WAAP_OVERRIDE) << "Start matching override rules ..."; for (const Waap::Override::Rule &rule : rules) { + if (m_exceptionsByPractice.getAllExceptions().size() > 0 && + !m_exceptionsByPractice.isIDInWebApp(rule.getId()) + ) { + dbgInfo(D_WAAP_OVERRIDE) + << "match rule id is not in web application exceptions by practice: " + << rule.getId(); + continue; + } dbgTrace(D_WAAP_OVERRIDE) << "Matching override rule ..."; rule.match(testFunctor, matchedBehaviors, matchedOverrideIds); } @@ -389,9 +442,17 @@ public: return m_isOverrideResponse; } + bool isValidRules() { + return !m_RequestOverrides.empty() || !m_ResponseOverrides.empty(); + } + + const ExceptionsByPractice& getExceptionsByPractice() const { + return m_exceptionsByPractice; + } private: std::vector m_RequestOverrides; //overrides that change request data std::vector m_ResponseOverrides; //overrides that change response/log data + ExceptionsByPractice m_exceptionsByPractice; bool m_isOverrideResponse; }; @@ -403,7 +464,7 @@ struct State { bool bForceException; std::set forceExceptionIds; // overrides decision in case log should be ignored - bool bIgnoreLog; + bool bSupressLog; // user identfier override to be applied bool bSourceIdentifierOverride; std::string sSourceIdentifierMatch; @@ -437,8 +498,8 @@ struct State { if (matchedBehavior.getLog() == "ignore") { - dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bIgnoreLog due to override behavior."; - bIgnoreLog = true; + dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bSupressLog due to override behavior."; + bSupressLog = true; } sSourceIdentifierMatch = matchedBehavior.getSourceIdentifier(); diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc index a2892bc..3dadfbb 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc @@ -26,8 +26,7 @@ namespace Waap { errorLimiter(false), rateLimiting(false), collectResponseForLog(false), - applyOverride(false), - triggerReport(false) + applyOverride(false) { } @@ -40,12 +39,11 @@ namespace Waap { " RateLimiting=" << rateLimiting << " ErrorLimiter=" << errorLimiter << " collectResponseForLog=" << collectResponseForLog << - " applyOverride=" << applyOverride << - " triggerReport=" << triggerReport; + " applyOverride=" << applyOverride; return openRedirect || errorDisclosure || rateLimiting || errorLimiter || - collectResponseForLog || applyOverride || triggerReport; + collectResponseForLog || applyOverride; } void @@ -93,14 +91,6 @@ namespace Waap { applyOverride = flag; } - void - ResponseInspectReasons::setTriggerReport(bool flag) - { - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setTriggerReport) " << triggerReport << " to " << - flag; - triggerReport = flag; - } - bool ResponseInspectReasons::getApplyOverride(void) { diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h index 51937fd..e2e58e0 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h @@ -25,7 +25,6 @@ public: void setErrorLimiter(bool flag); void setCollectResponseForLog(bool flag); void setApplyOverride(bool flag); - void setTriggerReport(bool flag); bool getApplyOverride(void); private: @@ -35,7 +34,6 @@ private: bool rateLimiting; bool collectResponseForLog; bool applyOverride; - bool triggerReport; }; } diff --git a/components/security_apps/waap/waap_clib/WaapTrigger.cc b/components/security_apps/waap/waap_clib/WaapTrigger.cc index 131fee9..a253ed4 100755 --- a/components/security_apps/waap/waap_clib/WaapTrigger.cc +++ b/components/security_apps/waap/waap_clib/WaapTrigger.cc @@ -75,5 +75,53 @@ bool Policy::operator==(const Policy &other) const return triggers == other.triggers; } +bool TriggersByPractice::operator==(const TriggersByPractice &other) const +{ + return m_web_app_ids == other.m_web_app_ids && + m_api_protect_ids == other.m_api_protect_ids && + m_anti_bot_ids == other.m_anti_bot_ids; +} + +const std::vector& TriggersByPractice::getTriggersByPractice(DecisionType practiceType) const +{ + switch (practiceType) + { + case DecisionType::AUTONOMOUS_SECURITY_DECISION: + return m_web_app_ids; + default: + dbgError(D_WAAP) << + "Can't find practice type for triggers by practice: " << + practiceType << + ", return web app triggers"; + return m_web_app_ids; + } +} + +const std::vector& TriggersByPractice::getAllTriggers() const +{ + return m_all_ids; +} + +bool WebUserResponseByPractice::operator==(const WebUserResponseByPractice &other) const +{ + return m_web_app_ids == other.m_web_app_ids && + m_api_protect_ids == other.m_api_protect_ids && + m_anti_bot_ids == other.m_anti_bot_ids; +} + +const std::vector& WebUserResponseByPractice::getResponseByPractice(DecisionType practiceType) const +{ + switch (practiceType) + { + case DecisionType::AUTONOMOUS_SECURITY_DECISION: + return m_web_app_ids; + default: + dbgDebug(D_WAAP) + << "Can't find practice type for triggers by practice: " + << practiceType + << ", return web app triggers"; + return m_web_app_ids; + } +} } } diff --git a/components/security_apps/waap/waap_clib/WaapTrigger.h b/components/security_apps/waap/waap_clib/WaapTrigger.h index 72bf4cf..01cbf55 100755 --- a/components/security_apps/waap/waap_clib/WaapTrigger.h +++ b/components/security_apps/waap/waap_clib/WaapTrigger.h @@ -19,6 +19,7 @@ #include #include #include "debug.h" +#include "DecisionType.h" USE_DEBUG_FLAG(D_WAAP); @@ -143,15 +144,86 @@ struct Trigger { std::shared_ptr log; }; +class TriggersByPractice +{ +public: + template + void serialize(_A& ar) + { + ar( + cereal::make_nvp("WebApplicationTriggers", m_web_app_ids), + cereal::make_nvp("APIProtectionTriggers", m_api_protect_ids), + cereal::make_nvp("AntiBotTriggers", m_anti_bot_ids) + ); + m_all_ids.insert(m_all_ids.end(), m_web_app_ids.begin(), m_web_app_ids.end()); + m_all_ids.insert(m_all_ids.end(), m_api_protect_ids.begin(), m_api_protect_ids.end()); + m_all_ids.insert(m_all_ids.end(), m_anti_bot_ids.begin(), m_anti_bot_ids.end()); + } + + bool operator==(const TriggersByPractice &other) const; + const std::vector& getTriggersByPractice(DecisionType practiceType) const; + const std::vector& getAllTriggers() const; +private: + std::vector m_web_app_ids; + std::vector m_api_protect_ids; + std::vector m_anti_bot_ids; + std::vector m_all_ids; +}; + +class WebUserResponseByPractice +{ +public: + template + void serialize(_A& ar) + { + ar( + cereal::make_nvp("WebApplicationResponse", m_web_app_ids), + cereal::make_nvp("APIProtectionResponse", m_api_protect_ids), + cereal::make_nvp("AntiBotResponse", m_anti_bot_ids) + ); + } + + bool operator==(const WebUserResponseByPractice &other) const; + const std::vector& getResponseByPractice(DecisionType practiceType) const; +private: + std::vector m_web_app_ids; + std::vector m_api_protect_ids; + std::vector m_anti_bot_ids; +}; + struct Policy { template Policy(_A &ar) { - ar(cereal::make_nvp("triggers", triggers)); + try { + ar( + cereal::make_nvp("triggersPerPractice", triggersByPractice) + ); + } + catch (std::runtime_error &e) { + ar.setNextName(nullptr); + dbgInfo(D_WAAP) << "Failed to load triggers per practice, error: " << e.what(); + triggersByPractice = TriggersByPractice(); + } + try { + ar( + cereal::make_nvp("webUserResponsePerPractice", responseByPractice) + ); + } + catch (std::runtime_error &e) { + ar.setNextName(nullptr); + dbgInfo(D_WAAP) << "Failed to load web user response per practice, error: " << e.what(); + responseByPractice = WebUserResponseByPractice(); + } + ar( + cereal::make_nvp("triggers", triggers) + ); } bool operator==(const Policy &other) const; std::vector triggers; + TriggersByPractice triggersByPractice; + WebUserResponseByPractice responseByPractice; }; } diff --git a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc index 7e94df0..2e7daf6 100755 --- a/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc +++ b/components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc @@ -229,6 +229,7 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val) case '(': case ')': case '|': + case '+': break; default: // Only alphanumeric characters and characters listed above are allowed, anything else disables diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 43480ff..5370567 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -45,8 +45,6 @@ #include #include "ParserDelimiter.h" #include "OpenRedirectDecision.h" -#include "DecisionType.h" -#include "generic_rulebase/triggers_config.h" #include "config.h" #include "LogGenWrapper.h" #include "reputation_features_events.h" @@ -59,6 +57,7 @@ USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION); USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); USE_DEBUG_FLAG(D_WAAP_HEADERS); + using namespace ReportIS; using namespace std; @@ -92,9 +91,6 @@ void Waf2Transaction::start_response(int response_status, int http_version) dbgTrace(D_WAAP) << "[transaction:" << this << "] start_response(response_status=" << response_status << "," << " http_version=" << http_version << ")"; m_responseStatus = response_status; - if (m_triggerReport) { - m_responseInspectReasons.setTriggerReport(false); - } if(m_responseStatus == 404) { @@ -324,11 +320,12 @@ Waf2Transaction::Waf2Transaction() : m_responseStatus(0), m_responseInspectReasons(), m_responseInjectReasons(), + m_practiceSubType("Web Application"), m_index(-1), - m_triggerLog(), m_triggerReport(false), is_schema_validation(false), - m_waf2TransactionFlags() + m_waf2TransactionFlags(), + m_temperature_detected(false) { m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = 0; I_TimeGet *timeGet = Singleton::Consume::by(); @@ -363,11 +360,12 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState m_responseStatus(0), m_responseInspectReasons(), m_responseInjectReasons(), + m_practiceSubType("Web Application"), m_index(-1), - m_triggerLog(), m_triggerReport(false), is_schema_validation(false), - m_waf2TransactionFlags() + m_waf2TransactionFlags(), + m_temperature_detected(false) { I_TimeGet *timeGet = Singleton::Consume::by(); m_entry_time = chrono::duration_cast(timeGet->getMonotonicTime()); @@ -574,6 +572,8 @@ void Waf2Transaction::start() { hdrs_map.clear(); m_request_body.clear(); m_response_body.clear(); + m_overrideStateByPractice.clear(); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = Waap::Override::State(); } void Waf2Transaction::set_transaction_time(const char* log_time) { @@ -639,6 +639,7 @@ bool Waf2Transaction::checkIsScanningRequired() result = true; } } + return result; } @@ -828,7 +829,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan } dbgTrace(D_WAAP) << "should_decode % = " << should_decode; - ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode); + ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode, false); up.push(p, buff_len); up.finish(); m_deepParser.m_key.pop(tag.c_str()); @@ -875,7 +876,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len) if (value_len > 0) { dbgTrace(D_WAAP_HEADERS) << "[transaction:" << this << "] scanning the cookie value"; m_deepParser.m_key.push("cookie", 6); - ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';', false); + ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';', false, false); cookieValueParser.push(value, value_len); cookieValueParser.finish(); m_deepParser.m_key.pop("cookie"); @@ -969,10 +970,7 @@ void Waf2Transaction::scanSpecificHeader(const char* name, int name_len, const c parseUnknownHeaderName(name, name_len); // Scan unknown headers whose values do not match "clean generic header" pattern. // Note that we do want to process special header named x-chkp-csrf-token header - it is treated specially. - if (!m_pWaapAssetState->getSignatures()->good_header_value_re.hasMatch(std::string(value, value_len)) || - headerName == "x-chkp-csrf-token" || headerType == HeaderType::OTHER_KNOWN_HEADERS) { - parseGenericHeaderValue(headerName, value, value_len); - } + parseGenericHeaderValue(headerName, value, value_len); break; } case HeaderType::USER_AGENT_HEADER: { @@ -1104,9 +1102,6 @@ void Waf2Transaction::end_request_hdrs() { if (m_isScanningRequired) { createUserLimitsState(); detectHeaders(); - if (isUserLimitReached()) { - return; - } } // Scan URL and url query if (m_isScanningRequired && !m_processedUri) { @@ -1117,6 +1112,16 @@ void Waf2Transaction::end_request_hdrs() { scanHeaders(); } + // Chack after scanning the URL and headers so we have the valuse for state. + if (m_siteConfig != NULL) + { + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(m_siteConfig); + } + + if(m_isScanningRequired && isUserLimitReached()) { + return; + } + if(m_siteConfig != NULL) { // Create rate limiting policy (lazy, on first request) @@ -1275,12 +1280,6 @@ void Waf2Transaction::end_request() { // Enable response headers processing if response scanning is enabled in policy auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL; m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable); - - auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL; - if (isTriggerReportExists(triggerPolicy)) { - m_responseInspectReasons.setTriggerReport(true); - dbgTrace(D_WAAP) << "setTriggerReport(true)"; - } } void Waf2Transaction::extractEnvSourceIdentifier() @@ -1360,7 +1359,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){ std::string body(data, data_len); if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body)) { - dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false"; + dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false. Html regex not matched."; return false; } dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: true"; @@ -1532,7 +1531,7 @@ Waf2Transaction::decideAfterHeaders() } m_isHeaderOverrideScanRequired = true; - m_overrideState = getOverrideState(sitePolicy); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy); // Select scores pool by location (but use forced pool when forced) std::string realPoolName = @@ -1551,7 +1550,7 @@ Waf2Transaction::decideAfterHeaders() UNKNOWN_TYPE ); - return finalizeDecision(sitePolicy, shouldBlock); + return shouldBlock; } @@ -1584,7 +1583,7 @@ Waf2Transaction::decideFinal( if (WaapConfigAPI::getWaapAPIConfig(ngenAPIConfig)) { dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant API configuration from the I/S"; sitePolicy = &ngenAPIConfig; - m_overrideState = getOverrideState(sitePolicy); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy); // User limits shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP); @@ -1592,7 +1591,7 @@ Waf2Transaction::decideFinal( else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) { dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S"; sitePolicy = &ngenSiteConfig; - m_overrideState = getOverrideState(sitePolicy); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy); shouldBlock = decideAutonomousSecurity( *sitePolicy, @@ -1600,7 +1599,8 @@ Waf2Transaction::decideFinal( false, transactionResult, realPoolName, - fpClassification); + fpClassification + ); // CSRF Protection auto csrfPolicy = m_siteConfig ? m_siteConfig->get_CsrfPolicy() : nullptr; @@ -1613,62 +1613,23 @@ Waf2Transaction::decideFinal( if (mode == 2) { decide( - m_overrideState.bForceBlock, - m_overrideState.bForceException, + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock, + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException, mode ); shouldBlock = isSuspicious(); } - return finalizeDecision(sitePolicy, shouldBlock); -} - -int -Waf2Transaction::finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock) -{ - auto decision = std::dynamic_pointer_cast( - m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); - // Send log - if (sitePolicy) - { - // auto reject should have default threat level info and above - if (m_overrideState.bForceBlock && decision->getThreatLevel() == ThreatLevel::NO_THREAT) - { - decision->setThreatLevel(ThreatLevel::THREAT_INFO); - } - } - - if (m_overrideState.bForceBlock) { - dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): setting shouldBlock to true due to override"; - shouldBlock = true; // BLOCK - } - else if (m_overrideState.bForceException) { - dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): setting shouldBlock to false due to override"; - shouldBlock = false; // PASS - } - - if (m_siteConfig) { - const std::shared_ptr triggerPolicy = m_siteConfig->get_TriggerPolicy(); - if (triggerPolicy) { - const std::shared_ptr triggerLog = getTriggerLog(triggerPolicy); - if (triggerLog && shouldSendExtendedLog(triggerLog)) - { - m_responseInspectReasons.setCollectResponseForLog(true); - } - } - } - - dbgTrace(D_WAAP) << "Waf2Transaction::finalizeDecision(): returning shouldBlock: " << shouldBlock; + dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): returning shouldBlock: " << shouldBlock; return shouldBlock; } void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, - const std::shared_ptr &triggerLog, + const LogTriggerConf &triggerLog, bool shouldBlock, const std::string& logOverride, const std::string& incidentType, - const std::string& practiceID, - const std::string& practiceName) const + DecisionType practiceType) const { auto env = Singleton::Consume::by(); auto active_id = env->get("ActiveTenantId"); @@ -1686,10 +1647,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId()); if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName()); - const auto& autonomousSecurityDecision = std::dynamic_pointer_cast( - m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); - bool send_extended_log = shouldSendExtendedLog(triggerLog); - if (triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) { + const auto& decision = m_waapDecision.getDecision(practiceType); + bool send_extended_log = shouldSendExtendedLog(triggerLog, practiceType); + if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webUrlPath) || decision->shouldForceLog()) { std::string httpUriPath = m_uriPath; if (httpUriPath.length() > MAX_LOG_FIELD_SIZE) @@ -1699,7 +1659,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, waapLog << LogField("httpUriPath", httpUriPath, LogFieldOption::XORANDB64); } - if (triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) { + if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webUrlQuery) || decision->shouldForceLog()) { std::string uriQuery = m_uriQuery; if (uriQuery.length() > MAX_LOG_FIELD_SIZE) { @@ -1707,16 +1667,16 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, } waapLog << LogField("httpUriQuery", uriQuery, LogFieldOption::XORANDB64); } - if (triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) { + if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webHeaders) || decision->shouldForceLog()) { waapLog << LogField("httpRequestHeaders", logHeadersStr(), LogFieldOption::XORANDB64); } // Log http response code if it is known - if (m_responseStatus != 0 && send_extended_log && triggerLog->responseCode) { + if (m_responseStatus != 0 && send_extended_log && triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode)) { waapLog << LogField("httpResponseCode", std::to_string(m_responseStatus)); } // Count of bytes available to send to the log - std::string requestBodyToLog = (triggerLog->webBody) ? + std::string requestBodyToLog = (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webBody)) ? m_request_body : std::string(); std::string responseBodyToLog = m_response_body; if (!shouldBlock && responseBodyToLog.empty()) @@ -1748,7 +1708,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, waapLog << LogField("httpRequestBody", requestBodyToLog, LogFieldOption::XORANDB64); } - if (!responseBodyToLog.empty() && send_extended_log && triggerLog->responseBody) + if (!responseBodyToLog.empty() && send_extended_log && triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)) { waapLog << LogField("httpResponseBody", responseBodyToLog, LogFieldOption::XORANDB64); } @@ -1757,10 +1717,10 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, waapLog << LogField("securityAction", shouldBlock ? "Prevent" : "Detect"); waapLog << LogField("waapOverride", logOverride); waapLog << LogField("practiceType", "Threat Prevention"); - waapLog << LogField("practiceSubType", m_siteConfig->get_PracticeSubType()); + waapLog << LogField("practiceSubType", m_practiceSubType); waapLog << LogField("ruleName", m_siteConfig->get_RuleName()); - waapLog << LogField("practiceId", practiceID); - waapLog << LogField("practiceName", practiceName); + waapLog << LogField("practiceId", m_siteConfig->get_PracticeIdByPactice(practiceType)); + waapLog << LogField("practiceName", m_siteConfig->get_PracticeNameByPactice(practiceType)); waapLog << LogField("waapIncidentType", incidentType); // Registering this value would append the list of matched override IDs to the unified log @@ -1786,19 +1746,24 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, void Waf2Transaction::sendLog() { - dbgFlow(D_WAAP); + dbgFlow(D_WAAP) << "send log"; m_waapDecision.orderDecisions(); if (m_siteConfig == NULL) { dbgWarning(D_WAAP) << "Waf2Transaction::sendLog: no site policy associated with transaction - not sending a log"; return; } + DecisionType decision_type = m_waapDecision.getHighestPriorityDecisionToLog(); + dbgTrace(D_WAAP) << "send log got decision type: " << decision_type; + auto final_decision = m_waapDecision.getDecision(decision_type); + if (!final_decision) { + final_decision = m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION); + } + std::string attackTypes = buildAttackTypes(); std::string logOverride = "None"; DecisionTelemetryData telemetryData; std::string assetId = m_siteConfig->get_AssetId(); - const auto& autonomousSecurityDecision = std::dynamic_pointer_cast( - m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); I_TimeGet *timeGet = Singleton::Consume::by(); auto finish = timeGet->getMonotonicTime(); @@ -1824,53 +1789,55 @@ Waf2Transaction::sendLog() telemetryData.responseCode = m_responseStatus; } - telemetryData.source = getSourceIdentifier(); telemetryData.assetName = m_siteConfig->get_AssetName(); telemetryData.practiceId = m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION); telemetryData.practiceName = m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION); - if (m_scanResult) { - telemetryData.attackTypes = m_scanResult->attack_types; - } - telemetryData.threat = autonomousSecurityDecision->getThreatLevel(); - if (m_overrideState.bForceBlock) { - telemetryData.blockType = FORCE_BLOCK; - } - else if (m_overrideState.bForceException) { - telemetryData.blockType = FORCE_EXCEPTION; - } - else if (m_waapDecision.getDecision(USER_LIMITS_DECISION)->shouldBlock()) { + telemetryData.temperatureDetected = wasTemperatureDetected(); + switch (decision_type) + { + case USER_LIMITS_DECISION: { telemetryData.blockType = LIMIT_BLOCK; + break; } - else if (autonomousSecurityDecision->shouldBlock()) { - telemetryData.blockType = WAF_BLOCK; - } - else if (m_waapDecision.getDecision(CSRF_DECISION)->shouldBlock()) { + case CSRF_DECISION: { telemetryData.blockType = CSRF_BLOCK; + break; } - else { + case AUTONOMOUS_SECURITY_DECISION: { + telemetryData.blockType = WAF_BLOCK; + break; + } + default: telemetryData.blockType = NOT_BLOCKING; - } - - WaapTelemetryEvent(assetId, telemetryData).notify(); - - if (m_overrideState.bIgnoreLog) { - dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: override is to ignore log - not sending a log"; - return; + break; } bool shouldBlock = false; - if (m_overrideState.bForceBlock) { + if (final_decision->shouldForceBlock()) { + telemetryData.blockType = FORCE_BLOCK; // If override forces "reject" decision, mention it in the "override" log field. logOverride = OVERRIDE_DROP; shouldBlock = true; - } else if (m_overrideState.bForceException) { + } + else if (final_decision->shouldForceAllow()) { + telemetryData.blockType = FORCE_EXCEPTION; // If override forces "allow" decision, mention it in the "override" log field. logOverride = OVERRIDE_ACCEPT; - } else if (m_scanner.getIgnoreOverride()) { + } else if (decision_type == AUTONOMOUS_SECURITY_DECISION && m_scanner.getIgnoreOverride()) { + // skip exception detected by scanner + dbgTrace(D_WAAP) << "should ignore override"; logOverride = OVERRIDE_IGNORE; } + WaapTelemetryEvent(assetId, telemetryData).notify(); + auto it = m_overrideStateByPractice.find(decision_type); + if ((it != m_overrideStateByPractice.end() && it->second.bSupressLog) || + (it == m_overrideStateByPractice.end() && m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog)) { + dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: override is to ignore log - not sending a log"; + return; + } + // Get triggers const std::shared_ptr triggerPolicy = m_siteConfig->get_TriggerPolicy(); @@ -1879,27 +1846,29 @@ Waf2Transaction::sendLog() return; } - const std::shared_ptr triggerLog = getTriggerLog(triggerPolicy); - + auto maybeTriggerLog = getTriggerLog(triggerPolicy, decision_type); // If there were no triggers of type Log - do not send log - if (!triggerLog) { + if (!maybeTriggerLog.ok()) { dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: found no triggers of type 'Log' - not sending a log"; return; } - - bool send_extended_log = shouldSendExtendedLog(triggerLog); + auto triggerLog = maybeTriggerLog.unpack(); + bool send_extended_log = shouldSendExtendedLog(triggerLog, decision_type); shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision(); // Do not send Detect log if trigger disallows it - if (!send_extended_log && shouldBlock == false && !triggerLog->tpDetect && - !autonomousSecurityDecision->getOverridesLog()) + // and we do not override log(for ignore exception) + if (!send_extended_log && shouldBlock == false && + !triggerLog.isDetectLogActive(LogTriggerConf::SecurityType::ThreatPrevention) && + !final_decision->shouldForceLog()) { dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: not sending Detect log (triggers)"; return; } // Do not send Prevent log if trigger disallows it - if (!send_extended_log && shouldBlock == true && !triggerLog->tpPrevent && - !autonomousSecurityDecision->getOverridesLog()) + if (!send_extended_log && shouldBlock == true && + !triggerLog.isPreventLogActive(LogTriggerConf::SecurityType::ThreatPrevention) && + !final_decision->shouldForceLog()) { dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: not sending Prevent log (triggers)"; return; @@ -1908,7 +1877,7 @@ Waf2Transaction::sendLog() // In case no decision to block or log - send log if extend log or override if (!m_waapDecision.anyDecisionsToLogOrBlock()) { - if (send_extended_log || autonomousSecurityDecision->getOverridesLog()) + if (send_extended_log || final_decision->shouldForceLog()) { sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes); dbgTrace(D_WAAP) << "Waf2Transaction::sendLog()::" << @@ -1921,24 +1890,14 @@ Waf2Transaction::sendLog() return; } - DecisionType decision_type = m_waapDecision.getHighestPriorityDecisionToLog(); if (decision_type == DecisionType::NO_WAAP_DECISION) { - if (send_extended_log || autonomousSecurityDecision->getOverridesLog()) { + if (send_extended_log || final_decision->shouldForceLog()) { sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes); } dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: decisions marked for block only"; return; } - std::set triggers_set; - for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { - triggers_set.insert(trigger.triggerId); - dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId; - } - ScopedContext ctx; - ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); - - auto maybeLogTriggerConf = getConfiguration("rulebase", "log"); switch (decision_type) { case USER_LIMITS_DECISION: { @@ -1960,7 +1919,7 @@ Waf2Transaction::sendLog() } LogGenWrapper logGenWrapper( - maybeLogTriggerConf, + maybeTriggerLog, "Web Request", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, @@ -1969,11 +1928,7 @@ Waf2Transaction::sendLog() shouldBlock); LogGen& waap_log = logGenWrapper.getLogGen(); - appendCommonLogFields( - waap_log, triggerLog, shouldBlock, logOverride, incidentType, - m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION), - m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION) - ); + appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType, decision_type); waap_log << LogField("waapIncidentDetails", incidentDetails); waap_log << LogField("eventConfidence", "High"); break; @@ -1983,7 +1938,7 @@ Waf2Transaction::sendLog() case RATE_LIMITING_DECISION: case ERROR_DISCLOSURE_DECISION: { LogGenWrapper logGenWrapper( - maybeLogTriggerConf, + maybeTriggerLog, "API Request", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, @@ -2006,18 +1961,14 @@ Waf2Transaction::sendLog() waap_log << LogField("waapFoundIndicators", getKeywordMatchesStr(), LogFieldOption::XORANDB64); } - appendCommonLogFields( - waap_log, triggerLog, shouldBlock, logOverride, incidentType, - m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION), - m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION) - ); + appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType, decision_type); waap_log << LogField("waapIncidentDetails", incidentDetails); break; } case CSRF_DECISION: { LogGenWrapper logGenWrapper( - maybeLogTriggerConf, + maybeTriggerLog, "CSRF Protection", ReportIS::Audience::SECURITY, LogTriggerConf::SecurityType::ThreatPrevention, @@ -2027,18 +1978,17 @@ Waf2Transaction::sendLog() LogGen& waap_log = logGenWrapper.getLogGen(); appendCommonLogFields( - waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery", - m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION), - m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION) + waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery", decision_type ); waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered."); break; } case AUTONOMOUS_SECURITY_DECISION: { - if (triggerLog->webRequests || + if (triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::webRequests) || send_extended_log || - autonomousSecurityDecision->getThreatLevel() != ThreatLevel::NO_THREAT || - autonomousSecurityDecision->getOverridesLog()) { + std::dynamic_pointer_cast(final_decision) + ->getThreatLevel() != ThreatLevel::NO_THREAT || + final_decision->shouldForceLog()) { sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes); } break; @@ -2080,7 +2030,8 @@ Waf2Transaction::decideAutonomousSecurity( // Do not call stage2 so it doesn't learn from exceptions. // Also do not call stage2 for attacks found in parameter name - if (!m_overrideState.bForceException && !(m_scanResult && m_scanResult->m_isAttackInParam)) { + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException && + !(m_scanResult && m_scanResult->m_isAttackInParam)) { if (!m_processedUri) { dbgWarning(D_WAAP) << "decideAutonomousSecurity(): processing URI although is was supposed " "to be processed earlier ..."; @@ -2135,7 +2086,8 @@ Waf2Transaction::decideAutonomousSecurity( } // Fill attack details for attacks found in parameter names - if (!m_overrideState.bForceException && m_scanResult && m_scanResult->m_isAttackInParam) { + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException && + m_scanResult && m_scanResult->m_isAttackInParam) { // Since stage2 learning doesn't run in this case, assume stage1 score is the final score float finalScore = m_scanResult->score; ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(finalScore); @@ -2153,13 +2105,18 @@ Waf2Transaction::decideAutonomousSecurity( transactionResult.d2Analysis.finalScore = finalScore; transactionResult.shouldBlock = shouldBlock; transactionResult.threatLevel = threat; + } else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock && + decision->getThreatLevel() == ThreatLevel::NO_THREAT) { + // If override forces block, set threat level to INFO + decision->setThreatLevel(ThreatLevel::THREAT_INFO); } dbgTrace(D_WAAP_OVERRIDE) << "override ids count: " << m_matchedOverrideIds.size(); // Apply overrides for (auto it = m_overridePostFilterMaxScore.begin(); it != m_overridePostFilterMaxScore.end(); it++) { const string id = it->first; - if (m_overrideState.forceBlockIds.find(id) != m_overrideState.forceBlockIds.end()) { + if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.find(id) != + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.end()) { // blocked effectivness is calculates later from the force block exception ids list continue; } @@ -2179,34 +2136,41 @@ Waf2Transaction::decideAutonomousSecurity( } } - if (m_overrideState.bForceBlock) { + if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) { dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() << " and override forces REJECT ..."; if (!decision->shouldBlock()) { - m_effectiveOverrideIds.insert(m_overrideState.forceBlockIds.begin(), m_overrideState.forceBlockIds.end()); + m_effectiveOverrideIds.insert( + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.begin(), + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].forceBlockIds.end() + ); } decision->setBlock(true); - if (!m_overrideState.bIgnoreLog) + decision->setForceBlock(true); + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog) { - decision->setOverridesLog(true); + decision->setForceLog(true); } } - else if (m_overrideState.bForceException) { - dbgTrace(D_WAAP) << "de cideAutonomousSecurity(): decision was " << decision->shouldBlock() << + else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) { + dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() << " and override forces ALLOW ..."; decision->setBlock(false); - if (!m_overrideState.bIgnoreLog) + decision->setForceAllow(true); + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog) { - decision->setOverridesLog(true); + decision->setForceLog(true); } } else if (!m_matchedOverrideIds.empty()) { - if (!m_overrideState.bIgnoreLog) + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bSupressLog) { - decision->setOverridesLog(true); + decision->setForceLog(true); } } - dbgTrace(D_WAAP_OVERRIDE) << "force exception: " << m_overrideState.bForceException << - " force block: " << m_overrideState.bForceBlock << + dbgTrace(D_WAAP_OVERRIDE) << + "force exception: " << + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException << + " force block: " << m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock << " matched overrides count: " << m_matchedOverrideIds.size() << " effective overrides count: " << m_effectiveOverrideIds.size() << " learned overrides count: " << m_exceptionLearned.size(); @@ -2214,8 +2178,9 @@ Waf2Transaction::decideAutonomousSecurity( bool log_all = false; const std::shared_ptr triggerPolicy = sitePolicy.get_TriggerPolicy(); if (triggerPolicy) { - const std::shared_ptr triggerLog = getTriggerLog(triggerPolicy); - if (triggerLog && triggerLog->webRequests) log_all = true; + auto triggerLog = getTriggerLog(triggerPolicy, AUTONOMOUS_SECURITY_DECISION); + if (triggerLog.ok() && triggerLog.unpack().isWebLogFieldActive(LogTriggerConf::WebLogFields::webRequests)) log_all = true; + if (triggerLog.ok() && shouldSendExtendedLog(triggerLog.unpack())) m_responseInspectReasons.setCollectResponseForLog(true); } if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) { @@ -2223,7 +2188,6 @@ Waf2Transaction::decideAutonomousSecurity( } else { decision->setLog(true); } - return decision->shouldBlock(); } @@ -2243,6 +2207,14 @@ bool Waf2Transaction::shouldInspectResponse() { return m_responseInspectReasons.shouldInspect() || m_responseInjectReasons.shouldInject(); } +bool +Waf2Transaction::shouldLimitResponseHeadersInspection() { + auto triggerPolicy = m_siteConfig ? m_siteConfig->get_TriggerPolicy() : NULL; + if (!shouldInspectResponse() && isTriggerReportExists(triggerPolicy)) { + return true; + } + return false; +} bool Waf2Transaction::shouldInjectResponse() { return m_responseInjectReasons.shouldInject(); @@ -2276,14 +2248,14 @@ bool Waf2Transaction::decideResponse() if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) { dbgTrace(D_WAAP) << "Waf2Transaction::decideResponse(): got relevant Application configuration from the I/S"; - m_overrideState = getOverrideState(&ngenSiteConfig); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(&ngenSiteConfig); // Apply overrides - if (m_overrideState.bForceBlock) { + if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) { dbgTrace(D_WAAP) << "Waf2Transaction::decideResponse(): setting shouldBlock to true due to override"; return false; // BLOCK } - else if (m_overrideState.bForceException) { + else if (m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) { dbgTrace(D_WAAP) << "Waf2Transaction::decideResponse(): setting shouldBlock to false due to override"; return true; // PASS @@ -2297,25 +2269,25 @@ bool Waf2Transaction::decideResponse() dbgTrace(D_WAAP) << "Trigger policy was not found. Returning true (accept)"; return true; // accept } - - const std::shared_ptr triggerLog = getTriggerLog(triggerPolicy); - if (!triggerLog) { + // response is only for WAF + auto maybeTriggerLog = getTriggerLog(triggerPolicy, AUTONOMOUS_SECURITY_DECISION); + if (!maybeTriggerLog.ok()) { dbgTrace(D_WAAP) << "Log trigger configuration was not found. Returning true (accept)"; return true; // accept } - + auto triggerLog = maybeTriggerLog.unpack(); auto env = Singleton::Consume::by(); auto http_chunk_type = env->get("HTTP Chunk type"); bool should_send_extended_log = shouldSendExtendedLog(triggerLog) && http_chunk_type.ok(); if (should_send_extended_log && *http_chunk_type == ngx_http_chunk_type_e::RESPONSE_CODE && - !triggerLog->responseBody + !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) ) { should_send_extended_log = false; } else if (should_send_extended_log && *http_chunk_type == ngx_http_chunk_type_e::REQUEST_END && - !triggerLog->responseCode && - !triggerLog->responseBody + !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode) && + !triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) ) { should_send_extended_log = false; } @@ -2347,83 +2319,116 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) { bool Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { - auto exceptions = getConfiguration("rulebase", "exception"); - if (!exceptions.ok()) { - dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr(); - return false; - } - dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions"; + + dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions for should ignore"; std::unordered_map> exceptions_dict; + std::set behaviors; + std::set &ignored_keywords = getAssetState()->m_filtersMngr->getMatchedOverrideKeywords(); - if (res.location != "referer") { - // collect param name - exceptions_dict["paramName"].insert(res.param_name); - exceptions_dict["paramName"].insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this)); + // collect param name + exceptions_dict["paramName"].insert(res.param_name); + exceptions_dict["paramName"].insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this)); - std::set param_name_set; - param_name_set.insert(res.param_name); - param_name_set.insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this)); + std::set param_name_set; + param_name_set.insert(res.param_name); + param_name_set.insert(IndicatorsFiltersManager::generateKey(res.location, res.param_name, this)); - // collect param value - exceptions_dict["paramValue"].insert(res.unescaped_line); + // collect param value + exceptions_dict["paramValue"].insert(res.unescaped_line); - // collect param location - exceptions_dict["paramLocation"].insert(res.location); + // collect param location + exceptions_dict["paramLocation"].insert(res.location); - ScopedContext ctx; - ctx.registerValue("paramValue", res.unescaped_line); - ctx.registerValue>("paramName", param_name_set); + ScopedContext ctx; + ctx.registerValue("paramValue", res.unescaped_line); + ctx.registerValue>("paramName", param_name_set); - // collect sourceip, sourceIdentifier, url - exceptions_dict["sourceIP"].insert(m_remote_addr); - exceptions_dict["sourceIdentifier"].insert(m_source_identifier); - exceptions_dict["url"].insert(getUriStr()); - exceptions_dict["hostName"].insert(m_hostStr); - exceptions_dict["method"].insert(m_methodStr); + // collect sourceip, sourceIdentifier, url + exceptions_dict["sourceIP"].insert(m_remote_addr); + exceptions_dict["sourceIdentifier"].insert(m_source_identifier); + exceptions_dict["url"].insert(getUriStr()); + exceptions_dict["hostName"].insert(m_hostStr); + exceptions_dict["method"].insert(m_methodStr); - for (auto &keyword : res.keyword_matches) { - exceptions_dict["indicator"].insert(keyword); - } - for (auto &it : res.found_patterns) { - exceptions_dict["indicator"].insert(it.first); - } + for (auto &keyword : res.keyword_matches) { + exceptions_dict["indicator"].insert(keyword); + } + for (auto &it : res.found_patterns) { + exceptions_dict["indicator"].insert(it.first); + } - // calling behavior and check if there is a behavior that match to this specific param name. - auto behaviors = exceptions.unpack().getBehavior(exceptions_dict, - getAssetState()->m_filtersMngr->getMatchedOverrideKeywords()); - for (const auto &behavior : behaviors) { - dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId(); - if (!res.filtered_keywords.empty() || res.score > 0) { - dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators"; - std::string overrideId = behavior.getId(); - if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){ - m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter; - m_overridePostFilterMaxScore[overrideId] = res.score; - } else { - if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) { - m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter; - } - if (res.score > m_overridePostFilterMaxScore[overrideId]) { - m_overridePostFilterMaxScore[overrideId] = res.score; - } - } - if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) { - m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter; - } - } - if (behavior == action_ignore) - { - dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore."; - std::string overrideId = behavior.getId(); - if (!overrideId.empty()) { - m_matchedOverrideIds.insert(overrideId); - } - return true; - } + bool isConfigExist = false; + if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) { + dbgTrace(D_WAAP_OVERRIDE) << "waap api config found"; + m_siteConfig = &m_ngenAPIConfig; + isConfigExist = true; + } else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) { + dbgTrace(D_WAAP_OVERRIDE) << "waap web application config found"; + m_siteConfig = &m_ngenSiteConfig; + isConfigExist = true; + } + std::vector site_exceptions; + if (isConfigExist) { + dbgTrace(D_WAAP_OVERRIDE) << "config exists, get override policy"; + std::shared_ptr overridePolicy = m_siteConfig->get_OverridePolicy(); + if (overridePolicy) { + site_exceptions = overridePolicy->getExceptionsByPractice() + .getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION); } } + if (!site_exceptions.empty()) { + dbgTrace(D_WAAP_OVERRIDE) << "get behaviors by web app practice"; + I_GenericRulebase *i_rulebase = Singleton::Consume::by(); + for (const auto &id : site_exceptions) { + dbgTrace(D_WAAP_OVERRIDE) << "get parameter exception: " << id; + auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict, ignored_keywords); + behaviors.insert(params.begin(), params.end()); + } + } else { + auto exceptions = getConfiguration("rulebase", "exception"); + if (!exceptions.ok()) { + dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr(); + return false; + } + // calling behavior and check if there is a behavior that match to this specific param name. + behaviors = exceptions.unpack().getBehavior(exceptions_dict, ignored_keywords); + } + dbgTrace(D_WAAP_OVERRIDE) << "got "<< behaviors.size() << " behaviors and " << + ignored_keywords.size() << " ignored keywords"; + + for (const auto &behavior : behaviors) { + dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId(); + if (!res.filtered_keywords.empty() || res.score > 0) { + dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators"; + std::string overrideId = behavior.getId(); + if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){ + m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter; + m_overridePostFilterMaxScore[overrideId] = res.score; + } else { + if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) { + m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter; + } + if (res.score > m_overridePostFilterMaxScore[overrideId]) { + m_overridePostFilterMaxScore[overrideId] = res.score; + } + } + if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) { + m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter; + } + } + if (behavior == action_ignore) + { + dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore."; + std::string overrideId = behavior.getId(); + if (!overrideId.empty()) { + m_matchedOverrideIds.insert(overrideId); + } + return true; + } + } + dbgTrace(D_WAAP_OVERRIDE) << "should not ignore"; return false; } @@ -2485,9 +2490,15 @@ void Waf2Transaction::collectFoundPatterns() } } -bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptr &trigger_log) const +bool Waf2Transaction::shouldSendExtendedLog(const LogTriggerConf &trigger_log, DecisionType practiceType) const { - if (!trigger_log->extendLogging) + if (practiceType != AUTONOMOUS_SECURITY_DECISION) + { + dbgTrace(D_WAAP) << "Should not send extended logging. Practice type is: " << practiceType; + return false; + } + auto extend_logging_severity = trigger_log.getExtendLoggingSeverity(); + if(extend_logging_severity == LogTriggerConf::extendLoggingSeverity::None) { dbgTrace(D_WAAP) << "Should not send extended log. Extended log is disabled."; return false; @@ -2498,7 +2509,7 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptrgetThreatLevel()); - if (trigger_log->extendLoggingMinSeverity == "Critical" || trigger_log->extendLoggingMinSeverity == "critical") + if (extend_logging_severity == LogTriggerConf::extendLoggingSeverity::Critical) { if (severity == ReportIS::Severity::CRITICAL) { @@ -2508,7 +2519,7 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptrextendLoggingMinSeverity == "High" || trigger_log->extendLoggingMinSeverity == "high") + else if (extend_logging_severity == LogTriggerConf::extendLoggingSeverity::High) { if (severity == ReportIS::Severity::CRITICAL || severity == ReportIS::Severity::HIGH) { @@ -2519,6 +2530,19 @@ bool Waf2Transaction::shouldSendExtendedLog(const std::shared_ptrextendLoggingMinSeverity; + dbgTrace(D_WAAP) << "Should not send extended logging. Min Severity: " << (int) extend_logging_severity; return false; } + + +void +Waf2Transaction::setTemperatureDetected(bool detected) +{ + m_temperature_detected = detected; +} + +bool +Waf2Transaction::wasTemperatureDetected() const +{ + return m_temperature_detected; +} diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index 12a4e4b..ebf0a5f 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -22,17 +22,22 @@ #include "PatternMatcher.h" #include "generic_rulebase/rulebase_config.h" #include "generic_rulebase/evaluators/trigger_eval.h" +#include "i_generic_rulebase.h" +#include "generic_rulebase/parameters_config.h" #include "Waf2Util.h" #include "WaapConfigApplication.h" #include "WaapConfigApi.h" #include "WaapDecision.h" +#include "DecisionType.h" #include "DeepAnalyzer.h" #include +#include #include #include #include #include #include +#include #include // uuid class #include // uuid generators #include @@ -65,7 +70,8 @@ class Waf2Transaction : private boost::noncopyable, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: Waf2Transaction(std::shared_ptr pWaapAssetState); @@ -131,6 +137,7 @@ public: ngx_http_cp_verdict_e getUserLimitVerdict(); const std::string getUserLimitVerdictStr() const; const std::string getViolatedUserLimitTypeStr() const; + const std::string getCurrentWebUserResponse(); virtual HeaderType detectHeaderType(const char* name, int name_len); HeaderType checkCleanHeader(const char* name, int name_len, const char* value, int value_len) const; @@ -195,7 +202,9 @@ public: void handleSecurityHeadersInjection(std::vector>& injectHeaderStrs); void disableShouldInjectSecurityHeaders(); - bool shouldSendExtendedLog(const std::shared_ptr &trigger_log) const; + bool shouldSendExtendedLog( + const LogTriggerConf &trigger_log, DecisionType practiceType = AUTONOMOUS_SECURITY_DECISION + ) const; // query virtual bool isSuspicious() const; @@ -232,24 +241,26 @@ public: // LCOV_EXCL_START Reason: This function is tested in system tests bool checkIsHeaderOverrideScanRequired(); // LCOV_EXCL_STOP + bool shouldLimitResponseHeadersInspection(); + + void setTemperatureDetected(bool detected); + bool wasTemperatureDetected() const; private: - int finalizeDecision(IWaapConfig *sitePolicy, bool shouldBlock); - const std::shared_ptr getTriggerLog(const std::shared_ptr& - triggerPolicy) const; + const Maybe getTriggerLog(const std::shared_ptr& + triggerPolicy, DecisionType practiceType) const; bool isTriggerReportExists(const std::shared_ptr &triggerPolicy); void sendAutonomousSecurityLog( - const std::shared_ptr& triggerLog, + const LogTriggerConf& triggerLog, bool shouldBlock, const std::string& logOverride, const std::string& attackTypes) const; void appendCommonLogFields(LogGen& waapLog, - const std::shared_ptr &triggerLog, + const LogTriggerConf &triggerLog, bool shouldBlock, const std::string& logOverride, const std::string& incidentType, - const std::string& practiceID, - const std::string& practiceName) const; + DecisionType practiceType) const; std::string getUserReputationStr(double relativeReputation) const; bool isTrustedSource() const; @@ -258,6 +269,12 @@ private: bool setCurrentAssetContext(); bool checkIsScanningRequired(); Waap::Override::State getOverrideState(IWaapConfig* sitePolicy); + std::set getBehaviors( + const std::unordered_map> &exceptions_dict, + const std::vector& exceptions, bool checkResponse); + std::unordered_map> getExceptionsDict(DecisionType practiceType); + bool shouldEnforceByPracticeExceptions(DecisionType practiceType); + void setOverrideState(const std::set& behaviors, Waap::Override::State& state); // User limits functions void createUserLimitsState(); @@ -360,15 +377,16 @@ private: Waap::ResponseInspectReasons m_responseInspectReasons; Waap::ResponseInjectReasons m_responseInjectReasons; WaapDecision m_waapDecision; - Waap::Override::State m_overrideState; + std::unordered_map m_overrideStateByPractice; + std::string m_practiceSubType; uint64_t m_index; - // Cached pointer to const triggerLog (hence mutable) - mutable std::shared_ptr m_triggerLog; bool m_triggerReport; bool is_schema_validation = false; Waf2TransactionFlags m_waf2TransactionFlags; + + bool m_temperature_detected = false; // Tracks if temperature was detected }; #endif // __WAF2_TRANSACTION_H__99e4201a diff --git a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc index 8cb8cb1..0cf658d 100755 --- a/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc +++ b/components/security_apps/waap/waap_clib/Waf2EngineGetters.cc @@ -17,7 +17,6 @@ #include // uuid generators #include #include -#include "generic_rulebase/triggers_config.h" #include "config.h" #include "LogGenWrapper.h" #include @@ -343,7 +342,7 @@ Waap::CSRF::State& Waf2Transaction::getCsrfState() } void Waf2Transaction::sendAutonomousSecurityLog( - const std::shared_ptr& triggerLog, + const LogTriggerConf& triggerLog, bool shouldBlock, const std::string& logOverride, const std::string& attackTypes) const @@ -352,11 +351,11 @@ void Waf2Transaction::sendAutonomousSecurityLog( m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); ReportIS::Severity severity = Waap::Util::computeSeverityFromThreatLevel( autonomousSecurityDecision->getThreatLevel()); - if (autonomousSecurityDecision->getOverridesLog() && logOverride == OVERRIDE_DROP) + if (autonomousSecurityDecision->shouldForceLog() && logOverride == OVERRIDE_DROP) { severity = ReportIS::Severity::MEDIUM; } - else if (autonomousSecurityDecision->getOverridesLog() && logOverride == OVERRIDE_ACCEPT) + else if (autonomousSecurityDecision->shouldForceLog() && logOverride == OVERRIDE_ACCEPT) { severity = ReportIS::Severity::INFO; } @@ -381,11 +380,7 @@ void Waf2Transaction::sendAutonomousSecurityLog( waap_log << LogField("eventConfidence", confidence); } - appendCommonLogFields( - waap_log, triggerLog, shouldBlock, logOverride, attackTypes, - m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION), - m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION) - ); + appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, attackTypes, AUTONOMOUS_SECURITY_DECISION); std::string sampleString = getSample(); if (sampleString.length() > MAX_LOG_FIELD_SIZE) { @@ -470,24 +465,36 @@ Waf2Transaction::getUserLimitVerdict() auto decision = m_waapDecision.getDecision(USER_LIMITS_DECISION); if (mode == AttackMitigationMode::LEARNING) { decision->setLog(true); - decision->setBlock(false); - if (isIllegalMethodViolation()) { - dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode"; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; - } - else { - dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason; - verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceBlock) { + // detect mode and no active drop exception + decision->setBlock(false); + if (isIllegalMethodViolation()) { + dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode"; + verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + } + else { + dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason; + verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; + } + } else { + // detect mode and active drop exception + decision->setBlock(true); + decision->setForceBlock(true); + dbgInfo(D_WAAP_ULIMITS) << msg << "Override Block" << reason; + verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; } } else if (mode == AttackMitigationMode::PREVENT) { decision->setLog(true); - if (!m_overrideState.bForceException) { + if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) { + // prevent mode and no active accept exception decision->setBlock(true); dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason; verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; } else { - decision->setBlock(true); + // prevent mode and active accept exception + decision->setBlock(false); + decision->setForceAllow(true); dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason; verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; } @@ -585,20 +592,206 @@ bool Waf2Transaction::checkIsHeaderOverrideScanRequired() return m_isHeaderOverrideScanRequired; } +const std::string Waf2Transaction::getCurrentWebUserResponse() { + dbgFlow(D_WAAP); + m_waapDecision.orderDecisions(); + DecisionType practiceType = m_waapDecision.getHighestPriorityDecisionToLog(); + + dbgTrace(D_WAAP) << "set current web user response for practice: " << practiceType; + + const std::shared_ptr triggerPolicy = m_siteConfig->get_TriggerPolicy(); + if (!triggerPolicy) { + dbgTrace(D_WAAP) << "No trigger policy, can't set web user response for practice: " << practiceType; + return ""; + } + auto responses = triggerPolicy->responseByPractice.getResponseByPractice(practiceType); + if (responses.empty()) { + dbgTrace(D_WAAP) << "No web user response for practice: " << practiceType; + return ""; + } + dbgTrace(D_WAAP) << "Found web user response trigger by practice, ID: " << responses[0]; + return responses[0]; +} + +std::unordered_map> +Waf2Transaction::getExceptionsDict(DecisionType practiceType) { + std::unordered_map> exceptions_dict; + exceptions_dict["url"].insert(m_uriPath); + exceptions_dict["sourceIP"].insert(m_remote_addr); + exceptions_dict["method"].insert(m_methodStr); + + extractEnvSourceIdentifier(); + exceptions_dict["sourceIdentifier"].insert(m_source_identifier); + if ( + practiceType == DecisionType::AUTONOMOUS_SECURITY_DECISION + ) { + for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) { + exceptions_dict["paramName"].insert(keywordInfo.getName()); + } + } + if (practiceType == DecisionType::AUTONOMOUS_SECURITY_DECISION) { + exceptions_dict["hostName"].insert(m_hostStr); + for (const std::string& keywordStr : getKeywordMatches()) { + exceptions_dict["indicator"].insert(keywordStr); + } + for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) { + exceptions_dict["paramValue"].insert(keywordInfo.getValue()); + } + exceptions_dict["paramLocation"].insert(getLocation()); + if (!checkIsHeaderOverrideScanRequired()) { + dbgDebug(D_WAAP) << "Header name override scan is not required"; + } else { + for (auto& hdr_pair : getHdrPairs()) { + std::string name = hdr_pair.first; + std::transform(name.begin(), name.end(), name.begin(), ::tolower); + exceptions_dict["headerName"].insert(name); + std::string value = hdr_pair.second; + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + exceptions_dict["headerValue"].insert(value); + } + } + } + return exceptions_dict; +} + +std::set +Waf2Transaction::getBehaviors( + const std::unordered_map> &exceptions_dict, + const std::vector& exceptions, bool checkResponse = false) +{ + std::set all_params; + I_GenericRulebase *i_rulebase = Singleton::Consume::by(); + for (const auto &id : exceptions) { + dbgTrace(D_WAAP) << "get parameter exception for: " << id; + auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict); + + if (checkResponse && !getResponseBody().empty()) { + std::unordered_map> response_dict = { + {"responseBody", {getResponseBody()}} + }; + auto params = i_rulebase->getParameterException(id).getBehavior(response_dict); + if (params.size() > 0) { + dbgTrace(D_WAAP) << "got responseBody behavior, setApplyOverride(true)"; + m_responseInspectReasons.setApplyOverride(true); + all_params.insert(params.begin(), params.end()); + // once found, no need to check again + checkResponse = false; + } + } + + dbgTrace(D_WAAP) << "got "<< params.size() << " behaviors"; + all_params.insert(params.begin(), params.end()); + } + return all_params; +} + + +bool Waf2Transaction::shouldEnforceByPracticeExceptions(DecisionType practiceType) +{ + dbgFlow(D_WAAP); + auto decision = m_waapDecision.getDecision(practiceType); + bool shouldEnforce = false; + std::shared_ptr overridePolicy = m_siteConfig->get_OverridePolicy(); + if (overridePolicy) { + auto exceptions = overridePolicy->getExceptionsByPractice().getExceptionsOfPractice(practiceType); + + if (!exceptions.empty()) { + dbgTrace(D_WAAP) << "get behaviors for practice: " << practiceType; + + auto behaviors = getBehaviors(getExceptionsDict(practiceType), exceptions); + if (behaviors.size() > 0) + { + auto it = m_overrideStateByPractice.find(practiceType); + if (it == m_overrideStateByPractice.end()) { + dbgWarning(D_WAAP) << "no override state for practice: " << practiceType; + return false; + } + setOverrideState(behaviors, it->second); + if (it->second.bForceBlock) { + dbgTrace(D_WAAP) + << "should block by exceptions for practice: " << practiceType; + decision->setBlock(true); + decision->setForceBlock(true); + shouldEnforce = true; + } + if (it->second.bForceException) { + dbgTrace(D_WAAP) + << "should not block by exceptions for practice: " << practiceType; + decision->setBlock(false); + decision->setForceAllow(true); + shouldEnforce = true; + } + } + } + } + if (shouldEnforce) { + decision->setLog(true); + if(!m_overrideStateByPractice[practiceType].bSupressLog) { + decision->setForceLog(true); + } + std::shared_ptr autonomousDecision = + std::dynamic_pointer_cast( + m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION) + ); + if (autonomousDecision->getThreatLevel() <= ThreatLevel::THREAT_INFO) { + autonomousDecision->setLog(false); + } + } else if(!m_matchedOverrideIds.empty() && !m_overrideStateByPractice[practiceType].bSupressLog) { + decision->setForceLog(true); + } + return shouldEnforce; +} + +void Waf2Transaction::setOverrideState(const std::set& behaviors, Waap::Override::State& state) { + dbgFlow(D_WAAP) << "setOverrideState(): from exceptions per practice"; + for (auto const &behavior : behaviors) { + m_matchedOverrideIds.insert(behavior.getId()); + if (behavior.getKey() == BehaviorKey::ACTION) { + if (behavior.getValue() == BehaviorValue::ACCEPT) { + dbgTrace(D_WAAP) << "setting bForceException due to override behavior: " << behavior.getId(); + state.bForceException = true; + state.forceExceptionIds.insert(behavior.getId()); + } else if (behavior.getValue() == BehaviorValue::REJECT) { + dbgTrace(D_WAAP) << "setting bForceBlock due to override behavior: " << behavior.getId(); + state.bForceBlock = true; + state.forceBlockIds.insert(behavior.getId()); + } + } else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) { + dbgTrace(D_WAAP) << "setting bSupressLog due to override behavior: " << behavior.getId(); + state.bSupressLog = true; + } + } +} + Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy) { Waap::Override::State overrideState; std::shared_ptr overridePolicy = sitePolicy->get_OverridePolicy(); if (overridePolicy) { // at first we will run request overrides (in order to set the source) + auto exceptions = overridePolicy->getExceptionsByPractice(). + getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION); + if (!exceptions.empty()) { + dbgTrace(D_WAAP) << "get behaviors for override state"; + m_responseInspectReasons.setApplyOverride(false); + auto behaviors = getBehaviors(getExceptionsDict(AUTONOMOUS_SECURITY_DECISION), exceptions, true); + if (behaviors.size() > 0) { + dbgTrace(D_WAAP) << "set override state by practice found behaviors"; + setOverrideState(behaviors, m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]); + } + m_isHeaderOverrideScanRequired = false; + return m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]; + } m_responseInspectReasons.setApplyOverride(overridePolicy->isOverrideResponse()); overrideState.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, true); } if (overridePolicy) { // later we will run response overrides - m_overrideState.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false); + m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].applyOverride( + *overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false + ); } m_isHeaderOverrideScanRequired = false; - return m_overrideState; + return m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]; } Waf2TransactionFlags &Waf2Transaction::getTransactionFlags() @@ -606,22 +799,32 @@ Waf2TransactionFlags &Waf2Transaction::getTransactionFlags() return m_waf2TransactionFlags; } -const std::shared_ptr Waf2Transaction::getTriggerLog(const std::shared_ptr< - Waap::Trigger::Policy> &triggerPolicy) const +const Maybe Waf2Transaction::getTriggerLog(const std::shared_ptr< + Waap::Trigger::Policy> &triggerPolicy, DecisionType practiceType) const { - // Trigger log already known (no need to extract it second time) - if (m_triggerLog) { - return m_triggerLog; - } - - // Walk over trigger logs and choose the last one of type Log - for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { - if (trigger.triggerType == "log") { - m_triggerLog = trigger.log; + dbgTrace(D_WAAP) << "Getting log trigger for practice type:" << practiceType; + std::set triggers_set; + auto triggers = triggerPolicy->triggersByPractice.getTriggersByPractice(practiceType); + if (!triggers.empty()) { + for (const auto &id : triggers) { + triggers_set.insert(id); + dbgTrace(D_WAAP) << "Add log trigger by practice to triggers set:" << id; + } + } else { + for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { + triggers_set.insert(trigger.triggerId); + dbgTrace(D_WAAP) << "Add log trigger waap to triggers set:" << trigger.triggerId; } } - return m_triggerLog; + ScopedContext ctx; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); + auto trigger_config = getConfiguration("rulebase", "log"); + + if (!trigger_config.ok()) { + dbgError(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr(); + } + return trigger_config; } bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr< @@ -633,12 +836,31 @@ bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr< if (m_triggerReport) { return m_triggerReport; } - for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { - if (trigger.triggerType == "report") { - return m_triggerReport = true; + std::set triggers_set; + auto triggers = triggerPolicy->triggersByPractice.getAllTriggers(); + if (!triggers.empty()) { + for (const auto &id : triggers) { + triggers_set.insert(id); } } - return m_triggerReport; + else { + for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) { + triggers_set.insert(trigger.triggerId); + } + } + + ScopedContext ctx; + ctx.registerValue>(TriggerMatcher::ctx_key, triggers_set); + auto trigger_config = getConfiguration("rulebase", "report"); + if (!trigger_config.ok()) { + dbgWarning(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr(); + m_triggerReport = false; + return false; + } + auto triggerName = trigger_config.unpack().getName(); + dbgTrace(D_WAAP) << "Got report trigger from rulbase: " << triggerName << ", m_triggerReport = true"; + m_triggerReport = true; + return true; } ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index b638e72..adf2c08 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -642,7 +642,7 @@ void unescapeUnicode(string& text) { STATE_ESCAPE_X } state = STATE_COPY; - for (; it != text.end(); ++it) { + for (; it != text.end() && result <= it; ++it) { const char ch = *it; switch (state) { @@ -682,16 +682,10 @@ void unescapeUnicode(string& text) { state = STATE_ESCAPE_U; } else if (ch == 'x') { -#if 1 digitsAnticipated = 1; // anticipate at least one HEX digit after \x code = 0; nonZeroHexCounter = 0; state = STATE_ESCAPE_X; -#else - digitsAnticipated = 2; // parse/skip 2 hex digits - code = 0; - state = STATE_ESCAPE_U; -#endif } else { // this is invalid escape sequence: rollback and copy this character too @@ -794,10 +788,12 @@ void unescapeUnicode(string& text) { *result++ = ch; } + dbgAssertOpt(!pAcc || size_t(pAcc - acc) < sizeof(acc)) + << AlertInfo(AlertTeam::WAAP, "WAAP string unescape") << + "Buffer overflow detected"; // Accumulate - if (pAcc) { + if (pAcc && size_t(pAcc - acc) < sizeof(acc)) { // Ensure we don't have buffer overflow - assert(size_t(pAcc - acc) < sizeof(acc)); *pAcc++ = ch; } } @@ -806,7 +802,7 @@ void unescapeUnicode(string& text) { digitsAnticipated << ", acc='" << string(acc, pAcc ? (int)(pAcc - acc) : 0) << "'"; // Output code if we just finished decoding an escape sequence succesully and reached end of string - if (state == STATE_ESCAPE_U && digitsAnticipated == 0) { + if (state == STATE_ESCAPE_U && digitsAnticipated == 0 && result < text.end()) { // only output ASCII codes <= 127. "swallow" all unicode. if (code <= 127) { *result++ = (char)code; @@ -823,7 +819,7 @@ void unescapeUnicode(string& text) { if (isSpecialUnicode(code)) { *result++ = convertSpecialUnicode(code); } - else + else if (digitsAnticipated == 0) { *result++ = (char)code; } @@ -831,7 +827,7 @@ void unescapeUnicode(string& text) { // flush any accumulated left-overs into output buffer if (pAcc) { - for (p = acc; p < pAcc; p++) { + for (p = acc; p < pAcc && result < text.end(); p++) { *result++ = *p; } } @@ -1034,8 +1030,7 @@ base64_decode_status decideStatusBase64Decoded( && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD && !called_with_prefix && decoded.size() > BASE64_MIN_SIZE_LIMIT - && decoded.size() < BASE64_MAX_SIZE_LIMIT - && terminatorCharsSeen != 0; + && decoded.size() < BASE64_MAX_SIZE_LIMIT; if (!empiric_condition) { if (clear_on_error) decoded.clear(); return B64_DECODE_SUSPECTED; @@ -1070,7 +1065,8 @@ base64_decode_status decideStatusBase64Decoded( && decoded_entropy > BASE64_ENTROPY_DECODED_THRESHOLD && !called_with_prefix && decoded.size() > BASE64_MIN_SIZE_LIMIT - && decoded.size() < BASE64_MAX_SIZE_LIMIT; + && decoded.size() < BASE64_MAX_SIZE_LIMIT + && nonPrintableCharsCount != 0; if (empiric_condition) { dbgTrace(D_WAAP_BASE64) << "Empiric test failed, non-base64 chunk, return B64_DECODE_INVALID"; decoded.clear(); @@ -1372,10 +1368,23 @@ static const SingleRegex base64_key_detector_re( "^[^<>{};,&\\?|=\\s]+={1}", err, "base64_key"); +// Matches valid Base64 prefixes: "data:" URIs with optional parameters or bare "base64," +// strings. Ensures correct MIME types, strict Base64 encoding with padding, and anchoring +// at string-start or after '=' (with optional whitespace). Prevents invalid injections. static const SingleRegex base64_prefix_detector_re( - "data:\\S*;base64,\\S+|base64,\\S+", + "^data:[a-zA-Z0-9.+-]+/[a-zA-Z0-9.+-]+(?:;[a-zA-Z0-9.+-]+=[a-zA-Z0-9.+-]+)*;base64,[A-Za-z0-9+/]+" + "={0,2}|^base64,[A-Za-z0-9+/]+={0,2}|=\\s*data:[a-zA-Z0-9.+-]+/[a-zA-Z0-9.+-]+(?:;[a-zA-Z0-9.+-]+" + "=[a-zA-Z0-9.+-]+)*;base64,[A-Za-z0-9+/]+={0,2}|=\\s*base64,[A-Za-z0-9+/]+={0,2}", err, "base64_prefix"); +static const SingleRegex percent_encoding_re( + "%[A-Fa-f0-9]{2}", + err, + "percent_encoding"); +static const SingleRegex nosql_key_evasion_detector_re( + "\\w{1,48}\\s*[\\[\\{\\(]\\s*\\$\\w{1,48}\\s*[\\]\\}\\)]", + err, + "nosql_key_evasion_detector"); // looks for combination ={*:*} //used to allow parsing param=JSON to reduce false positives @@ -1909,6 +1918,16 @@ containsInvalidUtf8(const string &payload) return invalid_hex_evasion_re.hasMatch(payload); } +bool +containsPercentEncoding(const string &payload) +{ + static const size_t min_matches = 2; + vector regex_matches; + size_t counter = percent_encoding_re.findAllMatches(payload, regex_matches, min_matches); + // Check if there are at least two matches + return counter >= min_matches; +} + string unescapeInvalidUtf8(const string &payload) { @@ -2183,6 +2202,10 @@ void decodeUtf16Value(const ValueStatsAnalyzer &valueStats, string &cur_val) cur_val = utf8Out; } +bool testNoSQLKeySuspect(const string &key) { + return Waap::Util::nosql_key_evasion_detector_re.hasMatch(key); +} + bool testUrlBareUtf8Evasion(const string &line) { size_t percentPos = 0; diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index d91da07..98e3adb 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -230,7 +230,7 @@ _IT escape_backslashes(_IT first, _IT last) { _IT src = first; _IT dst = first; _IT mark = first; - + enum { STATE_COPY, STATE_ESCAPE, STATE_OCTAL, STATE_HEX } state = STATE_COPY; unsigned char accVal = 0; unsigned char digitsCount = 0; @@ -1137,6 +1137,8 @@ namespace Util { bool containsInvalidUtf8(const std::string &payload); + bool containsPercentEncoding(const std::string &payload); + // based on invalid utf-8 evasion from here: https://www.cgisecurity.com/lib/URLEmbeddedAttacks.html std::string unescapeInvalidUtf8(const std::string &text); @@ -1145,6 +1147,8 @@ namespace Util { bool containsCspReportPolicy(const std::string &payload); + bool testNoSQLKeySuspect(const std::string &key); + bool testUrlBareUtf8Evasion(const std::string &line); bool testUrlBadUtf8Evasion(const std::string &line); diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 0fa0f26..741c584 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -48,6 +48,7 @@ WaapComponent::Impl::Impl() : pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT), accept_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT), drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP), + limit_response_headers(ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS), waapStateTable(NULL), transactionsCount(0), deepAnalyzer() @@ -246,7 +247,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event) // Delete state before returning any verdict which is not pending if ((verdict.getVerdict() != pending_response.getVerdict()) && waapStateTable->hasState()) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, verdict); } else { } @@ -283,8 +284,9 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event) waf2Transaction.add_request_body_chunk(dataBuf, dataBufLen); ngx_http_cp_verdict_e verdict = waf2Transaction.getUserLimitVerdict(); + EventVerdict eventVedict(verdict); if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, eventVedict); } return EventVerdict(verdict); @@ -323,9 +325,10 @@ WaapComponent::Impl::respond(const EndRequestEvent &) // Delete state before returning any verdict which is not pending if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && + verdict.getVerdict() != ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS && waapStateTable->hasState() ) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, verdict); } return verdict; @@ -374,7 +377,7 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event) verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, verdict); } return verdict; @@ -412,6 +415,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) ngx_http_cp_verdict_e verdict = pending_response.getVerdict(); HttpHeaderModification modifications; + std::string webUserResponseByPractice; bool isSecurityHeadersInjected = false; if (waf2Transaction.shouldInjectSecurityHeaders()) { @@ -477,16 +481,16 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event) // disable should inject security headers after injection to avoid response body scanning when it's unnecessary waf2Transaction.disableShouldInjectSecurityHeaders(); } - + EventVerdict eventVedict(move(modifications.getModificationList()), verdict); // Delete state before returning any verdict which is not pending if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, eventVedict); } - return EventVerdict(move(modifications.getModificationList()), verdict); + return eventVedict; } EventVerdict @@ -528,6 +532,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) ngx_http_cp_verdict_e verdict = pending_response.getVerdict(); HttpBodyModification modifications; + std::string webUserResponseByPractice; // Set drop verdict if waap engine decides to drop response. if (!waf2Transaction.decideResponse()) { @@ -582,16 +587,16 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) dbgTrace(D_WAAP) << " * \e[32m HttpBodyResponse: shouldInspectResponse==false: ACCEPT\e[0m"; verdict = accept_response.getVerdict(); } - + EventVerdict eventVedict(modifications.getModificationList(), verdict); // Delete state before returning any verdict which is not pending or inject if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT && verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT && waapStateTable->hasState() ) { - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, eventVedict); } - return EventVerdict(modifications.getModificationList(), verdict); + return eventVedict; } EventVerdict @@ -630,7 +635,7 @@ WaapComponent::Impl::respond(const EndTransactionEvent &) } // This is our last chance to delete the state. The verdict must not be "PENDING" at this point. - finishTransaction(waf2Transaction); + finishTransaction(waf2Transaction, verdict); return verdict; } @@ -638,11 +643,13 @@ EventVerdict WaapComponent::Impl::waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction) { dbgTrace(D_WAAP) << "waapDecisionAfterHeaders() started"; + EventVerdict verdict = pending_response; if (waf2Transaction.decideAfterHeaders()) { dbgTrace(D_WAAP) << "WaapComponent::Impl::waapDecisionAfterHeaders(): returning DROP response."; - return drop_response; + verdict = drop_response; + return verdict; } - return pending_response; + return verdict; } EventVerdict @@ -660,8 +667,11 @@ WaapComponent::Impl::waapDecision(IWaf2Transaction& waf2Transaction) // (in the latter case - decision to drop/pass should be governed by failopen setting) if (verdictCode == 0) { waf2Transaction.checkShouldInject(); - - if (waf2Transaction.shouldInspectResponse()) { + if (waf2Transaction.shouldLimitResponseHeadersInspection()) { + dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mLIMIT RESPONSE HEADERS\e[0m)"; + verdict = limit_response_headers; + } else if (waf2Transaction.shouldInspectResponse()) { + dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mPENDING RESPONSE\e[0m)"; verdict = pending_response; } else { dbgTrace(D_WAAP) << "WAF VERDICT: " << verdictCode << " (\e[32mPASS\e[0m)"; @@ -678,10 +688,12 @@ WaapComponent::Impl::waapDecision(IWaf2Transaction& waf2Transaction) } void -WaapComponent::Impl::finishTransaction(IWaf2Transaction& waf2Transaction) +WaapComponent::Impl::finishTransaction(IWaf2Transaction& waf2Transaction, EventVerdict& verdict) { + dbgTrace(D_WAAP) << "finishTransaction() started"; waf2Transaction.collectFoundPatterns(); waf2Transaction.sendLog(); + verdict.setWebUserResponseByPractice(waf2Transaction.getCurrentWebUserResponse()); ReportIS::Severity severity = waf2Transaction.computeEventSeverityFromDecision(); validateFirstRequestForAsset(severity); waapStateTable->deleteState(); diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index 91ab2c3..2c233e9 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -59,7 +59,7 @@ private: void init(const std::string &waapDataFileName); EventVerdict waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction); EventVerdict waapDecision(IWaf2Transaction& waf2Transaction); - void finishTransaction(IWaf2Transaction& waf2Transaction); + void finishTransaction(IWaf2Transaction& waf2Transaction, EventVerdict& verdict); bool waf2_proc_start(const std::string& waapDataFileName); void waf2_proc_exit(); @@ -73,6 +73,7 @@ private: EventVerdict pending_response; EventVerdict accept_response; EventVerdict drop_response; + EventVerdict limit_response_headers; WaapMetricWrapper waap_metric; AssetsMetric assets_metric; I_Table* waapStateTable; diff --git a/components/utils/generic_rulebase/generic_rulebase.cc b/components/utils/generic_rulebase/generic_rulebase.cc index 228d5d3..ed13b7c 100644 --- a/components/utils/generic_rulebase/generic_rulebase.cc +++ b/components/utils/generic_rulebase/generic_rulebase.cc @@ -87,6 +87,7 @@ GenericRulebase::Impl::preload() addMatcher(); BasicRuleConfig::preload(); LogTriggerConf::preload(); + ReportTriggerConf::preload(); ParameterException::preload(); registerExpectedConfiguration("rulebase", "zones"); registerExpectedConfigFile("zones", Config::ConfigFileType::Policy); diff --git a/components/utils/generic_rulebase/match_query.cc b/components/utils/generic_rulebase/match_query.cc index 53b350a..17584e4 100644 --- a/components/utils/generic_rulebase/match_query.cc +++ b/components/utils/generic_rulebase/match_query.cc @@ -50,7 +50,7 @@ static const string ip_proto_type_name = "IP protocol"; static const unordered_map string_to_key = { { "sourceIP", MatchQuery::StaticKeys::SrcIpAddress }, - { "sourceIpAddr", MatchQuery::StaticKeys::SrcIpAddress }, + { "sourceIdentifier", MatchQuery::StaticKeys::SrcIpAddress }, { "destinationIP", MatchQuery::StaticKeys::DstIpAddress }, { "destinationIpAddr", MatchQuery::StaticKeys::DstIpAddress }, { "ipAddress", MatchQuery::StaticKeys::IpAddress }, @@ -319,7 +319,7 @@ MatchQuery::matchAttributes( match = matchAttributesString(values); dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match; } - + dbgTrace(D_RULEBASE_CONFIG) << "Should negate match? " << negate; return negate ? !match : match; } diff --git a/components/utils/generic_rulebase/parameters_config.cc b/components/utils/generic_rulebase/parameters_config.cc index e2575a7..8c0d953 100644 --- a/components/utils/generic_rulebase/parameters_config.cc +++ b/components/utils/generic_rulebase/parameters_config.cc @@ -125,9 +125,10 @@ ParameterException::getBehavior( // When matching indicators with action=ignore, we expect no behavior override. // Instead, a matched keywords list should be returned which will be later removed from score calculation if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) { - dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore"; matched_override_keywords.insert(match_res.matched_keywords->begin(), match_res.matched_keywords->end()); + dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore, found " << + matched_override_keywords.size() << "keywords"; } else { matched_behaviors.insert(match_behavior_pair.behavior); } @@ -143,6 +144,8 @@ ParameterException::getBehavior( if (match_res.matched_keywords->size() > 0 && behavior == action_ignore) { matched_override_keywords.insert(match_res.matched_keywords->begin(), match_res.matched_keywords->end()); + dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore, found " << + matched_override_keywords.size() << "keywords"; } else { matched_behaviors.insert(behavior); } @@ -155,6 +158,6 @@ ParameterException::getBehavior( set ParameterException::getBehavior(const unordered_map> &key_value_pairs) const { - set keywords; + set keywords; // placeholder only, this function will be used where there's no need for ignored keywords return getBehavior(key_value_pairs, keywords); } diff --git a/components/utils/generic_rulebase/triggers_config.cc b/components/utils/generic_rulebase/triggers_config.cc index 62df576..930dc15 100644 --- a/components/utils/generic_rulebase/triggers_config.cc +++ b/components/utils/generic_rulebase/triggers_config.cc @@ -241,3 +241,9 @@ LogTriggerConf::load(cereal::JSONInputArchive& archive_in) archive_in.setNextName(nullptr); } } + +void +ReportTriggerConf::load(cereal::JSONInputArchive& archive_in) +{ + parseJSONKey("triggerName", name, archive_in); +} diff --git a/components/utils/http_transaction_data/http_transaction_data.cc b/components/utils/http_transaction_data/http_transaction_data.cc index 31714f3..ec9c5ee 100644 --- a/components/utils/http_transaction_data/http_transaction_data.cc +++ b/components/utils/http_transaction_data/http_transaction_data.cc @@ -220,6 +220,18 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data) dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized parsed URI: " << ngx_parsed_uri.unpack(); } + // Try to read waf_tag if available + string waf_tag; + if (cur_pos < transaction_raw_data.size()) { + Maybe maybe_waf_tag = deserializeStrParam(transaction_raw_data, cur_pos); + if (maybe_waf_tag.ok()) { + waf_tag = maybe_waf_tag.unpackMove(); + dbgTrace(D_NGINX_ATTACHMENT) << "Successfully deserialized waf_tag: " << waf_tag; + } + } else { + dbgTrace(D_NGINX_ATTACHMENT) << "No waf_tag to deserialize, using empty string"; + } + // Fail if after parsing exact number of items, we didn't exactly consume whole buffer if (cur_pos != transaction_raw_data.size()) { dbgWarning(D_NGINX_ATTACHMENT) << "Nothing to deserialize, but raw data still remain"; @@ -239,6 +251,7 @@ HttpTransactionData::createTransactionData(const Buffer &transaction_raw_data) client_port.unpackMove() ); + transaction.setWafTag(waf_tag); return transaction; } diff --git a/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc b/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc index e15bc60..959afdb 100644 --- a/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc +++ b/components/utils/http_transaction_data/http_transaction_data_ut/http_transaction_data_ut.cc @@ -12,7 +12,7 @@ Buffer encodeInt16(uint16_t val) { vector raw_data(reinterpret_cast(&val), reinterpret_cast(&val) + sizeof(uint16_t)); - return move(Buffer(raw_data)); + return Buffer(raw_data); } class HttpTransactionTest : public Test diff --git a/components/utils/ip_utilities/ip_utilities.cc b/components/utils/ip_utilities/ip_utilities.cc index 9f77754..68ce26e 100644 --- a/components/utils/ip_utilities/ip_utilities.cc +++ b/components/utils/ip_utilities/ip_utilities.cc @@ -265,7 +265,7 @@ IpAddrToString(const IpAddress &address) sa6.sin6_addr = address.ip.ipv6; inet_ntop(AF_INET6, &(sa6.sin6_addr), ip_str, INET6_ADDRSTRLEN); - return move(string(ip_str)); + return string(ip_str); } char ip_str[INET_ADDRSTRLEN]; @@ -275,7 +275,7 @@ IpAddrToString(const IpAddress &address) sa.sin_addr = address.ip.ipv4; inet_ntop(AF_INET, &(sa.sin_addr), ip_str, INET_ADDRSTRLEN); - return move(string(ip_str)); + return string(ip_str); } IpAddress diff --git a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt index 56072c4..6e56c58 100755 --- a/components/utils/utilities/nginx_conf_collector/CMakeLists.txt +++ b/components/utils/utilities/nginx_conf_collector/CMakeLists.txt @@ -8,7 +8,7 @@ link_directories(${CMAKE_BINARY_DIR}/core) link_directories(${CMAKE_BINARY_DIR}/core/compression) SET(EXECUTABLE_NAME "nginx_conf_collector_bin") -add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc) +add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc fog_connection.cc) target_compile_definitions(${EXECUTABLE_NAME} PRIVATE "NGINX_CONF_COLLECTOR_VERSION=\"$ENV{CI_PIPELINE_ID}\"") target_link_libraries(${EXECUTABLE_NAME} @@ -26,6 +26,7 @@ target_link_libraries(${EXECUTABLE_NAME} report config environment + curl_http_client singleton rest boost_context diff --git a/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc b/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc index 092e15f..9a66369 100755 --- a/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc +++ b/components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc @@ -11,8 +11,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include +#include +#include +#include #include "agent_core_utilities.h" #include "debug.h" @@ -20,6 +24,7 @@ #include "mainloop.h" #include "nginx_utils.h" #include "time_proxy.h" +#include "fog_connection.h" using namespace std; @@ -43,6 +48,7 @@ public: environment.fini(); time_proxy.fini(); } + private: ShellCmd shell_cmd; MainloopComponent mainloop; @@ -63,12 +69,17 @@ printVersion() void printUsage(const char *prog_name) { - cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << '\n'; + cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << + " [--upload --token [--fog
]]" << '\n'; cout << " -V Print version" << '\n'; cout << " -v Enable verbose output" << '\n'; cout << " -i input_file Specify input file (default is /etc/nginx/nginx.conf)" << '\n'; cout << " -o output_file Specify output file (default is ./full_nginx.conf)" << '\n'; cout << " -h Print this help message" << '\n'; + cout << " --upload Upload configuration to FOG (requires --token)" << '\n'; + cout << " --token profile token for FOG upload" << '\n'; + cout << " --fog
FOG server address (default: inext-agents.cloud.ngen.checkpoint.com)" << '\n'; + cout << " --proxy
Proxy server to send the request through" << '\n'; } int @@ -76,9 +87,21 @@ main(int argc, char *argv[]) { string nginx_input_file = "/etc/nginx/nginx.conf"; string nginx_output_file = "full_nginx.conf"; - + string fog_address = "inext-agents.cloud.ngen.checkpoint.com"; + string token; + string proxy_host; + bool upload_flag = false; int opt; - while ((opt = getopt(argc, argv, "Vvhi:o:h")) != -1) { + + static struct option long_options[] = { + {"upload", no_argument, 0, 'u'}, + {"token", required_argument, 0, 1001}, + {"fog", required_argument, 0, 1002}, + {"proxy", required_argument, 0, 1003}, + {0, 0, 0, 0} + }; + + while ((opt = getopt_long(argc, argv, "Vvhi:o:", long_options, nullptr)) != -1) { switch (opt) { case 'V': printVersion(); @@ -95,18 +118,36 @@ main(int argc, char *argv[]) case 'h': printUsage(argv[0]); return 0; + case 'u': + upload_flag = true; + break; + case 1001: // --token + token = optarg; + break; + case 1002: // --fog + fog_address = optarg; + break; + case 1003: // --proxy + proxy_host = optarg; + break; default: printUsage(argv[0]); return 1; } } - for (int i = optind; i < argc;) { + for (int i = optind; i < argc; ++i) { cerr << "Unknown argument: " << argv[i] << '\n'; printUsage(argv[0]); return 1; } + if (upload_flag && token.empty()) { + cerr << "Error: --upload requires --token to be specified" << '\n'; + printUsage(argv[0]); + return 1; + } + dbgTrace(D_NGINX_MANAGER) << "Starting nginx configuration collector"; MainComponent main_component; @@ -144,5 +185,43 @@ main(int argc, char *argv[]) cout << "Full nginx configuration file was successfully generated: " << result.unpack() << '\n'; + if (upload_flag) { + cout << "Uploading configuration to FOG server: " << fog_address << '\n'; + + string full_fog_url = fog_address; + if (fog_address.find("http://") != 0 && fog_address.find("https://") != 0) { + full_fog_url = "https://" + fog_address; + } + + FogConnection fog_connection(token, full_fog_url); + + if (!proxy_host.empty()) { + fog_connection.setProxy(proxy_host); + } + + auto credentials_result = fog_connection.getCredentials(); + if (!credentials_result.ok()) { + cerr + << "Failed to register agent with the FOG. with error: " + << credentials_result.getErr() + << '\n'; + return 1; + } + + auto jwt_result = fog_connection.getJWT(); + if (!jwt_result.ok()) { + cerr << "Failed to get JWT token. with error:" << jwt_result.getErr() << '\n'; + return 1; + } + + auto upload_result = fog_connection.uploadNginxConfig(result.unpack()); + if (!upload_result.ok()) { + cerr << "Failed to upload nginx config file to FOG. with error:" << upload_result.getErr() << '\n'; + return 1; + } + + cout << "Successfully uploaded configuration to FOG server." << '\n'; + } + return 0; } diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 38fb5f6..610f9d6 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -32,6 +32,7 @@ add_subdirectory(compression) add_subdirectory(attachments) add_subdirectory(report_messaging) add_subdirectory(env_details) +add_subdirectory(curl_http_client) add_library(ngen_core SHARED ".") target_link_libraries( diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 6c24616..4e6b63f 100644 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -539,6 +539,14 @@ trim(string str) return removeLeadingWhitespaces(removeTrailingWhitespaces(str)); } +string +toLower(string str) +{ + transform(str.begin(), str.end(), str.begin(), ::tolower); + return str; +} + + } // namespace Strings } // namespace NGEN diff --git a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc index 334cd56..89d90d2 100644 --- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc +++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc @@ -197,6 +197,12 @@ TEST_F(AgentCoreUtilUT, trimTest) EXPECT_EQ(NGEN::Strings::trim(str_with_leading_and_trailing_whitespace), "str_with_whitespace"); } +TEST_F(AgentCoreUtilUT, toLowerTest) +{ + string str = "ThIS Is A 123 TEsT StRiNG"; + EXPECT_EQ(NGEN::Strings::toLower(str), "this is a 123 test string"); +} + TEST_F(AgentCoreUtilUT, resolveFullPathTest) { string working_dir = cptestFnameInExeDir(""); @@ -208,3 +214,43 @@ TEST_F(AgentCoreUtilUT, resolveFullPathTest) EXPECT_EQ(full_path, working_dir + "test.txt"); ASSERT_TRUE(NGEN::Filesystem::deleteFile(working_dir + "test.txt")); } + +TEST_F(AgentCoreUtilUT, regexReplaceTest) +{ + struct TestCase { + std::string input; + std::string expected; + }; + + std::vector test_cases = { + {"my?invalid//:filename*test.txt", "my_invalid_filename_test.txt"}, + {"hello///world", "hello_world"}, + {"file@@name..txt", "file_name..txt"}, + {"file--name", "file--name"}, + {"some@@@file!!name.txt", "some_file_name.txt"}, + {"https://some_file_name.txt", "https_some_file_name.txt"}, + {"spaces in filename.txt", "spaces_in_filename.txt"}, + {"trailing-dash-", "trailing-dash-"}, + {"trailing.dot.", "trailing.dot."}, + {"file name with (parens).txt", "file_name_with_parens_.txt"}, + {"$pecial#Chars&here.txt", "_pecial_Chars_here.txt"}, + {"___leading_underscores", "___leading_underscores"}, + {"<<<>>filename", "_weird_filename"}, + {"double..dots...txt", "double..dots...txt"}, + {"a:b|c*d?eg/h.txt", "a_b_c_d_e_f_g_h.txt"}, + {"/leading/slash", "_leading_slash"}, + {"back\\slash\\file", "back_slash_file"}, + {"file.with..multiple.dots.txt", "file.with..multiple.dots.txt"}, + {"CAPITAL&LETTERS^HERE", "CAPITAL_LETTERS_HERE"}, + {"123_456-789.ok", "123_456-789.ok"}, + {"__", "__"}, + {"*.*", "_._"} + }; + + boost::regex regex("[^\\w.-]+"); // Matches one or more non-word, non-dot, non-hyphen characters + + for (const auto& testCase : test_cases) { + std::string replaced = NGEN::Regex::regexReplace(__FILE__, __LINE__, testCase.input, regex, "_"); + EXPECT_EQ(replaced, testCase.expected); + } +} diff --git a/core/agent_details/agent_details.cc b/core/agent_details/agent_details.cc index 69dd219..b817b12 100644 --- a/core/agent_details/agent_details.cc +++ b/core/agent_details/agent_details.cc @@ -40,13 +40,19 @@ void AgentDetails::init() { registerMachineType(); - loadAccessToken(); - Singleton::Consume::by()->addRecurringRoutine( - I_MainLoop::RoutineType::System, - chrono::seconds(60), - [this] () { loadAccessToken(); }, - "Load access token" - ); + bool load_access_token = + getConfigurationWithDefault(true, "Agent details", "Load Access Token"); + + if (load_access_token) { + loadAccessToken(); + Singleton::Consume::by()->addRecurringRoutine( + I_MainLoop::RoutineType::System, + chrono::seconds(60), + [this] () { loadAccessToken(); }, + "Load access token" + ); + } + proxies = { {ProxyProtocol::HTTP, ProxyData()}, {ProxyProtocol::HTTPS, ProxyData()} @@ -277,6 +283,7 @@ AgentDetails::preload() { registerExpectedConfiguration("orchestration", "Agent details path"); registerExpectedConfiguration("Agent details", "File path"); + registerExpectedConfiguration("Agent details", "Load Access Token"); registerConfigLoadCb([this] () { readAgentDetails(); }); } diff --git a/core/buffers/buffer.cc b/core/buffers/buffer.cc index 37c16d1..50aa134 100644 --- a/core/buffers/buffer.cc +++ b/core/buffers/buffer.cc @@ -394,7 +394,7 @@ Buffer::isEqualLowerCase(const Buffer &buf) const { if (len != buf.size()) return false; for (uint i = 0; i < len; i++) { - if (tolower((*this)[i]) != buf[i]) return false; + if (tolower((*this)[i]) != tolower(buf[i])) return false; } return true; } diff --git a/core/config/config.cc b/core/config/config.cc index 31db962..2443b17 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -179,6 +179,8 @@ private: bool sendOrchestatorConfMsg(int env_listening_port) { + dbgTrace(D_CONFIG) << "Sending configuration to orchestrator, listening port: " << env_listening_port; + registerExpectedConfigUpdates config_updates; config_updates.service_name = executable_name; @@ -204,6 +206,7 @@ private: service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); service_config_req_md.setSuspension(false); + service_config_req_md.setShouldSendAccessToken(false); auto service_config_status = messaging->sendSyncMessage( HTTPMethod::POST, "/set-nano-service-config", @@ -212,10 +215,17 @@ private: service_config_req_md ); if (!service_config_status.ok()) { + dbgWarning(D_CONFIG) + << "Could not send configuration to orchestrator 7777, error: " + << service_config_status.getErr().getBody() + << ", error-code: " + << static_cast(service_config_status.getErr().getHTTPStatusCode()); + MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); secondary_port_req_md.setSuspension(false); + secondary_port_req_md.setShouldSendAccessToken(false); service_config_status = messaging->sendSyncMessage( HTTPMethod::POST, "/set-nano-service-config", @@ -224,8 +234,25 @@ private: secondary_port_req_md ); } + + if (!service_config_status.ok()) { + dbgWarning(D_CONFIG) + << "Could not send configuration to orchestrator 7778, error: " + << service_config_status.getErr().getBody() + << ", error-code: " + << static_cast(service_config_status.getErr().getHTTPStatusCode()); - return service_config_status.ok() && config_updates.status.get(); + return false; + } + + if (is_force_reload && config_updates.status.get()) { + dbgWarning(D_CONFIG) << "Reloading configuration due to force-reload config"; + if (reloadConfiguration("", false, 0) == I_Config::AsyncLoadConfigStatus::Success) is_force_reload = false; + } + + dbgTrace(D_CONFIG) << "Configuration successfully sent to orchestrator"; + + return config_updates.status.get() && !is_force_reload; } void @@ -254,6 +281,7 @@ private: service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); service_config_req_md.setSuspension(false); + service_config_req_md.setShouldSendAccessToken(false); bool service_config_status = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", @@ -262,10 +290,12 @@ private: service_config_req_md ); if (!service_config_status) { + dbgWarning(D_CONFIG) << "Could not send reconf-status to orchestrator 7777, retrying on 7778"; MessageMetadata secondary_port_req_md("127.0.0.1", 7778); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN); secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); secondary_port_req_md.setSuspension(false); + secondary_port_req_md.setShouldSendAccessToken(false); service_config_status = messaging->sendSyncMessageWithoutResponse( HTTPMethod::POST, "/set-reconf-status", @@ -275,9 +305,10 @@ private: ); } if (!service_config_status) { - dbgWarning(D_CONFIG) << "Unsuccessful attempt to send configuration reload status"; + dbgWarning(D_CONFIG) << "Could not send reconf-status to orchestrator 7778"; return false; } + return true; } @@ -303,6 +334,7 @@ private: vector configuration_abort_cbs; bool is_continuous_report = false; + bool is_force_reload = getenv("FORCE_CONFIG_RELOAD") && string(getenv("FORCE_CONFIG_RELOAD")) == "TRUE"; const string default_tenant_id = ""; const string default_profile_id = ""; string executable_name = ""; @@ -342,14 +374,19 @@ ConfigComponent::Impl::init() tenant_manager = Singleton::Consume::by(); if (!Singleton::exists()) return; - auto mainloop = Singleton::Consume::by(); - mainloop->addOneTimeRoutine( - I_MainLoop::RoutineType::System, - [this] () { periodicRegistrationRefresh(); }, - "Configuration update registration", - false - ); + bool periodic_registration_refresh = + getConfigurationWithDefault(true, "Config Component", "Periodic Registration Refresh"); + + if (periodic_registration_refresh) { + auto mainloop = Singleton::Consume::by(); + mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::System, + [this] () { periodicRegistrationRefresh(); }, + "Configuration update registration", + false + ); + } } static bool @@ -893,8 +930,9 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as dbgTrace(D_CONFIG) << "Could not open configuration file. Path: " << file.first; } } - + env->registerValue("Is Async Config Load", is_async); bool res = loadConfiguration(archives, is_async); + env->unregisterKey("Is Async Config Load"); if (res) env->registerValue("Current Policy Version", version); return res; } @@ -976,6 +1014,7 @@ ConfigComponent::preload() { registerExpectedConfiguration("Config Component", "configuration path"); registerExpectedConfiguration("Config Component", "Refresh config update registration time interval"); + registerExpectedConfiguration("Config Component", "Periodic Registration Refresh"); registerExpectedResource("Config Component", "Config Load Test"); registerExpectedSetting("agentSettings"); pimpl->preload(); diff --git a/core/include/attachments/compression_utils.h b/core/include/attachments/compression_utils.h index 5a25420..b867c3f 100644 --- a/core/include/attachments/compression_utils.h +++ b/core/include/attachments/compression_utils.h @@ -45,7 +45,8 @@ typedef enum CompressionType { NO_COMPRESSION, GZIP, - ZLIB + ZLIB, + BROTLI } CompressionType; typedef struct CompressionResult diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h index 7f6353f..9ab8305 100644 --- a/core/include/attachments/nginx_attachment_common.h +++ b/core/include/attachments/nginx_attachment_common.h @@ -146,7 +146,8 @@ typedef enum ngx_http_cp_verdict TRAFFIC_VERDICT_INJECT, TRAFFIC_VERDICT_IRRELEVANT, TRAFFIC_VERDICT_RECONF, - TRAFFIC_VERDICT_WAIT + TRAFFIC_VERDICT_WAIT, + LIMIT_RESPONSE_HEADERS } ngx_http_cp_verdict_e; #ifdef __cplusplus @@ -190,6 +191,8 @@ typedef enum ngx_http_meta_data PARSED_HOST_DATA, PARSED_URI_SIZE, PARSED_URI_DATA, + WAF_TAG_SIZE, + WAF_TAG_DATA, META_DATA_COUNT } ngx_http_meta_data_e; diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index 6b1de2c..1894299 100644 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -29,6 +29,7 @@ namespace Intelligence { class Invalidation; class Response; +class TimeRangeInvalidations; } // namespace Intelligence @@ -39,7 +40,8 @@ public: virtual bool isIntelligenceHealthy() const = 0; virtual Maybe registerInvalidation( const Intelligence::Invalidation &invalidation, - const std::function &callback + const std::function &callback, + const std::string &AgentId = "" ) = 0; virtual void unregisterInvalidation(uint id) = 0; virtual Maybe @@ -59,6 +61,10 @@ public: const MessageMetadata &req_md ) const = 0; + virtual Maybe> getInvalidations( + Intelligence::TimeRangeInvalidations request + ) const = 0; + template Maybe>> queryIntelligence( diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h index 49db3ff..33e5892 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h @@ -70,6 +70,31 @@ private: std::vector> sources; }; +class ExternalSourceError +{ +public: + ExternalSourceError() {} + + const std::string & getSourceID() const { return source_id; } + const std::string & getSourceName() const { return source_name; } + uint getStatusCode() const { return status_code; } + const std::string & getErrorMessage() const { return error_message; } + + void setSourceID(const std::string &id) { source_id = id; } + void setSourceName(const std::string &name) { source_name = name; } + void setStatusCode(uint code) { status_code = code; } + void setErrorMessage(const std::string &message) { error_message = message; } + + template + void serialize(Archive &ar); + +private: + std::string source_id = ""; + std::string source_name = ""; + uint status_code = 0; + std::string error_message = ""; +}; + class IntelligenceQueryResponse { public: @@ -83,6 +108,7 @@ public: Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; } const std::string & getCursor() const { return cursor; } uint getAmountOfAssets() const { return total_num_assets; } + const std::vector & getExternalSourcesErrorStatus() const; bool isValidInBulk() const { return !partial_fail_in_bulk; } void setFailInBulk() { partial_fail_in_bulk = true; } @@ -91,6 +117,7 @@ private: uint total_num_assets = 0; std::string cursor = ""; bool partial_fail_in_bulk = false; + std::vector external_sources_errors; }; template diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h index f1dea48..e62b8ab 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h @@ -21,6 +21,7 @@ #include #include "asset_reply.h" #include "bulk_query_response_v2.h" +#include "intelligence_invalidation.h" USE_DEBUG_FLAG(D_INTELLIGENCE); @@ -39,6 +40,11 @@ public: Maybe load(); Intelligence_IS_V2::ResponseStatus getResponseStatus() const; const std::string getCursor() const { return single_response.getCursor(); } + const std::vector & getExternalSourcesErrorStatus() const + { + return single_response.getExternalSourcesErrorStatus(); + } + void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; } template IntelligenceQueryResponseT getSerializableResponse() const @@ -98,10 +104,14 @@ public: return bulk_data; } + const std::vector& getInvalidations() const { return invalidations; } + Maybe loadInvalidations(); + private: std::string json_response; std::vector responses; IntelligenceQueryResponse single_response; + std::vector invalidations; size_t size = 0; bool is_bulk = false; }; diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h index ebcab54..448407c 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h @@ -34,7 +34,8 @@ public: const std::string &key, const std::string &value, bool full_response, - AttributeKeyType type = AttributeKeyType::MAIN + AttributeKeyType type = AttributeKeyType::MAIN, + bool _external_sources_error_status = false ); QueryRequest( @@ -42,7 +43,8 @@ public: const std::string &key, const int64_t &value, bool full_response, - AttributeKeyType type = AttributeKeyType::MAIN + AttributeKeyType type = AttributeKeyType::MAIN, + bool _external_sources_error_status = false ); QueryRequest( @@ -50,7 +52,8 @@ public: const std::string &key, const std::vector &value, bool full_response, - AttributeKeyType type = AttributeKeyType::MAIN + AttributeKeyType type = AttributeKeyType::MAIN, + bool _external_sources_error_status = false ); void saveToJson(cereal::JSONOutputArchive &ar) const; @@ -115,6 +118,7 @@ public: private: uint assets_limit = default_assets_limit; bool full_response = false; + bool external_sources_error_status = false; Maybe object_type = genError("uninitialized"); Maybe cursor = genError("Cursor not initialized"); SerializableQueryFilter query; diff --git a/core/include/services_sdk/interfaces/messaging/interface_impl.h b/core/include/services_sdk/interfaces/messaging/interface_impl.h index a571bd3..ea6ea6d 100644 --- a/core/include/services_sdk/interfaces/messaging/interface_impl.h +++ b/core/include/services_sdk/interfaces/messaging/interface_impl.h @@ -19,8 +19,12 @@ USE_DEBUG_FLAG(D_MESSAGING); -MessageMetadata::MessageMetadata() +MessageMetadata::MessageMetadata(bool immediate_tracing) { + if (immediate_tracing && Singleton::exists()) { + insertHeaders(Singleton::Consume::by()->getCurrentHeadersMap()); + } + if (!Singleton::exists() || !Singleton::exists()) return; auto i_agent_details = Singleton::Consume::by(); auto i_proxy_configuration = Singleton::Consume::by(); @@ -137,6 +141,8 @@ I_Messaging::sendAsyncMessage( return; } + dbgTrace(D_MESSAGING) << "Sending async message. URI: " << uri << ", Body: " << req_body.unpack(); + sendAsyncMessage( method, uri, diff --git a/core/include/services_sdk/interfaces/messaging/messaging_enums.h b/core/include/services_sdk/interfaces/messaging/messaging_enums.h index 81f9702..1d3674e 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_enums.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h @@ -30,6 +30,7 @@ enum class MessageConnectionConfig UNSECURE_CONN, ONE_TIME_CONN, IGNORE_SSL_VALIDATION, + ONE_TIME_FOG_CONN, // used for learning mechanism - one time connection sent by dedicated thread COUNT }; diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h index af7ef27..c0208a3 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -9,6 +9,7 @@ #include "singleton.h" #include "i_agent_details.h" #include "i_time_get.h" +#include "i_environment.h" class MessageProxySettings { @@ -55,14 +56,27 @@ private: uint16_t proxy_port = 0; }; -class MessageMetadata : Singleton::Consume +class MessageMetadata : Singleton::Consume, Singleton::Consume { public: - inline MessageMetadata(); + inline MessageMetadata(bool immediate_tracing = false); - MessageMetadata(const std::string &_host_name, uint16_t _port_num, bool _buffer = false, bool _fog = false) : - host_name(_host_name), port_num(_port_num), should_buffer(_buffer), is_to_fog(_fog) - {} + MessageMetadata( + const std::string &_host_name, + uint16_t _port_num, + bool _buffer = false, + bool _fog = false, + bool immediate_tracing = false + ) : + host_name(_host_name), + port_num(_port_num), + should_buffer(_buffer), + is_to_fog(_fog) + { + if (immediate_tracing && Singleton::exists()) { + insertHeaders(Singleton::Consume::by()->getCurrentHeadersMap()); + } + } MessageMetadata( std::string _host_name, @@ -70,7 +84,8 @@ public: Flags _conn_flags, bool _should_buffer = false, bool _is_to_fog = false, - bool _should_suspend = true + bool _should_suspend = true, + bool immediate_tracing = false ) : host_name(_host_name), port_num(_port_num), @@ -79,7 +94,11 @@ public: is_to_fog(_is_to_fog), should_send_access_token(true), should_suspend(_should_suspend) - {} + { + if (immediate_tracing && Singleton::exists()) { + insertHeaders(Singleton::Consume::by()->getCurrentHeadersMap()); + } + } const bool & shouldSendAccessToken() const @@ -135,6 +154,14 @@ public: return headers; } + Maybe + getTraceId() const + { + auto trace_id = headers.find("X-Trace-Id"); + if (trace_id != headers.end()) return trace_id->second; + return genError("Trace ID not found"); + } + std::string getCaPath() const { diff --git a/core/include/services_sdk/interfaces/mock/mock_intelligence.h b/core/include/services_sdk/interfaces/mock/mock_intelligence.h index ab8365e..f851e21 100644 --- a/core/include/services_sdk/interfaces/mock/mock_intelligence.h +++ b/core/include/services_sdk/interfaces/mock/mock_intelligence.h @@ -16,16 +16,31 @@ operator<<(std::ostream &os, const Intelligence::Invalidation &) return os; } +std::ostream & +operator<<(std::ostream &os, const std::vector &) +{ + return os; +} + class MockIntelligence : public Singleton::Provide::From> { public: using InvalidationCb = std::function; using Invalidation = Intelligence::Invalidation; using Response = Intelligence::Response; + using TimeRangeInvalidations = Intelligence::TimeRangeInvalidations; MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation)); + MOCK_CONST_METHOD1(getInvalidations, Maybe>(TimeRangeInvalidations)); MOCK_CONST_METHOD0(isIntelligenceHealthy, bool(void)); - MOCK_METHOD2(registerInvalidation, Maybe(const Invalidation &invalidation, const InvalidationCb &callback)); + MOCK_METHOD3( + registerInvalidation, + Maybe( + const Invalidation &invalidation, + const InvalidationCb &callback, + const std::string &AgentId + ) + ); MOCK_METHOD1(unregisterInvalidation, void(uint id)); MOCK_CONST_METHOD5( getResponse, diff --git a/core/include/services_sdk/resources/config/config_loader.h b/core/include/services_sdk/resources/config/config_loader.h index 673e68d..8ef7a28 100644 --- a/core/include/services_sdk/resources/config/config_loader.h +++ b/core/include/services_sdk/resources/config/config_loader.h @@ -55,7 +55,7 @@ public: std::pair>, TypeWrapper> getLoaderConfig() { - return std::move(std::make_pair(std::move(context), TypeWrapper(value))); + return std::make_pair(std::move(context), TypeWrapper(value)); } private: diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index b6bf3d6..51604a3 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -35,6 +35,7 @@ DEFINE_FLAG(D_INFRA, D_ALL) DEFINE_FLAG(D_TENANT_MANAGER, D_INFRA) DEFINE_FLAG(D_MONITORING, D_INFRA) DEFINE_FLAG(D_SERVICE_HEALTH_STATUS, D_INFRA) + DEFINE_FLAG(D_LOGGING, D_INFRA) DEFINE_FLAG(D_REPORT, D_INFRA) DEFINE_FLAG(D_REPORT_BULK, D_REPORT) DEFINE_FLAG(D_TRACE, D_INFRA) @@ -48,6 +49,9 @@ DEFINE_FLAG(D_INFRA, D_ALL) DEFINE_FLAG(D_CONNECTION, D_MESSAGING) DEFINE_FLAG(D_MESSAGING_BUFFER, D_MESSAGING) DEFINE_FLAG(D_HTTP_REQUEST, D_MESSAGING) + DEFINE_FLAG(D_TRACE_ID, D_MESSAGING) + DEFINE_FLAG(D_MEMORY, D_INFRA) + DEFINE_FLAG(D_WAAP_MEMORY, D_MEMORY) DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_PRELOAD, D_COMPONENT) @@ -72,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_SAMPLE_SCAN, D_WAAP) DEFINE_FLAG(D_WAAP_ASSET_STATE, D_WAAP) DEFINE_FLAG(D_WAAP_CONFIDENCE_CALCULATOR, D_WAAP) + DEFINE_FLAG(D_WAAP_SERIALIZE, D_WAAP) DEFINE_FLAG(D_WAAP_REPUTATION, D_WAAP) DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP) DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP) @@ -153,6 +158,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_SDWAN, D_COMPONENT) DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN) DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN) + DEFINE_FLAG(D_SDWAN_DATA_SENDER, D_SDWAN_DATA) DEFINE_FLAG(D_SDWAN_FEATURE_FLAG, D_SDWAN) DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN) DEFINE_FLAG(D_SDWAN_API, D_SDWAN) @@ -196,6 +202,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT) DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT) DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT) + DEFINE_FLAG(D_AIGUARD, D_COMPONENT) DEFINE_FLAG(D_FLOW, D_ALL) DEFINE_FLAG(D_DROP, D_FLOW) diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h index 87d680e..d9bd543 100644 --- a/core/include/services_sdk/resources/generic_metric.h +++ b/core/include/services_sdk/resources/generic_metric.h @@ -116,7 +116,7 @@ private: friend class MetricCalc; void addCalc(MetricCalc *calc); - std::vector getPromMetricsData(); + std::vector getPromMetricsData(const std::vector *allowed_calcs = nullptr); void handleMetricStreamSending(); void generateLog(); diff --git a/core/include/services_sdk/resources/intelligence_invalidation.h b/core/include/services_sdk/resources/intelligence_invalidation.h index d56c139..e051a13 100644 --- a/core/include/services_sdk/resources/intelligence_invalidation.h +++ b/core/include/services_sdk/resources/intelligence_invalidation.h @@ -19,6 +19,9 @@ #include #include #include +#include "rest.h" +#include "messaging/messaging_enums.h" +#include "messaging/messaging_metadata.h" #include "maybe_res.h" #include "enum_array.h" @@ -32,6 +35,56 @@ enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND }; enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED }; enum class InvalidationType { ADD, DELETE, UPDATE }; +static const std::map stringToObjectTypeMap = { + {"asset", ObjectType::ASSET}, + {"zone", ObjectType::ZONE}, + {"policyPackage", ObjectType::POLICY_PACKAGE}, + {"configuration", ObjectType::CONFIGURATION}, + {"session", ObjectType::SESSION}, + {"shortLived", ObjectType::SHORTLIVED} +}; + +static const std::map stringToInvalidationTypeMap = { + {"add", InvalidationType::ADD}, + {"delete", InvalidationType::DELETE}, + {"update", InvalidationType::UPDATE} +}; + +class TimeRangeInvalidations +{ +public: + TimeRangeInvalidations(uint64_t start_time, uint64_t end_time) : time_range{start_time, end_time} {} + + Maybe genJson() const + { + try { + std::stringstream out; + { + cereal::JSONOutputArchive out_ar(out); + out_ar(cereal::make_nvp("timeRange", time_range)); + } + return out.str(); + } catch (const std::exception &e) { + return genError("Failed to generate JSON for TimeRangeInvalidations. Error: " + std::string(e.what())); + } + } + +private: + struct TimeRange + { + uint64_t start; + uint64_t end; + + template + void serialize(Archive &ar) + { + ar(cereal::make_nvp("start", start), cereal::make_nvp("end", end)); + } + }; + + TimeRange time_range; +}; + class StrAttributes { public: @@ -101,6 +154,7 @@ private: class Invalidation { public: + Invalidation(); Invalidation(const std::string &class_value); Invalidation & setClassifier(ClassifierType type, const std::string &val); @@ -113,14 +167,18 @@ public: std::string getClassifier(ClassifierType type) const { return classifiers[type]; } std::vector getMainAttributes() const { return main_attributes; } std::vector getAttributes() const { return attributes; } - const Maybe & getSourceId() const { return source_id; } - const Maybe & getObjectType() const { return object_type; } - const Maybe & getInvalidationType() const { return invalidation_type; } - Maybe getRegistrationID() const; + const Maybe & getSourceId() const { return source_id; } + const Maybe & getObjectType() const { return object_type; } + const Maybe & getInvalidationType() const { return invalidation_type; } + Maybe getRegistrationID() const; bool report(I_Intelligence_IS_V2 *interface) const; - Maybe startListening(I_Intelligence_IS_V2 *interface, const std::function &cb); + Maybe startListening( + I_Intelligence_IS_V2 *interface, + const std::function &cb, + const std::string &AgentId = "" + ); void stopListening(I_Intelligence_IS_V2 *interface); Maybe genJson() const; @@ -128,6 +186,7 @@ public: bool isLegalInvalidation() const; bool matches(const Invalidation &other) const; + void serialize(cereal::JSONInputArchive &ar); private: bool attr_matches(const std::vector ¤t, const std::vector &other) const; @@ -136,11 +195,11 @@ private: EnumArray classifiers; std::vector main_attributes; std::vector attributes; - Maybe source_id; - Maybe object_type; - Maybe invalidation_type; - Maybe listening_id; - Maybe registration_id; + Maybe source_id; + Maybe object_type; + Maybe invalidation_type; + Maybe listening_id; + Maybe registration_id; }; } // namespace Intelligence diff --git a/core/include/services_sdk/resources/log_generator.h b/core/include/services_sdk/resources/log_generator.h index 8d42618..b62268b 100644 --- a/core/include/services_sdk/resources/log_generator.h +++ b/core/include/services_sdk/resources/log_generator.h @@ -31,16 +31,6 @@ class LogGen Singleton::Consume { public: - template - LogGen( - const Trigger &trigger, - const std::string &title, - Args ...args) - : - LogGen(trigger(title, std::forward(args)...)) - { - } - template LogGen( const std::string &title, diff --git a/core/include/services_sdk/resources/report/report_bulks.h b/core/include/services_sdk/resources/report/report_bulks.h index b8274f7..ae417fe 100644 --- a/core/include/services_sdk/resources/report/report_bulks.h +++ b/core/include/services_sdk/resources/report/report_bulks.h @@ -51,6 +51,9 @@ public: dbgDebug(D_REPORT_BULK) << "Adding a new bulk to queue"; bulks.push(LogBulkRest(bulk_size));; } + dbgTrace(D_REPORT_BULK) + << "Adding report to bulk, for asset: " + << (report.getStringData("assetName").ok() ? *report.getStringData("assetName") : "unknown"); bulks.back().push(report); ++elem_in_quque; } diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 6358673..3c3cfd3 100644 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -71,6 +71,7 @@ enum class Tags { DEPLOYMENT_DOCKER, WEB_SERVER_SWAG, WEB_SERVER_NGINX_UNIFIED, + AIGUARD, COUNT }; diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index e6a8e5c..ea0a821 100644 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -85,6 +85,7 @@ namespace Strings std::string removeTrailingWhitespaces(std::string str); std::string removeLeadingWhitespaces(std::string str); std::string trim(std::string str); +std::string toLower(std::string str); } // namespace Strings diff --git a/core/intelligence_is_v2/asset_replay.cc b/core/intelligence_is_v2/asset_replay.cc index 20ef167..374923c 100644 --- a/core/intelligence_is_v2/asset_replay.cc +++ b/core/intelligence_is_v2/asset_replay.cc @@ -15,6 +15,18 @@ using namespace std; +template +void +ExternalSourceError::serialize(Archive &ar) +{ + ar( + cereal::make_nvp("sourceID", source_id), + cereal::make_nvp("sourceName", source_name), + cereal::make_nvp("statusCode", status_code), + cereal::make_nvp("errorMessage", error_message) + ); +} + void IntelligenceQueryResponse::loadFromJson(const std::string &json_response) { @@ -35,4 +47,14 @@ IntelligenceQueryResponse::serialize(Archive &ar) try { ar(cereal::make_nvp("cursor", cursor)); } catch (...) {} + + try { + ar(cereal::make_nvp("externalSourcesErrorStatus", external_sources_errors)); + } catch (...) {} +} + +const std::vector & +IntelligenceQueryResponse::getExternalSourcesErrorStatus() const +{ + return external_sources_errors; } diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 89008eb..193e11b 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -36,6 +36,7 @@ static const string query_uri = "/api/v2/intelligence/assets/query"; static const string queries_uri = "/api/v2/intelligence/assets/queries"; static const string fog_health_uri = "/access-manager/health/live"; static const string intelligence_health_uri = "/show-health"; +static const string time_range_invalidation_uri = "/api/v2/intelligence/invalidation/get"; class I_InvalidationCallBack { @@ -83,6 +84,12 @@ public: first = false; } + void + setAgentId(const string &_agent_id) + { + agent_id = _agent_id; + } + RestCall genJson() const { @@ -90,7 +97,7 @@ public: res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", \"callbackType\": \"invalidation\", "; auto details = Singleton::Consume::by(); - res << "\"name\": \"" << details->getAgentId() << "\", "; + res << "\"name\": \"" << (agent_id.empty() ? details->getAgentId() : agent_id) << "\", "; auto rest = Singleton::Consume::by(); res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", "; res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, "; @@ -106,6 +113,7 @@ public: private: bool first = true; stringstream stream; + string agent_id = ""; }; class InvalidationCallBack : Singleton::Provide::SelfInterface @@ -137,10 +145,12 @@ public: bool empty() const { return callbacks.empty(); } InvalidationRegistration::RestCall - getRegistration() const + getRegistration(const string& agent_id) const { InvalidationRegistration registration; + registration.setAgentId(agent_id); + for (auto ®isted_invalidation : callbacks) { registration.addInvalidation(registed_invalidation.second.first); } @@ -394,11 +404,24 @@ public: return sendIntelligence(invalidation).ok(); } + Maybe> + getInvalidations(TimeRangeInvalidations request) const override + { + auto res = sendIntelligence(request); + if (res.ok()) return res.unpack().getInvalidations(); + return res.passErr(); + } + Maybe - registerInvalidation(const Invalidation &invalidation, const function &cb) override + registerInvalidation( + const Invalidation &invalidation, + const function &cb, + const string &AgentId + ) override { if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation"); auto res = invalidations.emplace(invalidation, cb); + agent_id = AgentId; sendRecurringInvalidationRegistration(); return res; } @@ -504,7 +527,7 @@ private: const string &server, const string &port_setting, const bool should_send_access_token = false -) const + ) const { auto port = getSetting("intelligence", port_setting); if (!port.ok()) { @@ -548,6 +571,16 @@ private: return load_status.passErr(); } + Maybe + createResponse(const string &response_body) const + { + Response response(response_body, 0, false); + auto load_status = response.loadInvalidations(); + if (load_status.ok()) return response; + dbgWarning(D_INTELLIGENCE) << "Could not create intelligence response."; + return load_status.passErr(); + } + Maybe sendIntelligenceRequestImpl(const Invalidation &invalidation, const MessageMetadata &local_req_md) const { @@ -642,6 +675,39 @@ private: return createResponse(req_data->getBody(), query_request); } + Maybe + sendIntelligenceRequestImpl( + const TimeRangeInvalidations &request, + const MessageMetadata &global_req_md) const + { + dbgFlow(D_INTELLIGENCE) << "Sending time range invalidations request"; + + auto json_body = request.genJson(); + if (!json_body.ok()) return json_body.passErr(); + + auto req_data = message->sendSyncMessage( + HTTPMethod::POST, + time_range_invalidation_uri, + *json_body, + MessageCategory::INTELLIGENCE, + global_req_md + ); + if (!req_data.ok()) { + dbgWarning(D_INTELLIGENCE) + << "Could not send time range invalidations request. " + << req_data.getErr().getBody() + << " " + << req_data.getErr().toString(); + return genError("Could not send time range invalidations request."); + } + if (req_data->getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) { + dbgWarning(D_INTELLIGENCE) << "Invalid intelligence response: " << req_data->toString(); + return genError(req_data->toString()); + } + + return createResponse(req_data->getBody()); + } + map getHTTPHeaders() const { @@ -651,7 +717,7 @@ private: if (tenant == "") tenant = "Global"; headers["X-Tenant-Id"] = tenant; auto rest = Singleton::Consume::by(); - auto agent = details->getAgentId() + ":" + to_string(rest->getListeningPort()); + auto agent = (agent_id.empty() ? details->getAgentId() : agent_id) + ":" + to_string(rest->getListeningPort()); headers["X-Source-Id"] = agent; return headers; @@ -662,7 +728,7 @@ private: { if (invalidations.empty()) return; - sendLocalIntelligenceToLocalServer(invalidations.getRegistration()); + sendLocalIntelligenceToLocalServer(invalidations.getRegistration(agent_id)); } Maybe @@ -678,6 +744,7 @@ private: InvalidationCallBack invalidations; I_Messaging *message = nullptr; I_MainLoop *mainloop = nullptr; + string agent_id = ""; }; IntelligenceComponentV2::IntelligenceComponentV2() diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index 9957550..5015596 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -1,8 +1,13 @@ #include -namespace Intelligence { class Response; } +#include +namespace Intelligence +{ + class Response; + class Invalidation; +} std::ostream & operator<<(std::ostream &os, const Intelligence::Response &); - -#include "intelligence_comp_v2.h" +std::ostream & operator<<(std::ostream &os, const Intelligence::Invalidation &); +std::ostream & operator<<(std::ostream &os, const std::vector &); #include "config.h" #include "config_component.h" @@ -15,6 +20,8 @@ std::ostream & operator<<(std::ostream &os, const Intelligence::Response &); #include "mock/mock_agent_details.h" #include "read_attribute_v2.h" #include "singleton.h" +#include "intelligence_comp_v2.h" +#include "intelligence_invalidation.h" using namespace std; using namespace testing; @@ -1397,3 +1404,68 @@ TEST_F(IntelligenceComponentTestV2, localIntelligenceHealthy) EXPECT_TRUE(intell->isIntelligenceHealthy()); } + +TEST_F(IntelligenceComponentTestV2, getInvalidations) +{ + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + Debug::setNewDefaultStdout(&cout); + + HTTPResponse response_str( + HTTPStatusCode::HTTP_OK, + string( + "{ \"invalidations\": [ { " + "\"class\": \"aaa\", " + "\"category\": \"bbb\", " + "\"family\": \"ccc\", " + "\"group\": \"ddd\", " + "\"order\": \"eee\", " + "\"kind\": \"fff\", " + "\"objectType\": \"asset\", " + "\"sourceId\": \"id\", " + "\"mainAttributes\": [ { \"attr\": \"2\" } ], " + "\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.2\" ], " + "\"ipv4AddressesRange\": [ { \"max\": \"1.1.1.5\", \"min\": \"1.1.1.1\" } ] } ]" + " } ] }" + ) + ); + + Intelligence::TimeRangeInvalidations time_range_invalidations(1622505600, 1622592000); + auto json_body = time_range_invalidations.genJson(); + + EXPECT_TRUE(json_body.ok()); + EXPECT_CALL(mock_rest, getListeningPort()).WillOnce(Return(8888)); + EXPECT_CALL( + messaging_mock, + sendSyncMessage( + HTTPMethod::POST, + "/api/v2/intelligence/invalidation/get", + json_body.unpack(), + MessageCategory::INTELLIGENCE, + _ + ) + ).WillOnce(Return(response_str)); + + auto main_attr = Intelligence::StrAttributes() + .addStringAttr("attr", "2"); + Intelligence::IpAddressRange range("1.1.1.1", "1.1.1.5"); + Intelligence::IpAttributes attributes = Intelligence::IpAttributes() + .addIpv4Addresses("1.1.1.2") + .addIpv4AddressRanges(range); + + auto invalidation = Intelligence::Invalidation("aaa") + .addMainAttr(main_attr) + .addAttr(attributes) + .setSourceId("id") + .setClassifier(Intelligence::ClassifierType::FAMILY, "ccc") + .setClassifier(Intelligence::ClassifierType::CATEGORY, "bbb") + .setClassifier(Intelligence::ClassifierType::GROUP, "ddd") + .setClassifier(Intelligence::ClassifierType::ORDER, "eee") + .setClassifier(Intelligence::ClassifierType::KIND, "fff") + .setObjectType(Intelligence::ObjectType::ASSET); + + auto res = intell->getInvalidations(time_range_invalidations); + + EXPECT_TRUE(res.ok()); + EXPECT_TRUE(res.unpack().front().matches(invalidation)); +} diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc index 9c6283c..e4a3059 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc @@ -63,6 +63,28 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequestProxied) { EXPECT_EQ(*query.genJson(), expected); } +TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequestExternalError) { + QueryRequest request(Condition::EQUALS, "phase", "testing", true, AttributeKeyType::MAIN, true); + vector requests = {request}; + Intelligence::IntelligenceRequest query(requests, true, false, true, MessageMetadata("", 0)); + + std::string expected = "{\n" + " \"queryTypes\": {\n" + " \"proxyToCloud\": true\n" + " },\n" + " \"limit\": 20,\n" + " \"fullResponse\": true,\n" + " \"externalSourcesErrorStatus\": true,\n" + " \"query\": {\n" + " \"operator\": \"equals\",\n" + " \"key\": \"mainAttributes.phase\",\n" + " \"value\": \"testing\"\n" + " }\n" + "}"; + + EXPECT_EQ(*query.genJson(), expected); +} + TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); vector requests = {request}; diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc index 25ef4a3..dc7ab27 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc @@ -87,13 +87,14 @@ TEST(QueryRequestTestV2, QueryTest) range2.push_back("224.11.10.16"); range2.push_back("224.11.10.31"); - QueryRequest request3(Condition::RANGE, "ipv4AddressesRange", range1, true); + QueryRequest request3(Condition::RANGE, "ipv4AddressesRange", range1, true, AttributeKeyType::MAIN, true); request3.addCondition(Condition::RANGE, "ipv4AddressesRange", range2); string output_json3= "{\n" " \"limit\": 20,\n" " \"fullResponse\": true,\n" + " \"externalSourcesErrorStatus\": true,\n" " \"query\": {\n" " \"operator\": \"and\",\n" " \"operands\": [\n" diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc index 9fe71c4..ff00aa9 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc @@ -145,7 +145,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2) " ],\n" " \"status\": \"done\",\n" " \"totalNumAssets\": 2,\n" - " \"cursor\": \"start\"\n" + " \"cursor\": \"start\",\n" + " \"externalSourcesErrorStatus\": []\n" "}\n" ); @@ -159,6 +160,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2) EXPECT_EQ(obj2.getAmountOfAssets(), 2u); EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE); EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE); + EXPECT_TRUE(obj.getExternalSourcesErrorStatus().empty()); + EXPECT_TRUE(obj2.getExternalSourcesErrorStatus().empty()); EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1u); EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip"); EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1u); @@ -217,6 +220,86 @@ TEST(QueryResponseTestV2, QueryResponseTestV2) EXPECT_EQ(asset_sources_it->getData1().toString(), "Max"); } +TEST(QueryResponseTestV2, ExternalSourcesErrorStatusTestV2) +{ + DataString data; + IntelligenceQueryResponseT obj; + string string_attribute( + "{\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-ip\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"some-group-id\",\n" + " \"name\": \"[1.1.1.1]\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"ip\",\n" + " \"group\": \"\",\n" + " \"order\": \"\",\n" + " \"kind\": \"\",\n" + " \"mainAttributes\": {\n" + " \"team\": \"hapoel\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"color\": \"red\",\n" + " \"user\": \"Omry\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " ],\n" + " \"status\": \"done\",\n" + " \"totalNumAssets\": 1,\n" + " \"cursor\": \"start\",\n" + " \"externalSourcesErrorStatus\": [\n" + " {\n" + " \"sourceID\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"sourceName\": \"test-source-1\",\n" + " \"statusCode\": 500,\n" + " \"errorMessage\": \"Internal server error\"\n" + " },\n" + " {\n" + " \"sourceID\": \"a1b2c3d4-5678-9abc-def0-123456789abc\",\n" + " \"sourceName\": \"test-source-2\",\n" + " \"statusCode\": 404,\n" + " \"errorMessage\": \"Not found\"\n" + " }\n" + " ]\n" + "}\n" + ); + + stringstream ss(string_attribute); + { + cereal::JSONInputArchive ar(ss); + obj.serialize(ar); + } + + const auto& errors = obj.getExternalSourcesErrorStatus(); + EXPECT_EQ(errors.size(), 2u); + + EXPECT_EQ(errors[0].getSourceID(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00"); + EXPECT_EQ(errors[0].getSourceName(), "test-source-1"); + EXPECT_EQ(errors[0].getStatusCode(), 500u); + EXPECT_EQ(errors[0].getErrorMessage(), "Internal server error"); + + EXPECT_EQ(errors[1].getSourceID(), "a1b2c3d4-5678-9abc-def0-123456789abc"); + EXPECT_EQ(errors[1].getSourceName(), "test-source-2"); + EXPECT_EQ(errors[1].getStatusCode(), 404u); + EXPECT_EQ(errors[1].getErrorMessage(), "Not found"); +} + TEST(QueryResponseTestV2, MainAttributesTestV2) { DataString data; diff --git a/core/intelligence_is_v2/intelligence_response.cc b/core/intelligence_is_v2/intelligence_response.cc index 8ed56ba..e1cbb35 100644 --- a/core/intelligence_is_v2/intelligence_response.cc +++ b/core/intelligence_is_v2/intelligence_response.cc @@ -58,6 +58,20 @@ Response::load() return {}; } +Maybe +Response::loadInvalidations() +{ + try { + stringstream in; + in.str(json_response); + cereal::JSONInputArchive in_ar(in); + in_ar(cereal::make_nvp("invalidations", invalidations)); + } catch(const std::exception &e) { + return genError("Load invalidations failed. Error: " + string(e.what())); + } + return {}; +} + Intelligence_IS_V2::ResponseStatus Response::getResponseStatus() const { diff --git a/core/intelligence_is_v2/invalidation.cc b/core/intelligence_is_v2/invalidation.cc index 7281c7c..4dbfb97 100644 --- a/core/intelligence_is_v2/invalidation.cc +++ b/core/intelligence_is_v2/invalidation.cc @@ -25,13 +25,22 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); using namespace Intelligence; using namespace std; +Invalidation::Invalidation() + : + source_id(genError("")), + object_type(genError("")), + invalidation_type(genError("")), + listening_id(genError("")), + registration_id(genError("")) +{} + Invalidation::Invalidation(const string &class_value) : - source_id(genError()), - object_type(genError()), - invalidation_type(genError()), - listening_id(genError()), - registration_id(genError()) + source_id(genError("")), + object_type(genError("")), + invalidation_type(genError("")), + listening_id(genError("")), + registration_id(genError("")) { setClassifier(ClassifierType::CLASS, class_value); } @@ -72,10 +81,14 @@ Invalidation::report(I_Intelligence_IS_V2 *interface) const } Maybe -Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function &cb) +Invalidation::startListening( + I_Intelligence_IS_V2 *interface, + const function &cb, + const string &AgentId +) { registration_id = to_string(boost::uuids::random_generator()()); - auto res = interface->registerInvalidation(*this, cb); + auto res = interface->registerInvalidation(*this, cb, AgentId); if (res.ok()) listening_id = *res; return res; } @@ -243,6 +256,117 @@ Invalidation::matches(const Invalidation &other) const return true; } +void +Invalidation::serialize(cereal::JSONInputArchive &ar) +{ + std::string class_ = ""; + std::string category = ""; + std::string family = ""; + std::string group = ""; + std::string order = ""; + std::string kind = ""; + std::string object_type_; + std::string invalidation_type_; + std::string source_id_; + uint listening_id_; + std::string registration_id_; + + try { + ar(cereal::make_nvp("class", class_)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("category", category)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("family", family)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("group", group)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("order", order)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("kind", kind)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("mainAttributes", main_attributes)); + ar(cereal::make_nvp("attributes", attributes)); + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("objectType", object_type_)); + auto it = stringToObjectTypeMap.find(object_type_); + if (it != stringToObjectTypeMap.end()) { + object_type = it->second; + } else { + throw std::invalid_argument("Invalid string for ObjectType: " + object_type_); + } + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("sourceId", source_id_)); + source_id = source_id_; + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("invalidationRegistrationId", registration_id_)); + registration_id = registration_id_; + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("invalidationType", invalidation_type_)); + auto it = stringToInvalidationTypeMap.find(invalidation_type_); + if (it != stringToInvalidationTypeMap.end()) { + invalidation_type = it->second; + } else { + throw std::invalid_argument("Invalid string for InvalidationType: " + invalidation_type_); + } + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + try { + ar(cereal::make_nvp("listeningId", listening_id_)); + listening_id = listening_id_; + } catch (const cereal::Exception &e) { + dbgError(D_INTELLIGENCE) << e.what(); + } + + classifiers[ClassifierType::CLASS] = class_; + classifiers[ClassifierType::CATEGORY] = category; + classifiers[ClassifierType::FAMILY] = family; + classifiers[ClassifierType::GROUP] = group; + classifiers[ClassifierType::ORDER] = order; + classifiers[ClassifierType::KIND] = kind; +} + Invalidation & Invalidation::addAttr(const IpAttributes &attr) { @@ -257,7 +381,7 @@ Invalidation::addMainAttr(const StrAttributes &attr) return *this; } -Maybe +Maybe Invalidation::getRegistrationID() const{ return registration_id; } diff --git a/core/intelligence_is_v2/query_request_v2.cc b/core/intelligence_is_v2/query_request_v2.cc index cc722cc..bec0f11 100644 --- a/core/intelligence_is_v2/query_request_v2.cc +++ b/core/intelligence_is_v2/query_request_v2.cc @@ -30,11 +30,13 @@ QueryRequest::QueryRequest( const string &key, const string &value, bool full_reponse, - AttributeKeyType attribute_type + AttributeKeyType attribute_type, + bool _external_sources_error_status ) { query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value); assets_limit = default_assets_limit; full_response = full_reponse; + external_sources_error_status = _external_sources_error_status; } QueryRequest::QueryRequest( @@ -42,11 +44,13 @@ QueryRequest::QueryRequest( const string &key, const int64_t &value, bool full_reponse, - AttributeKeyType attribute_type + AttributeKeyType attribute_type, + bool _external_sources_error_status ) { query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value); assets_limit = default_assets_limit; full_response = full_reponse; + external_sources_error_status = _external_sources_error_status; } QueryRequest::QueryRequest( @@ -54,11 +58,13 @@ QueryRequest::QueryRequest( const string &key, const vector &value, bool full_reponse, - AttributeKeyType attribute_type + AttributeKeyType attribute_type, + bool _external_sources_error_status ) { query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value); assets_limit = default_assets_limit; full_response = full_reponse; + external_sources_error_status = _external_sources_error_status; } Maybe @@ -83,10 +89,15 @@ QueryRequest::save(cereal::JSONOutputArchive &ar) const { ar( cereal::make_nvp("limit", assets_limit), - cereal::make_nvp("fullResponse", full_response), - cereal::make_nvp("query", query) + cereal::make_nvp("fullResponse", full_response) ); + if (external_sources_error_status) { + ar(cereal::make_nvp("externalSourcesErrorStatus", external_sources_error_status)); + } + + ar(cereal::make_nvp("query", query)); + auto objTypeString = convertObjectTypeToString(); if (objTypeString.ok()) { ar(cereal::make_nvp("objectType", *objTypeString)); @@ -236,6 +247,7 @@ QueryRequest::calcQueryRequestOperator(const QueryRequest &other_query, const Op res_req_query.query = res_query_filter; res_req_query.assets_limit = this->assets_limit; res_req_query.full_response = this->full_response; + res_req_query.external_sources_error_status = this->external_sources_error_status; res_req_query.cursor = this->cursor; res_req_query.requested_attributes = this->requested_attributes; res_req_query.query_types = this->query_types; diff --git a/core/logging/fog_stream.cc b/core/logging/fog_stream.cc index 1a47e32..1a2a97c 100644 --- a/core/logging/fog_stream.cc +++ b/core/logging/fog_stream.cc @@ -37,6 +37,7 @@ FogStream::sendLog(const Report &log) ScopedContext ctx; ctx.registerValue("Obfuscate log field", true); + dbgTrace(D_REPORT) << "Sending log to fog"; LogRest rest(log); i_msg->sendAsyncMessage(HTTPMethod::POST, fog_log_uri, rest, MessageCategory::LOG); } diff --git a/core/logging/log_generator.cc b/core/logging/log_generator.cc index 1c80091..871ecbd 100644 --- a/core/logging/log_generator.cc +++ b/core/logging/log_generator.cc @@ -14,15 +14,23 @@ #include "log_generator.h" using namespace std; +USE_DEBUG_FLAG(D_LOGGING); extern const string unnamed_service; LogGen::~LogGen() { try { - if (send_log) Singleton::Consume::by()->sendLog(log); + if (send_log) { + dbgTrace(D_LOGGING) << "sending log"; + Singleton::Consume::by()->sendLog(log); + } else { + dbgTrace(D_LOGGING) << "not sending log"; + } } catch (...) { + dbgWarning(D_LOGGING) << "Failed to send log"; } + } LogGen & diff --git a/core/logging/log_streams.h b/core/logging/log_streams.h index 560309d..c2ceb34 100644 --- a/core/logging/log_streams.h +++ b/core/logging/log_streams.h @@ -157,7 +157,7 @@ protected: private: void init(); - void sendLog(const std::vector &data); + void sendLogData(const std::vector &data); I_MainLoop::RoutineID log_send_routine = -1; }; diff --git a/core/logging/logging.cc b/core/logging/logging.cc index 338150b..dce3967 100644 --- a/core/logging/logging.cc +++ b/core/logging/logging.cc @@ -162,11 +162,17 @@ public: sendLog(const Report &log) override { if (getConf("agent.config.log.useBulkMode", "Enable bulk of logs", true)) { + dbgTrace(D_REPORT) << "Adding log to bulk"; reports.setBulkSize(getConfigurationWithDefault(100, "Logging", "Sent log bulk size")); reports.push(log); if (reports.sizeQueue() >= 4) { auto persistence_only = getConf("agent.config.log.skip.enable", "Enable Log skipping", true); - sendBufferedLogsImpl(false, persistence_only); + dbgTrace(D_REPORT) + << "Sending buffered logs from queue size: " + << reports.sizeQueue() + << ", persistence_only: " + << persistence_only; + sendBufferedLogsImpl(false, persistence_only); } } else { LogEventLogsSent(true).notify(); @@ -230,10 +236,22 @@ private: for (auto &iter : local_streams) { LogBulkRest sub_batch; for (const auto &log : batch) { - if (log.isStreamActive(iter.first)) sub_batch.push(log); + if (log.isStreamActive(iter.first)) { + dbgTrace(D_REPORT) + << "stream: " + << (int) iter.first + << " is active. adding log to sub-batch"; + sub_batch.push(log); + } } if (sub_batch.size()) { + dbgTrace(D_REPORT) + << "Sending log to stream: " + << (int) iter.first + << ", batch size: " + << sub_batch.size(); + iter.second->sendLog(sub_batch, persistence_only); if (is_async) i_mainloop->yield(); } diff --git a/core/logging/syslog_stream.cc b/core/logging/syslog_stream.cc index 1fc675e..edaca1d 100644 --- a/core/logging/syslog_stream.cc +++ b/core/logging/syslog_stream.cc @@ -55,13 +55,13 @@ SyslogStream::sendLog(const Report &log) vector data(syslog_report.begin(), syslog_report.end()); log_send_routine = mainloop->addOneTimeRoutine( I_MainLoop::RoutineType::Offline, - [this, data] () { sendLog(data); }, + [this, data] () { sendLogData(data); }, "Logging Syslog stream messaging" ); } void -SyslogStream::sendLog(const vector &data) +SyslogStream::sendLogData(const vector &data) { dbgTrace(D_REPORT) << "Sending Syslog log." << " Max logs per send: " << max_logs_per_send; sendLogWithQueue(data); diff --git a/core/mainloop/mainloop.cc b/core/mainloop/mainloop.cc index 4302db5..b025cc5 100644 --- a/core/mainloop/mainloop.cc +++ b/core/mainloop/mainloop.cc @@ -98,12 +98,17 @@ public: init() { fini_signal_flag = false; - addOneTimeRoutine( - RoutineType::Offline, - [this](){ reportStartupEvent(); }, - "Nano service startup report", - false - ); + bool report_startup_event = + getConfigurationWithDefault(true, "Mainloop", "Report Startup Event"); + + if (report_startup_event) { + addOneTimeRoutine( + RoutineType::Offline, + [this](){ reportStartupEvent(); }, + "Nano service startup report", + false + ); + } metric_report_interval = chrono::seconds( getConfigurationWithDefault(600, "Mainloop", "metric reporting interval") @@ -319,7 +324,12 @@ MainloopComponent::Impl::run() ) { dbgWarning(D_MAINLOOP) << "Routine execution exceeded run time. Routine name: " - << curr_iter->second.getRoutineName(); + << curr_iter->second.getRoutineName() + << ", time slice: " + << time_slice_to_use + << ", exceeded time: " + << (getTimer()->getMonotonicTime() - stop_time).count() + << " microseconds"; } } @@ -642,5 +652,6 @@ MainloopComponent::preload() registerExpectedConfiguration("Mainloop", "Busy routine time slice"); registerExpectedConfiguration("Mainloop", "metric reporting interval"); registerExpectedConfiguration("Mainloop", "Exceed Warning"); + registerExpectedConfiguration("Mainloop", "Report Startup Event"); registerConfigLoadCb([&] () { pimpl->reloadConfigurationCb(); }); } diff --git a/core/messaging/connection/connection.cc b/core/messaging/connection/connection.cc index 2be34e1..ca4322c 100644 --- a/core/messaging/connection/connection.cc +++ b/core/messaging/connection/connection.cc @@ -21,6 +21,9 @@ #include #include #include +#include +#include +#include #include "config.h" #include "http_request.h" @@ -73,6 +76,7 @@ enum class ConnectionFlags { UNSECURE, ONE_TIME, + ASYNC_ONE_TIME, IGNORE_SSL_VALIDATION, PROXY, @@ -87,6 +91,9 @@ public: auto metadata_flags = metadata.getConnectionFlags(); if (metadata_flags.isSet(MessageConnectionConfig::UNSECURE_CONN)) flags.setFlag(ConnectionFlags::UNSECURE); if (metadata_flags.isSet(MessageConnectionConfig::ONE_TIME_CONN)) flags.setFlag(ConnectionFlags::ONE_TIME); + if (metadata_flags.isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) { + flags.setFlag(ConnectionFlags::ASYNC_ONE_TIME); + } if (metadata_flags.isSet(MessageConnectionConfig::IGNORE_SSL_VALIDATION)) { flags.setFlag(ConnectionFlags::IGNORE_SSL_VALIDATION); } @@ -99,7 +106,6 @@ public: sni_hostname = metadata.getSniHostName(); dn_host_name = metadata.getDnHostName(); - } void @@ -511,12 +517,29 @@ private: BioConnectionStatus bio_connect = tryToBioConnect(full_address); uint attempts_count = 0; auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); + auto maybe_is_orch = Singleton::Consume::by()->get("Is Orchestrator"); + auto is_orch = maybe_is_orch.ok() && *maybe_is_orch; while (i_time->getMonotonicTime() < conn_end_time && bio_connect == BioConnectionStatus::SHOULD_RETRY) { + if (is_orch) { // fixing code for orch case due to stability concerns - should be removed + if (isBioSocketReady()) { + bio_connect = tryToBioConnect(full_address); + } else { + i_mainloop->yield((attempts_count % 10) == 0); + } + continue; + } + if (isBioSocketReady()) { bio_connect = tryToBioConnect(full_address); - } else { - i_mainloop->yield((attempts_count % 10) == 0); } + + dbgTrace(D_CONNECTION) + << "Connection to: " + << full_address + << " should retry. number of made attempts: " + << ++attempts_count; + + i_mainloop->yield(true); } if (bio_connect == BioConnectionStatus::SUCCESS) { @@ -553,7 +576,8 @@ private: int data_sent_len = BIO_write(bio.get(), curr_data_to_send, data_left_to_send); if (data_sent_len >= 0) { - dbgTrace(D_CONNECTION) << "Sent " << data_sent_len << " bytes, out of: " << data_left_to_send << " bytes."; + dbgTrace(D_CONNECTION) << "Sent " << data_sent_len << " bytes, out of: " << data_left_to_send + << " bytes (total remaining: " << data_left_to_send - data_sent_len << " bytes)."; return data_sent_len; } @@ -637,13 +661,60 @@ private: I_TimeGet *i_time = Singleton::Consume::by(); auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); size_t data_left_to_send = request.length(); + atomic cancel_task{false}; - while (data_left_to_send > 0) { - if (i_time->getMonotonicTime() > sending_end_time) return genError(sending_timeout); - auto send_size = sendData(request, data_left_to_send); - if (!send_size.ok()) return send_size.passErr(); - data_left_to_send -= *send_size; - i_mainloop->yield(*send_size == 0); + // Use smaller chunks for one-time connections with yielding between chunks + if (flags.isSet(ConnectionFlags::ASYNC_ONE_TIME)) { + // Launch async task to send full request only + // note do not add debug logs inside the async task + // as it will cause a crash + auto task = async(launch::async, [this, request, &cancel_task]() -> Maybe { + size_t remaining = request.length(); + while (remaining > 0) { + if (cancel_task.load()) { + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, "Async send task was canceled")); + } + auto sent = sendData(request, remaining); + if (!sent.ok()) return genError(sent.getErr()); + remaining -= *sent; + if (*sent == 0) { + // sleep for 25ms to avoid busy waiting + this_thread::sleep_for(chrono::milliseconds(25)); + } + } + return Maybe(); + }); + + // Set a timeout of 1 minute for the task + auto timeout = chrono::minutes(1); + auto start_time = i_time->getMonotonicTime(); + + // poll and yield until send completes or timeout occurs + if (!task.valid()) { + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, + "Async send future is not valid (no_state)")); + } + // Modify the loop to yield after setting cancel_task to true + while (task.wait_for(chrono::milliseconds(0)) != future_status::ready) { + if (i_time->getMonotonicTime() - start_time > timeout) { + cancel_task.store(true); // Signal the task to cancel + i_mainloop->yield(chrono::milliseconds(50)); // Yield for 50ms after canceling + return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, "Async send task timed out")); + } + i_mainloop->yield(chrono::milliseconds(30)); // Yield for 30ms + dbgTrace(D_CONNECTION) << "Waiting for async send to complete..."; + } + dbgDebug(D_CONNECTION) << "Async send completed."; + auto send_res = task.get(); + if (!send_res.ok()) return send_res.passErr(); + } else { + while (data_left_to_send > 0) { + if (i_time->getMonotonicTime() > sending_end_time) return genError(sending_timeout); + auto send_size = sendData(request, data_left_to_send); + if (!send_size.ok()) return send_size.passErr(); + data_left_to_send -= *send_size; + i_mainloop->yield(*send_size == 0); + } } auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout(); @@ -660,11 +731,11 @@ private: return receieved.passErr(); } auto response = http_parser.parseData(*receieved, is_connect); + i_mainloop->yield(receieved.unpack().empty()); if (response.ok()) { dbgTrace(D_MESSAGING) << printOut(response.unpack().toString()); return response.unpack(); } - i_mainloop->yield(receieved.unpack().empty()); } return genError(parsing_error); } @@ -708,7 +779,6 @@ private: bool is_dual_auth = false; Maybe sni_hostname = genError("Uninitialized"); Maybe dn_host_name = genError("Uninitialized"); - }; Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata) diff --git a/core/messaging/connection/connection_comp.cc b/core/messaging/connection/connection_comp.cc index a033d79..f75ce55 100644 --- a/core/messaging/connection/connection_comp.cc +++ b/core/messaging/connection/connection_comp.cc @@ -97,7 +97,9 @@ private: if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate); auto connected = conn.establishConnection(); - persistent_connections.emplace(conn_key, conn); + if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) { + persistent_connections.emplace(conn_key, conn); + } if (!connected.ok()) { string connection_err = "Failed to establish connection. Error: " + connected.getErr(); @@ -140,8 +142,9 @@ private: } dbgTrace(D_CONNECTION) << "Connection over proxy established succssesfuly"; - - persistent_connections.emplace(conn_key, conn); + if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) { + persistent_connections.emplace(conn_key, conn); + } return conn; } diff --git a/core/messaging/connection/connection_ut/connection_comp_ut.cc b/core/messaging/connection/connection_ut/connection_comp_ut.cc index 4fe682b..869160b 100644 --- a/core/messaging/connection/connection_ut/connection_comp_ut.cc +++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc @@ -264,3 +264,51 @@ TEST_F(TestConnectionComp, testEstablishNewProxyConnection) auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); } + +TEST_F(TestConnectionComp, testSendRequestWithOneTimeFogConnection) +{ + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + conn_flags.setFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags); + + auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG); + ASSERT_TRUE(maybe_connection.ok()); + auto conn = maybe_connection.unpack(); + + auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body"); + ASSERT_TRUE(req.ok()); + + EXPECT_CALL(mock_mainloop, yield(A())) + .WillOnce( + InvokeWithoutArgs( + [&]() { + cerr << "accepting socket" << endl; + dummy_socket.acceptSocket(); + dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test"); + } + ) + ).WillRepeatedly(Return()); + + EXPECT_CALL(mock_timer, getMonotonicTime()) + .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 10); })); + + auto maybe_response = i_conn->sendRequest(conn, *req); + if (!maybe_response.ok()) { + cout << "Error: " << maybe_response.getErr().toString() << endl; + } + ASSERT_TRUE(maybe_response.ok()); + EXPECT_EQ((*maybe_response).getBody(), "my-test"); + + string expected_msg = + "POST /test HTTP/1.1\r\n" + "Accept-Encoding: identity\r\n" + "Authorization: Bearer accesstoken\r\n" + "Connection: keep-alive\r\n" + "Content-Length: 9\r\n" + "Content-type: application/json\r\n" + "Host: 127.0.0.1\r\n" + "\r\n" + "test-body"; + EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg); +} diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index 1e72482..fb80d06 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -25,6 +25,7 @@ using namespace std; USE_DEBUG_FLAG(D_MESSAGING); +USE_DEBUG_FLAG(D_TRACE_ID); class FogConnectionChecker : public ServerRest { @@ -95,13 +96,14 @@ isMessageToFog(const MessageMetadata message_metadata) Maybe MessagingComp::getConnection(MessageCategory category, const MessageMetadata &metadata) { - auto persistant_conn = getPersistentConnection(metadata, category); - if (persistant_conn.ok()) { - dbgTrace(D_MESSAGING) << "Found a persistant connection"; - return persistant_conn; + if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) { + auto persistant_conn = getPersistentConnection(metadata, category); + if (persistant_conn.ok()) { + dbgTrace(D_MESSAGING) << "Found a persistant connection"; + return persistant_conn; + } + dbgDebug(D_MESSAGING) << persistant_conn.getErr(); } - dbgDebug(D_MESSAGING) << persistant_conn.getErr(); - auto maybe_conn = i_conn->establishConnection(metadata, category); if (!maybe_conn.ok()) { dbgWarning(D_MESSAGING) << maybe_conn.getErr(); @@ -131,6 +133,7 @@ MessagingComp::sendMessage( bool is_to_fog = isMessageToFog(message_metadata); auto metadata = message_metadata; + if (is_to_fog) { if (method == HTTPMethod::GET && fog_get_requests_cache.doesKeyExists(uri)) { HTTPResponse res = fog_get_requests_cache.getEntry(uri); @@ -141,7 +144,16 @@ MessagingComp::sendMessage( auto i_env = Singleton::Consume::by(); metadata.insertHeader("User-Agent", "Infinity Next (a7030abf93a4c13)"); - metadata.insertHeaders(i_env->getCurrentHeadersMap()); + if (!metadata.getTraceId().ok()) { + metadata.insertHeaders(i_env->getCurrentHeadersMap()); + } + + Maybe trace_id = metadata.getTraceId(); + if (trace_id.ok()) { + dbgTrace(D_TRACE_ID) << "Sending message to fog (trace ID: " << trace_id.unpack() << ")"; + } else { + dbgTrace(D_TRACE_ID) << "Could not retrieve trace ID for fog message. Error: " << trace_id.getErr(); + } } auto req = HTTPRequest::prepareRequest( diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 014a002..48caa01 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -31,6 +31,36 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; } MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; } +static const set default_metrics = { + "watchdogProcessStartupEventsSum", + "reservedNgenA", + "reservedNgenB", + "reservedNgenC" + "reservedNgenD" + "reservedNgenE", + "reservedNgenF", + "reservedNgenG" + "reservedNgenH", + "reservedNgenI", + "reservedNgenJ", + "numberOfProtectedAssetsSample", + "preventEngineMatchesSample", + "detectEngineMatchesSample", + "ignoreEngineMatchesSample", + "cpuMaxSample", + "cpuAvgSample", + "cpuSample", + "serviceVirtualMemorySizeMaxSample", + "serviceVirtualMemorySizeMinSample", + "serviceVirtualMemorySizeAvgSample", + "serviceRssMemorySizeMaxSample", + "serviceRssMemorySizeMinSample", + "serviceRssMemorySizeAvgSample", + "generalTotalMemorySizeMaxSample", + "generalTotalMemorySizeMinSample", + "generalTotalMemorySizeAvgSample" +}; + // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage static ostream & operator<<(ostream &os, const CompressAndEncodeAIOPSMetrics &metrics) { @@ -165,6 +195,9 @@ GenericMetric::init( const string &_asset_id ) { + + if (!getConfigurationWithDefault(true, "metric", "genericMetricInitEnable")) return; + turnOnStream(Stream::FOG); turnOnStream(Stream::DEBUG); @@ -263,7 +296,22 @@ GenericMetric::respond(const AllMetricEvent &event) vector GenericMetric::respond(const MetricScrapeEvent &) { - return getPromMetricsData(); + bool enable_all_metrics = getProfileAgentSettingWithDefault(false, "enable_all_metrics"); + if (enable_all_metrics) { + dbgTrace(D_METRICS) << "Sensitive metrics enabled, returning all metrics"; + return getPromMetricsData(); + } + + vector allowed_calcs; + for (auto &calc : prometheus_calcs) { + auto metric_calc_name = calc->getMetricDotName() != "" ? calc->getMetricDotName() : calc->getMetricName(); + if (default_metrics.find(metric_calc_name) != default_metrics.end()) { + allowed_calcs.push_back(calc); + } + } + if (allowed_calcs.empty()) return {}; + + return getPromMetricsData(&allowed_calcs); } string GenericMetric::getListenerName() const { return metric_name; } @@ -329,7 +377,7 @@ GenericMetric::generateLog() } vector -GenericMetric::getPromMetricsData() +GenericMetric::getPromMetricsData(const vector *allowed_calcs) { vector all_metrics; bool enable_prometheus = false; @@ -345,7 +393,8 @@ GenericMetric::getPromMetricsData() if (!enable_prometheus) return all_metrics; dbgTrace(D_METRICS) << "Get prometheus metrics"; - for (auto &calc : prometheus_calcs) { + const vector &calcs_to_use = allowed_calcs ? *allowed_calcs : prometheus_calcs; + for (auto &calc : calcs_to_use) { const auto &calc_prom_metrics = calc->getPrometheusMetrics(metric_name, asset_id); all_metrics.insert(all_metrics.end(), calc_prom_metrics.begin(), calc_prom_metrics.end()); calc->reset(); @@ -416,5 +465,6 @@ GenericMetric::preload() registerExpectedConfiguration("metric", "debugMetricSendEnable"); registerExpectedConfiguration("metric", "aiopsMetricSendEnable"); registerExpectedConfiguration("metric", "fogMetricUri"); + registerExpectedConfiguration("metric", "genericMetricInitEnable"); registerExpectedConfiguration("metric", "metricsOutputTmpFile"); } diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index e951ee0..f32d73a 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -542,7 +542,8 @@ TEST_F(MetricTest, getPromeathusMetric) metric_scraper.init(); stringstream configuration; - configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n"; + configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},"; + configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n"; EXPECT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(configuration)); @@ -634,7 +635,8 @@ TEST_F(MetricTest, getPromeathusMultiMap) metric_scraper.init(); stringstream configuration; - configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n"; + configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},"; + configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n"; EXPECT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(configuration)); @@ -699,7 +701,8 @@ TEST_F(MetricTest, getPromeathusTwoMetrics) metric_scraper.init(); stringstream configuration; - configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n"; + configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},"; + configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n"; EXPECT_TRUE(Singleton::Consume::from(conf)->loadConfiguration(configuration)); diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 1c711b0..2a69307 100644 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -115,7 +115,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX}, {"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER}, {"SWAG Server", ReportIS::Tags::WEB_SERVER_SWAG}, - {"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED} + {"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED}, + {"AI Guard", ReportIS::Tags::AIGUARD} }; auto report_is_tag = strings_to_tags.find(tag); @@ -326,7 +327,8 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "APISIX Server", "Docker Deployment", "SWAG Server", - "NGINX Unified Server" + "NGINX Unified Server", + "AI Guard" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/core/rest/rest_server.cc b/core/rest/rest_server.cc index e71438d..a92a4fa 100644 --- a/core/rest/rest_server.cc +++ b/core/rest/rest_server.cc @@ -46,8 +46,10 @@ public: void startNewConnection() const; - bool bindRestServerSocket(struct sockaddr_in &addr, vector port_range); - bool bindRestServerSocket(struct sockaddr_in6 &addr, vector port_range); + bool setupIpv4ServerSocket(bool accept_get_from_external_ip); + bool setupIpv6ServerSocket(bool &finish_port_range); + bool createIpv4Socket(); + bool createIpv6Socket(); bool addRestCall(RestAction oper, const string &uri, unique_ptr &&init) override; bool addGetCall(const string &uri, const function &cb) override; bool addWildcardGetCall(const string &uri, const function &callback); @@ -74,40 +76,141 @@ private: }; bool -RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector port_range) +RestServer::Impl::createIpv4Socket() +{ + fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd == -1) { + dbgAssert(fd >= 0) << alert << "Failed to open a socket"; + } + int socket_enable = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) { + dbgWarning(D_API) << "Could not set the socket options"; + } + dbgDebug(D_API) << "IPv4 socket opened successfully"; + return true; +} + +bool +RestServer::Impl::createIpv6Socket() +{ + fd = socket(AF_INET6, SOCK_STREAM, 0); + if (fd == -1) { + return false; + } + int socket_enable = 1; + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) { + dbgWarning(D_API) << "Could not set the socket options"; + } + dbgDebug(D_API) << "IPv6 socket opened successfully"; + int option = 0; + + if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) { + dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option"; + } + return true; +} + +bool +RestServer::Impl::setupIpv4ServerSocket(bool accept_get_from_external_ip) { dbgFlow(D_API) << "Binding IPv4 socket"; + struct sockaddr_in addr; + bzero(&addr, sizeof(addr)); + addr.sin_family = AF_INET; + if (accept_get_from_external_ip) { + addr.sin_addr.s_addr = htonl(INADDR_ANY); + dbgDebug(D_API) << "Socket listening on any address"; + } else { + addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + dbgDebug(D_API) << "Socket listening on local address"; + } + + bool create_socket = true; for (uint16_t port : port_range) { + if(create_socket) { + if (!createIpv4Socket()) { + dbgDebug(D_API) << "Failed creating Ipv4 socket!"; + return false; + } + create_socket = false; + } + addr.sin_port = htons(port); - if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true; + if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) != 0) { + if (errno == EADDRINUSE) { + dbgDebug(D_API) << "Port " << port << " is already in use"; + } else { + dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + } + continue; + } + + if (listen(fd, listen_limit) == 0) { + listening_port = ntohs(addr.sin_port); + return true; + } if (errno == EADDRINUSE) { - dbgDebug(D_API) << "Port " << port << " is already in use"; + dbgDebug(D_API) << "Another socket is already listening on the port: " << port; } else { - dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + dbgDebug(D_API) << "Failed to listen to socket with error: " << strerror(errno); } + + create_socket = true; + close(fd); + fd = -1; } return false; } bool -RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector port_range) +RestServer::Impl::setupIpv6ServerSocket(bool &finish_port_range) { dbgFlow(D_API) << "Binding IPv6 socket"; + struct sockaddr_in6 addr6; + bzero(&addr6, sizeof(addr6)); + addr6.sin6_family = AF_INET6; + addr6.sin6_addr = in6addr_any; + dbgDebug(D_API) << "Socket listening on any address"; + bool create_socket = true; for (uint16_t port : port_range) { - addr.sin6_port = htons(port); + if(create_socket) { + if (!createIpv6Socket()) { + dbgDebug(D_API) << "Failed creating Ipv6 socket!"; + return false; + } + create_socket = false; + } - if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true; + addr6.sin6_port = htons(port); + + if (bind(fd, (struct sockaddr *)&addr6, sizeof(struct sockaddr_in6)) != 0) { + if (errno == EADDRINUSE) { + dbgDebug(D_API) << "Port " << port << " is already in use"; + } else { + dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + } + continue; + } + + if (listen(fd, listen_limit) == 0) { + listening_port = ntohs(addr6.sin6_port); + return true; + } if (errno == EADDRINUSE) { - dbgDebug(D_API) << "Port " << port << " is already in use"; + dbgDebug(D_API) << "Another socket is already listening on the port: " << port; } else { - dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno); + dbgDebug(D_API) << "Failed to listen to socket with error: " << strerror(errno); } + create_socket = true; + close(fd); + fd = -1; } + finish_port_range = true; return false; } @@ -163,60 +266,28 @@ RestServer::Impl::init() } } - bool is_ipv6 = false; + bool finish_port_range = false; + bool failed_to_listen = false; if (accept_get_from_external_ip) { - is_ipv6 = true; - fd = socket(AF_INET6, SOCK_STREAM, 0); + while (!setupIpv6ServerSocket(finish_port_range) && finish_port_range) { + dbgWarning(D_API) << "Failed to bind to any of the (IPv6) ports in the port range"; + failed_to_listen = true; + finish_port_range = false; + mainloop->yield(bind_retry_interval_msec); + } } if (fd == -1) { - fd = socket(AF_INET, SOCK_STREAM, 0); - is_ipv6 = false; + while (!setupIpv4ServerSocket(accept_get_from_external_ip)) { + dbgWarning(D_API) << "Failed to bind to any of the (IPv4) ports in the port range"; + failed_to_listen = true; + mainloop->yield(bind_retry_interval_msec); + } + } + if (failed_to_listen) { + dbgWarning(D_API) << "Manage to listen on port:" << listening_port << " after failure"; } dbgAssert(fd >= 0) << alert << "Failed to open a socket"; - int socket_enable = 1; - if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) { - dbgWarning(D_API) << "Could not set the socket options"; - } - - if (is_ipv6) { - dbgDebug(D_API) << "IPv6 socket opened successfully"; - int option = 0; - if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) { - dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option"; - } - - struct sockaddr_in6 addr6; - bzero(&addr6, sizeof(addr6)); - addr6.sin6_family = AF_INET6; - addr6.sin6_addr = in6addr_any; - dbgDebug(D_API) << "Socket listening on any address"; - - while (!bindRestServerSocket(addr6, port_range)) { - mainloop->yield(bind_retry_interval_msec); - } - listening_port = ntohs(addr6.sin6_port); - } else { - dbgDebug(D_API) << "IPv4 socket opened successfully"; - struct sockaddr_in addr; - bzero(&addr, sizeof(addr)); - addr.sin_family = AF_INET; - if (accept_get_from_external_ip) { - addr.sin_addr.s_addr = htonl(INADDR_ANY); - dbgDebug(D_API) << "Socket listening on any address"; - } else { - addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - dbgDebug(D_API) << "Socket listening on local address"; - } - - while (!bindRestServerSocket(addr, port_range)) { - mainloop->yield(bind_retry_interval_msec); - } - listening_port = ntohs(addr.sin_port); - } - - listen(fd, listen_limit); - auto is_primary = Singleton::Consume::by()->get("Is Rest primary routine"); id = mainloop->addFileRoutine( I_MainLoop::RoutineType::Offline, diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index ee811ea..4869e78 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -26,6 +26,7 @@ var_default_gem_fog_address="inext-agents.cloud.ngen.checkpoint.com" var_default_us_fog_address="inext-agents-us.cloud.ngen.checkpoint.com" var_default_au_fog_address="inext-agents-aus1.cloud.ngen.checkpoint.com" var_default_in_fog_address="inext-agents-ind1.cloud.ngen.checkpoint.com" +var_default_ae_fog_address="inext-agents-ae.cloud.ngen.checkpoint.com" #NOTE: open-appsec-ctl only supports nano services with name of the format cp-nano- cp_nano_service_name_prefix="cp-nano" @@ -1560,9 +1561,13 @@ set_mode() au_prefix_uppercase="CP-AU-" in_prefix="cp-in-" in_prefix_uppercase="CP-IN-" + ae_prefix="cp-ae-" + ae_prefix_uppercase="CP-AE-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" + elif [ "${var_token#"$ae_prefix"}" != "${var_token}" ] || [ "${var_token#"$ae_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_ae_fog_address" elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_au_fog_address" elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index d81eaf5..8071244 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -53,6 +53,7 @@ var_default_gem_fog_address="https://inext-agents.cloud.ngen.checkpoint.com" var_default_us_fog_address="https://inext-agents-us.cloud.ngen.checkpoint.com" var_default_au_fog_address="https://inext-agents-aus1.cloud.ngen.checkpoint.com" var_default_in_fog_address="https://inext-agents-ind1.cloud.ngen.checkpoint.com" +var_default_ae_fog_address="https://inext-agents-ae.cloud.ngen.checkpoint.com" var_fog_address= var_certs_dir= var_public_key= @@ -396,9 +397,13 @@ if [ "$RUN_MODE" = "install" ] && [ $var_offline_mode = false ]; then au_prefix_uppercase="CP-AU-" in_prefix="cp-in-" in_prefix_uppercase="CP-IN-" + ae_prefix="cp-ae-" + ae_prefix_uppercase="CP-AE-" if [ "${var_token#"$us_prefix"}" != "${var_token}" ] || [ "${var_token#"$us_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_us_fog_address" + elif [ "${var_token#"$ae_prefix"}" != "${var_token}" ] || [ "${var_token#"$ae_prefix_uppercase"}" != "${var_token}" ]; then + var_fog_address="$var_default_ae_fog_address" elif [ "${var_token#$au_prefix}" != "${var_token}" ] || [ "${var_token#"$au_prefix_uppercase"}" != "${var_token}" ]; then var_fog_address="$var_default_au_fog_address" elif [ "${var_token#$in_prefix}" != "${var_token}" ] || [ "${var_token#"$in_prefix_uppercase"}" != "${var_token}" ]; then @@ -500,26 +505,26 @@ cp_copy() # Initials - cc cp_print "Destination md5, after the copy:\n$DEST_AFTER_COPY" } -update_openappsec_manifest() +update_cloudguard_appsec_manifest() { - if [ -z ${OPENAPPSEC_NANO_AGENT} ] && { [ -z ${CLOUDGUARD_APPSEC_STANDALONE} ] || [ -z ${DOCKER_RPM_ENABLED} ]; }; then + if [ -z ${INFINITY_NEXT_NANO_AGENT} ] && { [ -z ${CLOUDGUARD_APPSEC_STANDALONE} ] || [ -z ${DOCKER_RPM_ENABLED} ]; }; then return fi - selected_openappsec_manifest_path="${TMP_FOLDER}/openappsec_manifest.json" - if [ "${DOCKER_RPM_ENABLED}" = "false" ] || [ "${OPENAPPSEC_NANO_AGENT}" = "TRUE" ]; then - selected_openappsec_manifest_path="${TMP_FOLDER}/self_managed_openappsec_manifest.json" + selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/cloudguard_appsec_manifest.json" + if [ "${DOCKER_RPM_ENABLED}" = "false" ] || [ "${INFINITY_NEXT_NANO_AGENT}" = "TRUE" ]; then + selected_cloudguard_appsec_manifest_path="${TMP_FOLDER}/self_managed_cloudguard_appsec_manifest.json" fi - if [ ! -f "$selected_openappsec_manifest_path" ]; then + if [ ! -f "$selected_cloudguard_appsec_manifest_path" ]; then return fi - openappsec_manifest_path="${selected_openappsec_manifest_path}.used" - mv "$selected_openappsec_manifest_path" "$openappsec_manifest_path" + cloudguard_appsec_manifest_path="${selected_cloudguard_appsec_manifest_path}.used" + mv "$selected_cloudguard_appsec_manifest_path" "$cloudguard_appsec_manifest_path" fog_host=$(echo "$var_fog_address" | sed 's/https\?:\/\///') fog_host=${fog_host%/} - sed "s/namespace/${fog_host}/g" ${openappsec_manifest_path} > "${FILESYSTEM_PATH}/${CONF_PATH}/manifest.json" + sed "s/namespace/${fog_host}/g" ${cloudguard_appsec_manifest_path} > "${FILESYSTEM_PATH}/${CONF_PATH}/manifest.json" } set_cloud_storage() @@ -644,13 +649,14 @@ install_watchdog() echo "ExecStart=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} echo "ExecStartPost=${FILESYSTEM_PATH}/${WATCHDOG_PATH}/wait-for-networking-inspection-modules.sh" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} else - echo "ExecStart=ip netns exec CTX0000${VS_ID} ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + CTX_VAL=$(printf "CTX%05d" "$VS_ID") + echo "ExecStart=ip netns exec ${CTX_VAL} ${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} fi echo "Environment=\"FILESYSTEM_PATH=${FILESYSTEM_PATH}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} if [ -n "${PROMETHEUS}" ] ; then echo "Environment=\"PROMETHEUS=${PROMETHEUS}\"" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} fi - + check_and_run_restorecon /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} cp_exec "systemctl daemon-reload" cp_exec "systemctl enable nano_agent" else @@ -709,6 +715,8 @@ install_cp_nano_ctl() OPEN_APPSEC_CTL="open-appsec-ctl" CP_NANO_YQ_LOCATION="./scripts/yq" CP_NANO_YQ="yq" + CP_NANO_STRINGS_LOCATION="./scripts/strings" + CP_NANO_STRINGS="strings" if [ -f $USR_SBIN_PATH/${CP_NANO_CTL_DEPRECATED} ]; then cp_exec "rm -rf $USR_SBIN_PATH/${CP_NANO_CTL_DEPRECATED}" @@ -742,6 +750,9 @@ install_cp_nano_ctl() cp_exec "cp -f ${CP_NANO_YQ_LOCATION} ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_YQ}" ${FORCE_STDOUT} cp_exec "chmod 700 ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_YQ}" + + cp_exec "cp -f ${CP_NANO_STRINGS_LOCATION} ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_STRINGS}" ${FORCE_STDOUT} + cp_exec "chmod 700 ${FILESYSTEM_PATH}/${BIN_PATH}/${CP_NANO_STRINGS}" } set_conf_temp_location() @@ -789,8 +800,8 @@ upgrade_conf_if_needed() [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && . "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && \ - previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') - if ! [ -z "$previous_mode" ]; then + previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') + if ! [ -z "$previous_mode" ]; then var_orchestration_mode=${previous_mode} fi @@ -815,6 +826,7 @@ upgrade_conf_if_needed() cp_print "Creating env details file" ${FORCE_STDOUT} if [ $var_container_mode = true ]; then echo 'IS_CONTAINER_ENV=true' >> "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}" + echo "IMAGE_VERSION=$IMAGE_VERSION" >> "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}" fi fi if cat ${FILESYSTEM_PATH}/${ORCHESTRATION_CONF_FILE} | grep -q '"/agents/log'; then @@ -873,6 +885,17 @@ copy_and_run_cloud_metadata_script() cp_exec "${FILESYSTEM_PATH}/${SCRIPTS_PATH}/${GET_CLOUD_METADATA_PATH}" } +check_and_run_restorecon() { + if ! command -v restorecon >/dev/null 2>&1; then + return + fi + + if [ -f /sys/fs/selinux/enforce ]; then + cp_print "SELinux is enabled. Running restorecon on: $1" + restorecon "$1" + fi +} + install_public_key() { return @@ -1025,8 +1048,8 @@ install_orchestration() [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && . "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" - [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && \ - previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') + [ -f "${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg" ] && \ + previous_mode=$(cat ${FILESYSTEM_PATH}/${SERVICE_PATH}/${ORCHESTRATION_FILE_NAME}.cfg | grep "orchestration-mode" | cut -d = -f 3 | sed 's/"//') if ! [ -z "$previous_mode" ]; then var_orchestration_mode=${previous_mode} @@ -1050,8 +1073,8 @@ install_orchestration() rm -f "${FILESYSTEM_PATH}/${CONF_PATH}/default_orchestration_flags" fi - update_openappsec_manifest - upgrade_conf_if_needed + update_cloudguard_appsec_manifest + upgrade_conf_if_needed cp_exec "${FILESYSTEM_PATH}/${WATCHDOG_PATH}/cp-nano-watchdog --un-register ${FILESYSTEM_PATH}/${SERVICE_PATH}/cp-nano-orchestration $var_arch_flag" if [ "$IS_K8S_ENV" = "true" ]; then @@ -1086,6 +1109,7 @@ install_orchestration() sed -i "$ d" /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} echo "EnvironmentFile=/etc/environment" >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} echo >> /etc/systemd/system/${NANO_AGENT_SERVICE_FILE} + check_and_run_restorecon "/etc/systemd/system/${NANO_AGENT_SERVICE_FILE}" cp_exec "systemctl daemon-reload" cp_exec "systemctl restart nano_agent" fi @@ -1107,7 +1131,7 @@ install_orchestration() cp_exec "mkdir -p ${LOG_FILE_PATH}/${LOG_PATH}" cp_exec "mkdir -p ${FILESYSTEM_PATH}/${DATA_PATH}" - update_openappsec_manifest + update_cloudguard_appsec_manifest if [ ! -f ${FILESYSTEM_PATH}/${DEFAULT_SETTINGS_PATH} ]; then echo "{\"agentSettings\": []}" > ${FILESYSTEM_PATH}/${DEFAULT_SETTINGS_PATH} @@ -1117,6 +1141,7 @@ install_orchestration() cp_print "Creating env details file" ${FORCE_STDOUT} if [ $var_container_mode = true ]; then echo 'IS_CONTAINER_ENV=true' >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} + echo "IMAGE_VERSION=$IMAGE_VERSION" >> "${FILESYSTEM_PATH}/${ENV_DETAILS_FILE}" fi echo "MAX_FILE_SIZE=${WATCHDOG_MAX_FILE_SIZE}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} echo "MAX_ROTATION=${WATCHDOG_MAX_ROTATIONS}" >> ${FILESYSTEM_PATH}/${ENV_DETAILS_FILE} diff --git a/nodes/prometheus/package/cp-nano-prometheus-conf.json b/nodes/prometheus/package/cp-nano-prometheus-conf.json index fb5184b..59e0c96 100755 --- a/nodes/prometheus/package/cp-nano-prometheus-conf.json +++ b/nodes/prometheus/package/cp-nano-prometheus-conf.json @@ -1,21 +1,21 @@ -{ - "connection": { - "Nano service API Port Primary": [ - { - "value": 7465 - } - ], - "Nano service API Port Alternative": [ - { - "value": 7466 - } - ] - }, - "message": { - "Connection timeout": [ - { - "value": 10000000 - } - ] - } -} +{ + "connection": { + "Nano service API Port Primary": [ + { + "value": 7465 + } + ], + "Nano service API Port Alternative": [ + { + "value": 7466 + } + ] + }, + "message": { + "Connection timeout": [ + { + "value": 10000000 + } + ] + } +} diff --git a/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json b/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json index df2394b..8bbbae6 100755 --- a/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json +++ b/nodes/prometheus/package/cp-nano-prometheus-debug-conf.json @@ -1,11 +1,11 @@ -{ - "Debug": [ - { - "Streams": [ - { - "Output": "nano_agent/cp-nano-prometheus.dbg" - } - ] - } - ] -} +{ + "Debug": [ + { + "Streams": [ + { + "Output": "nano_agent/cp-nano-prometheus.dbg" + } + ] + } + ] +}