diff --git a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h index cb51568..80691ce 100755 --- a/attachments/kernel_modules/core/include/common_is/kdebug_flags.h +++ b/attachments/kernel_modules/core/include/common_is/kdebug_flags.h @@ -32,5 +32,6 @@ DEFINE_KDEBUG_FLAG(statelessValidation) DEFINE_KDEBUG_FLAG(kernelMetric) DEFINE_KDEBUG_FLAG(tproxy) DEFINE_KDEBUG_FLAG(tenantStats) +DEFINE_KDEBUG_FLAG(uuidTranslation) #endif // DEFINE_KDEBUG_FLAG diff --git a/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc b/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc index 18a7cc6..fd2cf93 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc @@ -49,6 +49,8 @@ nginxIntakerEvent::resetAllCounters() req_proccessing_timeout = 0; res_proccessing_timeout = 0; req_failed_to_reach_upstream = 0; + req_overall_size = 0; + res_overall_size = 0; cpu_event.setCPU(0); } @@ -249,10 +251,22 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie cpu_event.setCPU(amount); break; } + case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: { + req_overall_size += amount; + static const uint64_t max_expected_res_size = 100ULL * 1024 * 1024 * 1024; + if (amount > max_expected_res_size) { + dbgWarning(D_METRICS_NGINX_ATTACHMENT) << "Requests sizes higher than expected: " << amount; + } + break; + } + case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: { + res_overall_size += amount; + break; + } default: dbgWarning(D_METRICS_NGINX_ATTACHMENT) << "Unsupported metric type. Type: " << static_cast(metric_type); - return; + break; } } } @@ -353,6 +367,10 @@ nginxIntakerEvent::getPluginMetricCounter(ngx_http_plugin_metric_type_e metric_t return req_failed_to_reach_upstream; case ngx_http_plugin_metric_type_e::CPU_USAGE: return static_cast(cpu_event.getCPU()); + case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: + return req_overall_size; + case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: + return res_overall_size; default: dbgWarning(D_METRICS_NGINX_ATTACHMENT) << "Unsupported metric type. Type: " << static_cast(metric_type); @@ -498,5 +516,11 @@ nginxIntakerMetric::upon(const nginxIntakerEvent &event) req_failed_to_reach_upstream.report( event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM) ); + req_overall_size.report( + event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT) + ); + res_overall_size.report( + event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT) + ); event.notifyCPU(); } diff --git a/components/include/generic_rulebase/parameters_config.h b/components/include/generic_rulebase/parameters_config.h index 665f8ec..4f4e52a 100755 --- a/components/include/generic_rulebase/parameters_config.h +++ b/components/include/generic_rulebase/parameters_config.h @@ -224,5 +224,6 @@ private: }; static const ParameterBehavior action_ignore(BehaviorKey::ACTION, BehaviorValue::IGNORE); +static const ParameterBehavior action_accept(BehaviorKey::ACTION, BehaviorValue::ACCEPT); #endif //__PARAMETERS_CONFIG_H__ diff --git a/components/include/i_orchestration_status.h b/components/include/i_orchestration_status.h index d99bb0c..4305c55 100755 --- a/components/include/i_orchestration_status.h +++ b/components/include/i_orchestration_status.h @@ -34,7 +34,6 @@ public: virtual const std::string & getUpdateTime() const = 0; virtual const std::string & getLastManifestUpdate() const = 0; virtual const std::string & getPolicyVersion() const = 0; - virtual const std::string & getWaapModelVersion() const = 0; virtual const std::string & getLastPolicyUpdate() const = 0; virtual const std::string & getLastSettingsUpdate() const = 0; virtual const std::string & getUpgradeMode() const = 0; diff --git a/components/include/i_service_controller.h b/components/include/i_service_controller.h index d1f9872..d3d4f42 100755 --- a/components/include/i_service_controller.h +++ b/components/include/i_service_controller.h @@ -31,7 +31,7 @@ public: virtual const std::string & getPolicyVersions() const = 0; virtual const std::string & getPolicyVersion() const = 0; virtual const std::string & getUpdatePolicyVersion() const = 0; - virtual void updateReconfStatus(int id, ReconfStatus status) = 0; + virtual void updateReconfStatus(int id, const std::string &service_name, ReconfStatus status) = 0; virtual void startReconfStatus( int id, ReconfStatus status, diff --git a/components/include/local_policy_mgmt_gen.h b/components/include/local_policy_mgmt_gen.h index 5c13929..fcb64a6 100644 --- a/components/include/local_policy_mgmt_gen.h +++ b/components/include/local_policy_mgmt_gen.h @@ -19,13 +19,19 @@ #include "i_mainloop.h" #include "i_local_policy_mgmt_gen.h" #include "i_env_details.h" +#include "i_shell_cmd.h" +#include "i_orchestration_tools.h" class LocalPolicyMgmtGenerator : public Component, Singleton::Provide, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume, + Singleton::Consume { public: LocalPolicyMgmtGenerator(); diff --git a/components/include/nginx_intaker_metric.h b/components/include/nginx_intaker_metric.h index b25e03b..decb4da 100755 --- a/components/include/nginx_intaker_metric.h +++ b/components/include/nginx_intaker_metric.h @@ -81,6 +81,8 @@ private: uint64_t req_proccessing_timeout = 0; uint64_t res_proccessing_timeout = 0; uint64_t req_failed_to_reach_upstream = 0; + uint64_t req_overall_size = 0; + uint64_t res_overall_size = 0; CPUEvent cpu_event; }; @@ -140,6 +142,8 @@ private: Counter thread_failure{this, "attachmentThreadFailureSum"}; Counter req_proccessing_timeout{this, "httpRequestProcessingReachedTimeoutSum"}; Counter res_proccessing_timeout{this, "httpResponseProcessingReachedTimeoutSum"}; + Counter req_overall_size{this, "httpRequestsSizeSum"}; + Counter res_overall_size{this, "httpResponsesSizeSum"}; Counter req_failed_to_reach_upstream{this, "httpRequestFailedToReachWebServerUpstreamSum"}; }; diff --git a/components/include/orchestration_status.h b/components/include/orchestration_status.h index b9adb63..84879a4 100755 --- a/components/include/orchestration_status.h +++ b/components/include/orchestration_status.h @@ -24,7 +24,6 @@ #include "i_time_get.h" #include "i_mainloop.h" #include "i_agent_details.h" -#include "i_details_resolver.h" #include "customized_cereal_map.h" class OrchestrationStatus @@ -33,7 +32,6 @@ class OrchestrationStatus Singleton::Provide, Singleton::Consume, Singleton::Consume, - Singleton::Consume, Singleton::Consume, Singleton::Consume { diff --git a/components/include/rate_limit.h b/components/include/rate_limit.h index 3c3e726..c4b7909 100755 --- a/components/include/rate_limit.h +++ b/components/include/rate_limit.h @@ -7,13 +7,15 @@ #include "singleton.h" #include "i_mainloop.h" #include "i_environment.h" +#include "i_generic_rulebase.h" class RateLimit : public Component, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: RateLimit(); diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control.cc b/components/security_apps/layer_7_access_control/layer_7_access_control.cc index 299e8da..780f263 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control.cc @@ -1,16 +1,14 @@ #include "layer_7_access_control.h" #include -#include #include +#include #include "config.h" #include "cache.h" #include "http_inspection_events.h" -#include "http_transaction_common.h" #include "nginx_attachment_common.h" #include "intelligence_comp_v2.h" -#include "intelligence_is_v2/intelligence_query_v2.h" #include "intelligence_is_v2/query_request_v2.h" #include "log_generator.h" @@ -103,7 +101,7 @@ private: unsigned int crowdsec_event_id; }; -class Layer7AccessControl::Impl : public Listener +class Layer7AccessControl::Impl : public Listener, Listener { public: void init(); @@ -126,27 +124,25 @@ public: return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; } - auto source_identifier = i_env->get(HttpTransactionData::source_identifier); - if (source_identifier.ok() && IPAddr::createIPAddr(source_identifier.unpack()).ok()) { - dbgTrace(D_L7_ACCESS_CONTROL) << "Found a valid source identifier value: " << source_identifier.unpack(); - return checkReputation(source_identifier.unpack()); - } + return handleEvent(); + } - auto orig_source_ip = i_env->get(HttpTransactionData::client_ip_ctx); - if (!orig_source_ip.ok()) { - dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context"; - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - } + EventVerdict + respond(const WaitTransactionEvent &) override + { + dbgFlow(D_L7_ACCESS_CONTROL) << "Handling wait verdict"; - stringstream ss_client_ip; - ss_client_ip << orig_source_ip.unpack(); - return checkReputation(ss_client_ip.str()); + return handleEvent(); } private: + void queryIntelligence(); + void scheduleIntelligenceQuery(const string &ip); + void processIntelligenceResponse(const string &ip, const vector> &response); Maybe getIpReputation(const string &ip); - ngx_http_cp_verdict_e checkReputation(const string &source_ip); - void generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const; + EventVerdict generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const; + EventVerdict queryIpReputation(const string &source_ip); + EventVerdict handleEvent(); bool isAppEnabled() const; bool isPrevent() const; @@ -154,9 +150,12 @@ private: Maybe genLogField(const string &log_key, const string &env_key) const; Maybe genLogIPField(const string &log_key, const string &env_key) const; + bool is_intelligence_routine_running = false; I_Environment *i_env = nullptr; I_Intelligence_IS_V2 *i_intelligence = nullptr; + I_MainLoop *i_mainloop = nullptr; TemporaryCache ip_reputation_cache; + unordered_set pending_ips; }; bool @@ -177,79 +176,139 @@ Layer7AccessControl::Impl::isPrevent() const return mode == "prevent"; } +void +Layer7AccessControl::Impl::scheduleIntelligenceQuery(const string &ip) +{ + dbgFlow(D_L7_ACCESS_CONTROL) << "Scheduling intelligence query about reputation of IP: " << ip; + + pending_ips.emplace(ip); + + if (!is_intelligence_routine_running) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Starting intelligence routine"; + is_intelligence_routine_running = true; + i_mainloop->addOneTimeRoutine( + I_MainLoop::RoutineType::System, + [&] () { queryIntelligence(); }, + "Check IP reputation" + ); + } +} + Maybe Layer7AccessControl::Impl::getIpReputation(const string &ip) { dbgFlow(D_L7_ACCESS_CONTROL) << "Getting reputation of IP " << ip; - if (ip_reputation_cache.doesKeyExists(ip)) return ip_reputation_cache.getEntry(ip); - dbgTrace(D_L7_ACCESS_CONTROL) << "Not found in cache - about to query intelligence"; + dbgTrace(D_L7_ACCESS_CONTROL) << ip << " reputation was not found in cache"; - QueryRequest request = QueryRequest( - Condition::EQUALS, - "ipv4Addresses", - ip, - true, - AttributeKeyType::REGULAR - ); - - auto response = i_intelligence->queryIntelligence(request); - - if (!response.ok()) { - dbgWarning(D_L7_ACCESS_CONTROL) << "Failed to query intelligence about reputation of IP: " << ip; - return genError("Failed to query intelligence"); - } - - auto &unpacked_response = response.unpack(); - if (unpacked_response.empty()) { - dbgTrace(D_L7_ACCESS_CONTROL) << "Intelligence reputation response collection is empty. IP is clean."; - return IntelligenceIpReputation(); - } - - for (const auto &intelligence_reply : unpacked_response) { - if (intelligence_reply.getAssetType() == crowdsec_asset_type && !intelligence_reply.getData().empty()){ - dbgTrace(D_L7_ACCESS_CONTROL) << intelligence_reply.getData().front(); - return intelligence_reply.getData().front(); - } - } - - - return IntelligenceIpReputation(); + return genError("Intelligence needed"); } -ngx_http_cp_verdict_e -Layer7AccessControl::Impl::checkReputation(const string &source_ip) +EventVerdict +Layer7AccessControl::Impl::queryIpReputation(const string &source_ip) { auto ip_reputation = getIpReputation(source_ip); if (!ip_reputation.ok()) { - dbgWarning(D_L7_ACCESS_CONTROL) << "Could not query intelligence. Retruning default verdict"; - bool is_drop_by_default = getProfileAgentSettingWithDefault(false, "layer7AccessControl.dropByDefault"); - if (!(is_drop_by_default && isPrevent())) return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; - generateLog(source_ip, IntelligenceIpReputation()); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + dbgTrace(D_L7_ACCESS_CONTROL) << "Scheduling Intelligence query - returning Wait verdict"; + scheduleIntelligenceQuery(source_ip); + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; } if (!ip_reputation.unpack().isMalicious()) { dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip; + ip_reputation_cache.deleteEntry(source_ip); return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; } - ip_reputation_cache.emplaceEntry(source_ip, ip_reputation.unpack()); + return generateLog(source_ip, ip_reputation.unpack()); +} - if (isPrevent()) { - dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip; - generateLog(source_ip, ip_reputation.unpack()); - return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; +EventVerdict +Layer7AccessControl::Impl::handleEvent() +{ + auto source_identifier = i_env->get(HttpTransactionData::source_identifier); + if (source_identifier.ok() && IPAddr::createIPAddr(source_identifier.unpack()).ok()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Found a valid source identifier value: " << source_identifier.unpack(); + return queryIpReputation(source_identifier.unpack()); } - dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip; - generateLog(source_ip, ip_reputation.unpack()); + auto orig_source_ip = i_env->get(HttpTransactionData::client_ip_ctx); + if (orig_source_ip.ok()) { + stringstream ss_client_ip; + ss_client_ip << orig_source_ip.unpack(); + return queryIpReputation(ss_client_ip.str()); + } + + dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context"; return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; } +void +Layer7AccessControl::Impl::processIntelligenceResponse( + const string &ip, + const vector> &response) +{ + if (response.empty()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Intelligence reputation response collection is empty. IP is clean."; + ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation()); + return; + } + + for (const auto &intelligence_reply : response) { + if (intelligence_reply.getAssetType() == crowdsec_asset_type && !intelligence_reply.getData().empty()) { + dbgTrace(D_L7_ACCESS_CONTROL) << intelligence_reply.getData().front(); + ip_reputation_cache.emplaceEntry(ip, intelligence_reply.getData().front()); + return; + } + } + + dbgTrace(D_L7_ACCESS_CONTROL) << "Could not find a matching intelligence asset type for IP: " << ip; + ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation()); +} void +Layer7AccessControl::Impl::queryIntelligence() +{ + dbgFlow(D_L7_ACCESS_CONTROL) << "Started IP reputation intelligence routine"; + + while (!pending_ips.empty()) { + i_mainloop->yield(); + + auto ip = *(pending_ips.begin()); + pending_ips.erase(pending_ips.begin()); + + if (ip_reputation_cache.doesKeyExists(ip)) continue; + + dbgTrace(D_L7_ACCESS_CONTROL) << "Querying intelligence about reputation of IP: " << ip; + + QueryRequest request = QueryRequest( + Condition::EQUALS, + "ipv4Addresses", + ip, + true, + AttributeKeyType::REGULAR + ); + + auto response = i_intelligence->queryIntelligence(request); + + if (!response.ok()) { + dbgWarning(D_L7_ACCESS_CONTROL) + << "Failed to query intelligence about reputation of IP: " + << ip + << ", error: " + << response.getErr(); + ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation()); + continue; + } + + processIntelligenceResponse(ip, response.unpack()); + } + + is_intelligence_routine_running = false; +} + +EventVerdict Layer7AccessControl::Impl::generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const { dbgFlow(D_L7_ACCESS_CONTROL) << "About to generate Layer-7 Access Control log"; @@ -287,6 +346,14 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen << ip_reputation.getOrigin() << ip_reputation.getIpv4Address() << ip_reputation.getScenario(); + + if (isPrevent()) { + dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; + } + + dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip; + return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; } Maybe @@ -315,6 +382,7 @@ Layer7AccessControl::Impl::init() registerListener(); i_env = Singleton::Consume::by(); i_intelligence = Singleton::Consume::by(); + i_mainloop = Singleton::Consume::by(); chrono::minutes expiration( getProfileAgentSettingWithDefault(60u, "layer7AccessControl.crowdsec.cacheExpiration") @@ -322,7 +390,7 @@ Layer7AccessControl::Impl::init() ip_reputation_cache.startExpiration( expiration, - Singleton::Consume::by(), + i_mainloop, Singleton::Consume::by() ); } diff --git a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc index d722443..48bc1a9 100644 --- a/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc +++ b/components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc @@ -52,6 +52,7 @@ public: const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP; const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT; const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT; + const EventVerdict wait_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT; Layer7AccessControl l7_access_control; ::Environment env; ConfigComponent config; @@ -62,6 +63,7 @@ public: NiceMock mock_rest; AgentDetails agent_details; IntelligenceComponentV2 intelligence_comp; + I_MainLoop::Routine query_intelligence_routine; Context ctx; }; @@ -273,6 +275,13 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict) const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 }; const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true}; + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(_, _, "Check IP reputation", _)) + .WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0)) + ); + EXPECT_CALL(mock_ml, yield(A())).Times(1); + EXPECT_THAT( HttpRequestHeaderEvent(header1).performNamedQuery(), ElementsAre(Pair("Layer-7 Access Control app", inspect_verdict)) @@ -283,6 +292,13 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict) ); EXPECT_THAT( HttpRequestHeaderEvent(header3).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", wait_verdict)) + ); + + query_intelligence_routine(); + + EXPECT_THAT( + WaitTransactionEvent().performNamedQuery(), ElementsAre(Pair("Layer-7 Access Control app", accept_verdict)) ); } @@ -299,6 +315,13 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation) sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) ).WillOnce(Return(malicious_intelligence_response)); + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(_, _, "Check IP reputation", _)) + .WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0)) + ); + EXPECT_CALL(mock_ml, yield(A())).Times(1); + registerTransactionData(); ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; @@ -310,7 +333,18 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation) EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); - EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict)); + + EXPECT_THAT( + HttpRequestHeaderEvent(header3).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", wait_verdict)) + ); + + query_intelligence_routine(); + + EXPECT_THAT( + WaitTransactionEvent().performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", drop_verdict)) + ); verifyReport(report, "1.2.3.4", "Prevent"); } @@ -327,6 +361,13 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased) sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) ).WillOnce(Return(malicious_intelligence_response)); + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(_, _, "Check IP reputation", _)) + .WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0)) + ); + EXPECT_CALL(mock_ml, yield(A())).Times(1); + registerTransactionData(); ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; @@ -338,7 +379,18 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased) EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); - EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict)); + + EXPECT_THAT( + HttpRequestHeaderEvent(header3).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", wait_verdict)) + ); + + query_intelligence_routine(); + + EXPECT_THAT( + WaitTransactionEvent().performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", drop_verdict)) + ); verifyReport(report, "1.2.3.4", "Prevent"); @@ -361,6 +413,13 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect) sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) ).WillOnce(Return(malicious_intelligence_response)); + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(_, _, "Check IP reputation", _)) + .WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0)) + ); + EXPECT_CALL(mock_ml, yield(A())).Times(1); + registerTransactionData(); ctx.registerValue(HttpTransactionData::source_identifier, "1.2.3.4"); const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; @@ -372,7 +431,18 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect) EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict)); - EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(accept_verdict)); + + EXPECT_THAT( + HttpRequestHeaderEvent(header3).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", wait_verdict)) + ); + + query_intelligence_routine(); + + EXPECT_THAT( + WaitTransactionEvent().performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", accept_verdict)) + ); verifyReport(report, "1.2.3.4", "Detect"); } @@ -389,6 +459,13 @@ TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop) sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE) ).WillOnce(Return(malicious_intelligence_response)); + EXPECT_CALL( + mock_ml, + addOneTimeRoutine(_, _, "Check IP reputation", _)) + .WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0)) + ); + EXPECT_CALL(mock_ml, yield(A())).Times(1); + registerTransactionData(); const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 }; const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1, true }; @@ -397,7 +474,18 @@ TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop) EXPECT_CALL(mock_logging, sendLog(_)).WillOnce(SaveArg<0>(&report)); EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict)); - EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(drop_verdict)); + + EXPECT_THAT( + HttpRequestHeaderEvent(header2).performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", wait_verdict)) + ); + + query_intelligence_routine(); + + EXPECT_THAT( + WaitTransactionEvent().performNamedQuery(), + ElementsAre(Pair("Layer-7 Access Control app", drop_verdict)) + ); verifyReport(report, "", "Prevent"); } diff --git a/components/security_apps/local_policy_mgmt_gen/CMakeLists.txt b/components/security_apps/local_policy_mgmt_gen/CMakeLists.txt index d76674b..d4f060c 100644 --- a/components/security_apps/local_policy_mgmt_gen/CMakeLists.txt +++ b/components/security_apps/local_policy_mgmt_gen/CMakeLists.txt @@ -20,4 +20,5 @@ add_library(local_policy_mgmt_gen new_exceptions.cc access_control_practice.cc configmaps.cc + reverse_proxy_section.cc ) diff --git a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc index 4b90fca..7b9a406 100755 --- a/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc @@ -316,7 +316,7 @@ TriggersInWaapSection::save(cereal::JSONOutputArchive &out_ar) const } ParsedMatch::ParsedMatch(const string &_operator, const string &_tag, const string &_value) - : + : operator_type(_operator), tag(_tag), value(_value) @@ -368,7 +368,7 @@ AppSecOverride::AppSecOverride(const SourcesIdentifiers &parsed_trusted_sources) // LCOV_EXCL_START Reason: no test exist AppSecOverride::AppSecOverride(const InnerException &parsed_exceptions) - : + : id(parsed_exceptions.getBehaviorId()), parsed_match(parsed_exceptions.getMatch()) { @@ -413,7 +413,7 @@ WebAppSection::WebAppSection( const string &default_mode, const AppSecTrustedSources &parsed_trusted_sources, const vector &parsed_exceptions) - : + : application_urls(_application_urls), asset_id(_asset_id), asset_name(_asset_name), @@ -460,7 +460,7 @@ WebAppSection::WebAppSection( const AppsecPracticeAntiBotSection &_anti_bots, const LogTriggerSection &parsed_log_trigger, const AppSecTrustedSources &parsed_trusted_sources) - : + : application_urls(_application_urls), asset_id(_asset_id), asset_name(_asset_name), @@ -477,6 +477,7 @@ WebAppSection::WebAppSection( { web_attack_mitigation = true; web_attack_mitigation_action = + web_attack_mitigation_mode != "Prevent" ? "Transparent" : web_attack_mitigation_severity == "critical" ? "low" : web_attack_mitigation_severity == "high" ? "balanced" : web_attack_mitigation_severity == "medium" ? "high" : @@ -584,6 +585,9 @@ ParsedRule::load(cereal::JSONInputArchive &archive_in) parseAppsecJSONKey("custom-response", custom_response, archive_in); parseAppsecJSONKey("source-identifiers", source_identifiers, archive_in); parseAppsecJSONKey("trusted-sources", trusted_sources, archive_in); + parseAppsecJSONKey("upstream", rpm_upstream, archive_in); + parseAppsecJSONKey("rp-settings", rpm_settings, archive_in); + parseAppsecJSONKey("ssl", rpm_is_ssl, archive_in); try { archive_in(cereal::make_nvp("host", host)); } catch (const cereal::Exception &e) @@ -620,6 +624,24 @@ ParsedRule::getMode() const return mode; } +const string & +ParsedRule::rpmGetUpstream() const +{ + return rpm_upstream; +} + +const std::string & +ParsedRule::rpmGetRPSettings() const +{ + return rpm_settings; +} + +bool +ParsedRule::rpmIsHttps() const +{ + return rpm_is_ssl; +} + void ParsedRule::setHost(const string &_host) { @@ -691,6 +713,7 @@ AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading Appsec Linux Policy"; parseAppsecJSONKey("policies", policies, archive_in); + parseAppsecJSONKey>("rp-settings", rpm_settings, archive_in); parseAppsecJSONKey>("practices", practices, archive_in); parseAppsecJSONKey>("log-triggers", log_triggers, archive_in); parseAppsecJSONKey>("custom-responses", custom_responses, archive_in); @@ -745,6 +768,13 @@ AppsecLinuxPolicy::getAppsecSourceIdentifierSpecs() const return sources_identifiers; } + +const vector & +AppsecLinuxPolicy::rpmGetRPSettings() const +{ + return rpm_settings; +} + void AppsecLinuxPolicy::addSpecificRule(const ParsedRule &_rule) { diff --git a/components/security_apps/local_policy_mgmt_gen/exceptions_section.cc b/components/security_apps/local_policy_mgmt_gen/exceptions_section.cc index 8ad559c..c60c4cd 100755 --- a/components/security_apps/local_policy_mgmt_gen/exceptions_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/exceptions_section.cc @@ -304,11 +304,13 @@ ExceptionMatch::getMatch() const ExceptionBehavior::ExceptionBehavior(const string &_value) { key = _value == "suppressLog" ? "log" : "action"; - value = key_to_action.at(_value); try { + value = key_to_action.at(_value); id = to_string(boost::uuids::random_generator()()); } catch (const boost::uuids::entropy_error &e) { dbgWarning(D_LOCAL_POLICY) << "Failed to generate exception behavior UUID. Error: " << e.what(); + } catch (std::exception &e) { + dbgWarning(D_LOCAL_POLICY) << "Failed to find exception name: " << _value << ". Error: " << e.what(); } } diff --git a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h index a113c6b..eaf9d8d 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h @@ -28,6 +28,7 @@ #include "triggers_section.h" #include "exceptions_section.h" #include "trusted_sources_section.h" +#include "reverse_proxy_section.h" #include "new_practice.h" class AppSecWebBotsURI @@ -148,7 +149,7 @@ public: PracticeAdvancedConfig() {} PracticeAdvancedConfig(const AppSecPracticeSpec &parsed_appsec_spec) - : + : http_header_max_size(parsed_appsec_spec.getWebAttacks().getMaxHeaderSizeBytes()), http_illegal_methods_allowed(0), http_request_body_max_size(parsed_appsec_spec.getWebAttacks().getMaxBodySizeKb()), @@ -162,7 +163,7 @@ public: int _http_request_body_max_size, int _json_max_object_depth, int _url_max_size) - : + : http_header_max_size(_http_header_max_size), http_illegal_methods_allowed(0), http_request_body_max_size(_http_request_body_max_size), @@ -186,7 +187,7 @@ class TriggersInWaapSection { public: TriggersInWaapSection(const LogTriggerSection &log_section) - : + : trigger_type("log"), id(log_section.getTriggerId()), name(log_section.getTriggerName()), @@ -241,13 +242,13 @@ public: AppsecPracticeAntiBotSection(const NewAppSecPracticeAntiBot &anti_bot) : injected_uris(anti_bot.getIjectedUris()), validated_uris(anti_bot.getValidatedUris()) - {}; + {}; // LCOV_EXCL_STOP AppsecPracticeAntiBotSection(const AppSecPracticeAntiBot &anti_bot) : injected_uris(anti_bot.getIjectedUris()), validated_uris(anti_bot.getValidatedUris()) - {}; + {}; void save(cereal::JSONOutputArchive &out_ar) const; @@ -278,20 +279,20 @@ public: ); WebAppSection( - const std::string &_application_urls, - const std::string &_asset_id, - const std::string &_asset_name, - const std::string &_rule_id, - const std::string &_rule_name, - const std::string &_practice_id, - const std::string &_practice_name, - const std::string &_context, - const std::string &_web_attack_mitigation_severity, - const std::string &_web_attack_mitigation_mode, - const PracticeAdvancedConfig &_practice_advanced_config, - const AppsecPracticeAntiBotSection &_anti_bots, - const LogTriggerSection &parsed_log_trigger, - const AppSecTrustedSources &parsed_trusted_sources); + const std::string &_application_urls, + const std::string &_asset_id, + const std::string &_asset_name, + const std::string &_rule_id, + const std::string &_rule_name, + const std::string &_practice_id, + const std::string &_practice_name, + const std::string &_context, + const std::string &_web_attack_mitigation_severity, + const std::string &_web_attack_mitigation_mode, + const PracticeAdvancedConfig &_practice_advanced_config, + const AppsecPracticeAntiBotSection &_anti_bots, + const LogTriggerSection &parsed_log_trigger, + const AppSecTrustedSources &parsed_trusted_sources); void save(cereal::JSONOutputArchive &out_ar) const; @@ -331,7 +332,7 @@ public: const std::string &_web_attack_mitigation_mode, bool _web_attack_mitigation, const PracticeAdvancedConfig &_practice_advanced_config) - : + : application_urls(_application_urls), asset_id(_asset_id), asset_name(_asset_name), @@ -345,7 +346,7 @@ public: web_attack_mitigation_mode(_web_attack_mitigation_mode), web_attack_mitigation(_web_attack_mitigation), practice_advanced_config(_practice_advanced_config) - {} + {} void save(cereal::JSONOutputArchive &out_ar) const; @@ -371,7 +372,7 @@ public: AppSecRulebase( std::vector _webApplicationPractices, std::vector _webAPIPractices) - : + : webApplicationPractices(_webApplicationPractices), webAPIPractices(_webAPIPractices) {} @@ -387,7 +388,7 @@ class AppSecWrapper { public: AppSecWrapper(const AppSecRulebase &_app_sec) - : + : app_sec_rulebase(_app_sec) {} @@ -409,6 +410,9 @@ public: const std::vector & getPractices() const; const std::string & getHost() const; const std::string & getMode() const; + const std::string &rpmGetUpstream() const; + const std::string &rpmGetRPSettings() const; + bool rpmIsHttps() const; void setHost(const std::string &_host); void setMode(const std::string &_mode); const std::string & getCustomResponse() const; @@ -424,6 +428,9 @@ private: std::string custom_response; std::string source_identifiers; std::string trusted_sources; + std::string rpm_upstream; + std::string rpm_settings; + bool rpm_is_ssl = false; }; class AppsecPolicySpec : Singleton::Consume @@ -453,7 +460,7 @@ public: const std::vector &_exceptions, const std::vector &_trusted_sources, const std::vector &_sources_identifiers) - : + : policies(_policies), practices(_practices), log_triggers(_log_triggers), @@ -471,6 +478,7 @@ public: const std::vector & getAppsecExceptions() const; const std::vector & getAppsecTrustedSourceSpecs() const; const std::vector & getAppsecSourceIdentifierSpecs() const; + const std::vector &rpmGetRPSettings() const; void addSpecificRule(const ParsedRule &_rule); private: @@ -481,6 +489,7 @@ private: std::vector exceptions; std::vector trusted_sources; std::vector sources_identifiers; + std::vector rpm_settings; }; #endif // __APPSEC_PRACTICE_SECTION_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h index 2ef8b31..5c04ef5 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h +++ b/components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h @@ -50,7 +50,7 @@ static const std::unordered_map string_to_trigger_type static const std::unordered_map key_to_practices_val = { { "prevent-learn", "Prevent"}, - { "detect-learn", "Detect"}, + { "detect-learn", "Learn"}, { "prevent", "Prevent"}, { "detect", "Detect"}, { "inactive", "Inactive"} @@ -70,9 +70,9 @@ parseAppsecJSONKey( archive_in.setNextName(nullptr); value = default_value; dbgDebug(D_LOCAL_POLICY) - << "Could not parse the required key. Key: " + << "Could not parse the required key. Key: \"" << key_name - << ", Error: " + << "\", Error: " << e.what(); } } diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_appsec_linux_policy.h b/components/security_apps/local_policy_mgmt_gen/include/new_appsec_linux_policy.h index 5b93901..cd664bd 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_appsec_linux_policy.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_appsec_linux_policy.h @@ -59,6 +59,7 @@ public: trusted_sources(_trusted_sources), sources_identifiers(_sources_identifiers) {} // LCOV_EXCL_STOP + void serialize(cereal::JSONInputArchive &archive_in); const NewAppsecPolicySpec & getAppsecPolicySpec() const; const std::vector & getAppSecPracticeSpecs() const; diff --git a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h index c9f6375..58deb43 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/new_practice.h +++ b/components/security_apps/local_policy_mgmt_gen/include/new_practice.h @@ -147,8 +147,8 @@ public: // LCOV_EXCL_STOP FileSecurityProtectionsSection( - int _file_size_limit, - int _archive_file_size_limit, + uint64_t _file_size_limit, + uint64_t _archive_file_size_limit, bool _allow_files_without_name, bool _required_file_size_limit, bool _required_archive_extraction, @@ -171,8 +171,8 @@ public: void save(cereal::JSONOutputArchive &out_ar) const; private: - int file_size_limit; - int archive_file_size_limit; + uint64_t file_size_limit; + uint64_t archive_file_size_limit; bool allow_files_without_name; bool required_file_size_limit; bool required_archive_extraction; @@ -233,13 +233,13 @@ class NewFileSecurityArchiveInspection public: void load(cereal::JSONInputArchive &archive_in); - int getArchiveFileSizeLimit() const; + uint64_t getArchiveFileSizeLimit() const; bool getrequiredArchiveExtraction() const; const std::string & getMultiLevelArchiveAction() const; const std::string & getUnopenedArchiveAction() const; private: - int scan_max_file_size; + uint64_t scan_max_file_size; bool extract_archive_files; std::string scan_max_file_size_unit; std::string archived_files_within_archived_files; @@ -251,11 +251,11 @@ class NewFileSecurityLargeFileInspection public: void load(cereal::JSONInputArchive &archive_in); - int getFileSizeLimit() const; + uint64_t getFileSizeLimit() const; const std::string & getFileSizeLimitAction() const; private: - int file_size_limit; + uint64_t file_size_limit; std::string file_size_limit_unit; std::string files_exceeding_size_limit_action; }; diff --git a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h index 2436766..8b9cedc 100755 --- a/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h +++ b/components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h @@ -40,6 +40,7 @@ #include "trusted_sources_section.h" #include "new_appsec_linux_policy.h" #include "access_control_practice.h" +#include "reverse_proxy_section.h" enum class AnnotationTypes { PRACTICE, @@ -109,11 +110,6 @@ private: }; class PolicyMakerUtils - : - Singleton::Consume, - Singleton::Consume, - Singleton::Consume, - Singleton::Consume { public: std::string proccesSingleAppsecPolicy( @@ -206,6 +202,7 @@ private: createThreatPreventionPracticeSections( const std::string &asset_name, const std::string &url, + const std::string &port, const std::string &uri, const std::string &default_mode, const V1beta2AppsecLinuxPolicy &policy, @@ -231,6 +228,11 @@ private: template void createAgentPolicyFromAppsecPolicy(const std::string &policy_name, const T &appsec_policy); + void rpmBuildNginxServers(const AppsecLinuxPolicy &policy); + void rpmReportInfo(const std::string &msg); + void rpmReportError(const std::string &msg); + + std::string policy_version_name; std::map log_triggers; std::map web_user_res_triggers; std::map> inner_exceptions; diff --git a/components/security_apps/local_policy_mgmt_gen/include/reverse_proxy_section.h b/components/security_apps/local_policy_mgmt_gen/include/reverse_proxy_section.h new file mode 100644 index 0000000..8e3e97f --- /dev/null +++ b/components/security_apps/local_policy_mgmt_gen/include/reverse_proxy_section.h @@ -0,0 +1,68 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __REVERSE_PROXY_SECTION_H__ +#define __REVERSE_PROXY_SECTION_H__ + +#include +#include + +#include "agent_core_utilities.h" +#include "i_shell_cmd.h" + +class ParsedRule; + +class RPMSettings +{ +public: + void load(cereal::JSONInputArchive &archive_in); + + const std::string & getName() const; + std::string applySettings(const std::string &server_content) const; + +private: + std::string name; + std::string host_hdr = "$host"; + std::string dns_resolver = "127.0.0.11"; +}; + +class ReverseProxyBuilder +{ +public: + static void init(); + + static Maybe addNginxServerLocation( + std::string location, + const std::string &host, + const ParsedRule &rule, + const RPMSettings &rp_settings); + + static Maybe createNewNginxServer( + const std::string &host, + const ParsedRule &rule, + const RPMSettings &rp_settings); + + static std::string replaceTemplate( + const std::string &content, + const boost::regex &nginx_directive_template, + const std::string &value); + + static Maybe reloadNginx(); + +private: + static Maybe createSSLNginxServer(const std::string &host, const RPMSettings &rp_settings); + static Maybe createHTTPNginxServer(const std::string &host, const RPMSettings &rp_settings); + + static Maybe getTemplateContent(const std::string &nginx_template_name); +}; +#endif // __REVERSE_PROXY_SECTION_H__ diff --git a/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h b/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h index 189cff4..98bac38 100644 --- a/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h +++ b/components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h @@ -90,6 +90,7 @@ public: RulesConfigRulebase( const std::string &_name, const std::string &_url, + const std::string &_port, const std::string &_uri, std::vector _practices, std::vector _parameters, diff --git a/components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc b/components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc index f246783..3dd0685 100644 --- a/components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc +++ b/components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc @@ -55,9 +55,7 @@ const static string default_local_mgmt_policy_path = "/conf/local_policy.yaml"; class LocalPolicyMgmtGenerator::Impl : - public Singleton::Provide::From, - public Singleton::Consume, - public Singleton::Consume + public Singleton::Provide::From { public: @@ -111,7 +109,6 @@ public: private: PolicyMakerUtils policy_maker_utils; - }; LocalPolicyMgmtGenerator::LocalPolicyMgmtGenerator() diff --git a/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc b/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc index 24dc962..c2a77e1 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc @@ -70,3 +70,31 @@ V1beta2AppsecLinuxPolicy::addSpecificRule(const NewParsedRule &_rule) policies.addSpecificRule(_rule); } // LCOV_EXCL_STOP + +void +V1beta2AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in) +{ + dbgInfo(D_LOCAL_POLICY) << "Loading Appsec V1Beta2 Linux Policy"; + + // Check for the presence of "apiVersion" key, present only from V1Beta2 + string api_version; + archive_in(cereal::make_nvp("apiVersion", api_version)); + if (api_version != "v1beta2") throw cereal::Exception("Failed to parse JSON as v1Beta2 version"); + + parseAppsecJSONKey("policies", policies, archive_in); + parseAppsecJSONKey>( + "threatPreventionPractices", + threat_prevection_practices, + archive_in + ); + parseAppsecJSONKey>( + "accessControlPractices", + access_control_practices, + archive_in + ); + parseAppsecJSONKey>("logTriggers", log_triggers, archive_in); + parseAppsecJSONKey>("customResponse", custom_responses, archive_in); + parseAppsecJSONKey>("exceptions", exceptions, archive_in); + parseAppsecJSONKey>("trustedSources", trusted_sources, archive_in); + parseAppsecJSONKey>("sourcesIdentifiers", sources_identifiers, archive_in); +} diff --git a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc index 115a781..2d6052b 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_exceptions.cc @@ -44,7 +44,7 @@ void NewAppsecException::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading New AppSec exception"; - parseAppsecJSONKey("name", name, archive_in); + parseAppsecJSONKey("name", name, archive_in, "exception"); parseAppsecJSONKey("action", action, archive_in); parseAppsecJSONKey("appsecClassName", appsec_class_name, archive_in); if (valid_actions.count(action) == 0) { diff --git a/components/security_apps/local_policy_mgmt_gen/new_practice.cc b/components/security_apps/local_policy_mgmt_gen/new_practice.cc index ab2fd01..9d1cf67 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_practice.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_practice.cc @@ -42,7 +42,7 @@ static const std::unordered_map key_to_mode_val = { { "detect", "Detect"}, { "inactive", "Inactive"} }; -static const std::unordered_map unit_to_int = { +static const std::unordered_map unit_to_int = { { "bytes", 1}, { "KB", 1024}, { "MB", 1048576}, @@ -631,8 +631,8 @@ NewIntrusionPrevention::getMode() const } FileSecurityProtectionsSection::FileSecurityProtectionsSection( - int _file_size_limit, - int _archive_file_size_limit, + uint64_t _file_size_limit, + uint64_t _archive_file_size_limit, bool _allow_files_without_name, bool _required_file_size_limit, bool _required_archive_extraction, @@ -720,7 +720,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security Archive Inspection practice"; parseAppsecJSONKey("extractArchiveFiles", extract_archive_files, archive_in); - parseAppsecJSONKey("scanMaxFileSize", scan_max_file_size, archive_in, 0); + parseAppsecJSONKey("scanMaxFileSize", scan_max_file_size, archive_in, 0); parseAppsecJSONKey("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "bytes"); if (size_unit.count(scan_max_file_size_unit) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -749,7 +749,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in) } } -int +uint64_t NewFileSecurityArchiveInspection::getArchiveFileSizeLimit() const { if (unit_to_int.find(scan_max_file_size_unit) == unit_to_int.end()) { @@ -784,7 +784,7 @@ void NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security large File Inspection practice"; - parseAppsecJSONKey("fileSizeLimit", file_size_limit, archive_in); + parseAppsecJSONKey("fileSizeLimit", file_size_limit, archive_in); parseAppsecJSONKey("fileSizeLimitUnit", file_size_limit_unit, archive_in, "bytes"); if (size_unit.count(file_size_limit_unit) == 0) { dbgWarning(D_LOCAL_POLICY) @@ -803,7 +803,7 @@ NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in) } } -int +uint64_t NewFileSecurityLargeFileInspection::getFileSizeLimit() const { if (unit_to_int.find(file_size_limit_unit) == unit_to_int.end()) { diff --git a/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc b/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc index ecff3b7..ef29d66 100755 --- a/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc +++ b/components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc @@ -64,7 +64,7 @@ void Identifier::load(cereal::JSONInputArchive &archive_in) { dbgTrace(D_LOCAL_POLICY) << "Loading source identifiers spec"; - parseAppsecJSONKey("sourceIdentifier", identifier, archive_in); + parseAppsecJSONKey("identifier", identifier, archive_in); if (valid_identifiers.count(identifier) == 0) { dbgWarning(D_LOCAL_POLICY) << "AppSec identifier invalid: " << identifier; } diff --git a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc index 9aea4b7..7f8c5a2 100755 --- a/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc +++ b/components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc @@ -13,6 +13,11 @@ #include "policy_maker_utils.h" +#include + +#include "local_policy_mgmt_gen.h" +#include "log_generator.h" + using namespace std; USE_DEBUG_FLAG(D_NGINX_POLICY); @@ -58,7 +63,7 @@ template Maybe PolicyMakerUtils::openFileAsJson(const string &path) { - auto maybe_file_as_json = Singleton::Consume::by()->getExecOutput( + auto maybe_file_as_json = Singleton::Consume::by()->getExecOutput( getFilesystemPathConfig() + "/bin/yq " + path + " -o json" ); @@ -67,7 +72,7 @@ PolicyMakerUtils::openFileAsJson(const string &path) return genError("Could not convert policy from yaml to json. Error: " + maybe_file_as_json.getErr()); } - auto i_orchestration_tools = Singleton::Consume::by(); + auto i_orchestration_tools = Singleton::Consume::by(); auto maybe_file = i_orchestration_tools->jsonStringToObject( maybe_file_as_json.unpack() ); @@ -136,10 +141,11 @@ PolicyMakerUtils::splitHostName(const string &host_name) url = url.substr(0, url.find(":")); } - if (host_name == "*") { + if (host_name == "*" || host_name == "*:*") { url = "Any"; uri = "Any"; } + return make_tuple(url, port, uri); } @@ -323,6 +329,7 @@ extractAnnotationsNames( if (!trusted_sources_annotation_name.empty()) { rule_annotation[AnnotationTypes::TRUSTED_SOURCES] = policy_name + "/" + trusted_sources_annotation_name; } + return rule_annotation; } // LCOV_EXCL_STOP @@ -451,6 +458,23 @@ getAppsecCustomResponseSpec(const string &custom_response_annotation_name, const return *custom_response_it; } +template +R +rpmGetAppsecRPSettingSpec(const string &rp_settings_name, const T &policy) +{ + auto rp_settings_vec = policy.rpmGetRPSettings(); + auto rp_settings_it = extractElement( + rp_settings_vec.begin(), + rp_settings_vec.end(), + rp_settings_name); + + if (rp_settings_it == rp_settings_vec.end()) { + dbgTrace(D_NGINX_POLICY) << "Failed to retrieve AppSec RP Settings"; + return R(); + } + return *rp_settings_it; +} + template R getAppsecSourceIdentifierSpecs(const string &source_identifiers_annotation_name, const T &policy) @@ -843,6 +867,7 @@ createUserIdentifiers( RulesConfigRulebase createMultiRulesSections( const string &url, + const string &port, const string &uri, const string &practice_id, const string &practice_name, @@ -878,6 +903,7 @@ createMultiRulesSections( RulesConfigRulebase rules_config = RulesConfigRulebase( asset_name, url, + port, uri, {practice}, exceptions_result, @@ -890,6 +916,7 @@ createMultiRulesSections( RulesConfigRulebase createMultiRulesSections( const string &url, + const string &port, const string &uri, const string &practice_id, const string &practice_name, @@ -907,7 +934,8 @@ createMultiRulesSections( const string &exception_name, const vector &exceptions) { - ParametersSection exception_param = ParametersSection(exceptions[0].getBehaviorId(), exception_name); + string behaviorId = exceptions.empty() ? "" : exceptions[0].getBehaviorId(); + ParametersSection exception_param = ParametersSection(behaviorId, exception_name); vector practices; if (!practice_id.empty()) { @@ -934,6 +962,7 @@ createMultiRulesSections( RulesConfigRulebase rules_config = RulesConfigRulebase( asset_name, url, + port, uri, practices, {exception_param}, @@ -983,7 +1012,7 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, const st auto snort_scriipt_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py"; auto cmd = "python " + snort_scriipt_path + " " + path + ".rule " + path + ".out " + path + ".err"; - auto res = Singleton::Consume::by()->getExecOutput(cmd); + auto res = Singleton::Consume::by()->getExecOutput(cmd); if (!res.ok()) { dbgWarning(D_LOCAL_POLICY) << res.getErr(); @@ -996,7 +1025,7 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, const st return; } - auto i_orchestration_tools = Singleton::Consume::by(); + auto i_orchestration_tools = Singleton::Consume::by(); i_orchestration_tools->removeFile(path + ".rule"); i_orchestration_tools->removeFile(path + ".out"); i_orchestration_tools->removeFile(path + ".err"); @@ -1153,12 +1182,15 @@ void PolicyMakerUtils::createThreatPreventionPracticeSections( const string &asset_name, const string &url, + const string &port, const string &uri, const string &default_mode, const V1beta2AppsecLinuxPolicy &policy, map &rule_annotations) { - if (rule_annotations[AnnotationTypes::PRACTICE].empty()) { + if (rule_annotations[AnnotationTypes::PRACTICE].empty() || + web_apps.count(asset_name) + ) { return; } string practice_id = ""; @@ -1170,6 +1202,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections( RulesConfigRulebase rule_config = createMultiRulesSections( url, + port, uri, practice_id, rule_annotations[AnnotationTypes::PRACTICE], @@ -1353,7 +1386,14 @@ PolicyMakerUtils::createPolicyElementsByRule( ); } - if (!rule_annotations[AnnotationTypes::PRACTICE].empty()) { + string full_url = rule.getHost() == "*" || rule.getHost() == "*:*" + ? "Any" + : rule.getHost(); + + + if (!rule_annotations[AnnotationTypes::PRACTICE].empty() && + !web_apps.count(full_url) + ) { string practice_id = ""; try { practice_id = to_string(boost::uuids::random_generator()()); @@ -1362,12 +1402,10 @@ PolicyMakerUtils::createPolicyElementsByRule( } tuple splited_host_name = splitHostName(rule.getHost()); - string full_url = rule.getHost() == "*" - ? "Any" - : rule.getHost(); RulesConfigRulebase rule_config = createMultiRulesSections( std::get<0>(splited_host_name), + std::get<1>(splited_host_name), std::get<2>(splited_host_name), practice_id, rule_annotations[AnnotationTypes::PRACTICE], @@ -1426,7 +1464,9 @@ PolicyMakerUtils::createPolicyElementsByRule rule_annotations = extractAnnotationsNames(rule, default_rule, policy_name); + if ( + rule_annotations.count(AnnotationTypes::TRIGGER) > 0 && !rule_annotations[AnnotationTypes::TRIGGER].empty() && !log_triggers.count(rule_annotations[AnnotationTypes::TRIGGER]) ) { @@ -1438,6 +1478,7 @@ PolicyMakerUtils::createPolicyElementsByRule 0 && !rule_annotations[AnnotationTypes::WEB_USER_RES].empty() && !web_user_res_triggers.count(rule_annotations[AnnotationTypes::WEB_USER_RES]) ) { @@ -1449,6 +1490,7 @@ PolicyMakerUtils::createPolicyElementsByRule 0 && !rule_annotations[AnnotationTypes::EXCEPTION].empty() && !inner_exceptions.count(rule_annotations[AnnotationTypes::EXCEPTION]) ) { @@ -1460,6 +1502,8 @@ PolicyMakerUtils::createPolicyElementsByRule 0 && + rule_annotations.count(AnnotationTypes::SOURCE_IDENTIFIERS) > 0 && !rule_annotations[AnnotationTypes::TRUSTED_SOURCES].empty() && !rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty() && !trusted_sources.count(rule_annotations[AnnotationTypes::TRUSTED_SOURCES]) @@ -1473,6 +1517,7 @@ PolicyMakerUtils::createPolicyElementsByRule 0 && !rule_annotations[AnnotationTypes::PRACTICE].empty() && !web_apps.count(rule_annotations[AnnotationTypes::PRACTICE]) ) { @@ -1484,7 +1529,7 @@ PolicyMakerUtils::createPolicyElementsByRule splited_host_name = splitHostName(rule.getHost()); @@ -1501,6 +1546,7 @@ PolicyMakerUtils::createPolicyElementsByRule(splited_host_name), + std::get<1>(splited_host_name), std::get<2>(splited_host_name), rule.getMode(), policy, @@ -1531,11 +1577,11 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy(const string &policy_name, c R default_rule = appsec_policy.getAppsecPolicySpec().getDefaultRule(); - // add default rule to policy - createPolicyElementsByRule(default_rule, default_rule, appsec_policy, policy_name); - vector specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules(); createPolicyElements(specific_rules, default_rule, appsec_policy, policy_name); + + // add default rule to policy + createPolicyElementsByRule(default_rule, default_rule, appsec_policy, policy_name); } // LCOV_EXCL_START Reason: no test exist @@ -1545,17 +1591,10 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy( - default_rule, - default_rule, - appsec_policy, - policy_name); - vector specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules(); createPolicyElements( specific_rules, @@ -1563,6 +1602,13 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy( + default_rule, + default_rule, + appsec_policy, + policy_name); } // LCOV_EXCL_STOP @@ -1572,15 +1618,31 @@ PolicyMakerUtils::proccesSingleAppsecPolicy( const string &policy_version, const string &local_appsec_policy_path) { - Maybe maybe_policy = openFileAsJson(policy_path); - if (!maybe_policy.ok()){ - dbgWarning(D_LOCAL_POLICY) << maybe_policy.getErr(); - return ""; + + Maybe maybe_policy_v1beta2 = openFileAsJson(policy_path); + if (maybe_policy_v1beta2.ok()) { + policy_version_name = "v1beta2"; + createAgentPolicyFromAppsecPolicy( + getPolicyName(policy_path), + maybe_policy_v1beta2.unpack() + ); + } else { + policy_version_name = "v1beta1"; + dbgInfo(D_LOCAL_POLICY) + << "Failed to retrieve AppSec local policy with version: v1beta2, Trying version: v1beta1"; + + Maybe maybe_policy_v1beta1 = openFileAsJson(policy_path); + if (!maybe_policy_v1beta1.ok()){ + dbgWarning(D_LOCAL_POLICY) << maybe_policy_v1beta1.getErr(); + return ""; + } + createAgentPolicyFromAppsecPolicy( + getPolicyName(policy_path), + maybe_policy_v1beta1.unpack() + ); + + if (getenv("OPENAPPSEC_STANDALONE")) rpmBuildNginxServers(maybe_policy_v1beta1.unpack()); } - createAgentPolicyFromAppsecPolicy( - getPolicyName(policy_path), - maybe_policy.unpack() - ); PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version); return dumpPolicyToFile( @@ -1588,3 +1650,114 @@ PolicyMakerUtils::proccesSingleAppsecPolicy( local_appsec_policy_path ); } + +void +PolicyMakerUtils::rpmReportInfo(const std::string &msg) +{ + dbgTrace(D_LOCAL_POLICY) << msg; + + LogGen( + msg, + ReportIS::Audience::SECURITY, + ReportIS::Severity::INFO, + ReportIS::Priority::LOW, + ReportIS::Tags::ORCHESTRATOR + ); +} + +void +PolicyMakerUtils::rpmReportError(const std::string &msg) +{ + dbgWarning(D_LOCAL_POLICY) << msg; + + LogGen( + msg, + ReportIS::Audience::SECURITY, + ReportIS::Severity::CRITICAL, + ReportIS::Priority::URGENT, + ReportIS::Tags::ORCHESTRATOR + ); +} + +void +PolicyMakerUtils::rpmBuildNginxServers(const AppsecLinuxPolicy &policy) +{ + rpmReportInfo("Started building NGINX servers"); + + ReverseProxyBuilder::init(); + bool full_success = true; + bool partial_success = false; + set> processed_rules; + for (ParsedRule const &rule : policy.getAppsecPolicySpec().getSpecificRules()) { + tuple splited_host_name = splitHostName(rule.getHost()); + string host = std::get<0>(splited_host_name); + if (host.empty() || rule.rpmGetUpstream().empty()) continue; + + string location = std::get<2>(splited_host_name); + if (location.empty()) location = "/"; + + dbgTrace(D_LOCAL_POLICY) + << "Building NGINX server: " + << host + << ", location: " + << location + << " RP-Settings: " + << rule.rpmGetRPSettings(); + + RPMSettings rp_settings = + rpmGetAppsecRPSettingSpec(rule.rpmGetRPSettings(), policy); + pair server = {host, rule.rpmIsHttps()}; + auto it = processed_rules.find(server); + if (it != processed_rules.end()) { + auto maybe_res = ReverseProxyBuilder::addNginxServerLocation(location, host, rule, rp_settings); + if (!maybe_res.ok()) { + rpmReportError( + "Could not add an NGINX server location: " + location + " to server: " + host + + ", error: " + maybe_res.getErr() + ); + full_success = false; + continue; + } + rpmReportInfo("NGINX server location: " + location + " was successfully added to server: " + host); + partial_success = true; + } else { + auto maybe_res = ReverseProxyBuilder::createNewNginxServer(host, rule, rp_settings); + if (!maybe_res.ok()) { + rpmReportError("Could not create a new NGINX server: " + host + ", error: " + maybe_res.getErr()); + full_success = false; + continue; + } + rpmReportInfo( + (rule.rpmIsHttps() ? string("SSL") : string("HTTP")) + " NGINX server: " + host + + " was successfully built" + ); + processed_rules.insert(server); + + maybe_res = ReverseProxyBuilder::addNginxServerLocation(location, host, rule, rp_settings); + if (!maybe_res.ok()) { + rpmReportError( + "Could not add an NGINX server location: " + location + " to server: " + host + + ", error: " + maybe_res.getErr() + ); + full_success = false; + continue; + } + rpmReportInfo("NGINX server location: " + location + " was successfully added to server: " + host); + partial_success = true; + } + } + + auto maybe_reload_nginx = ReverseProxyBuilder::reloadNginx(); + if (!maybe_reload_nginx.ok()) { + rpmReportError("Could not reload NGINX, error: " + maybe_reload_nginx.getErr()); + return; + } + + if (full_success) { + rpmReportInfo("NGINX configuration was loaded successfully!"); + } else if (partial_success) { + rpmReportInfo("NGINX configuration was partially loaded"); + } else { + rpmReportError("Could not load any NGINX configuration"); + } +} diff --git a/components/security_apps/local_policy_mgmt_gen/reverse_proxy_section.cc b/components/security_apps/local_policy_mgmt_gen/reverse_proxy_section.cc new file mode 100755 index 0000000..a24e97f --- /dev/null +++ b/components/security_apps/local_policy_mgmt_gen/reverse_proxy_section.cc @@ -0,0 +1,456 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "reverse_proxy_section.h" + +#include +#include +#include +#include +#include + +#include "local_policy_mgmt_gen.h" +#include "local_policy_common.h" +#include "appsec_practice_section.h" +#include "debug.h" + +using namespace std; + +USE_DEBUG_FLAG(D_LOCAL_POLICY); + +static string conf_base_path = "/etc/cp/conf/"; +static string certs_path = "/etc/certs/"; +static string nginx_templates_path = "/etc/nginx/nginx-templates/"; +static const string nginx_configuration_path = "openappsec-nginx-servers/"; +static const string nginx_http_server_template = "nginx-http-server"; +static const string nginx_ssl_server_template = "nginx-ssl-server"; +static const string nginx_location_template = "nginx-location-block"; + +static const boost::regex host_template(""); +static const boost::regex private_key_template(""); +static const boost::regex certificate_template(""); +static const boost::regex location_template(""); +static const boost::regex upstream_template(""); +static const boost::regex host_header_template(""); +static const boost::regex dns_resolver_template(""); + +class ReverseProxyCertUtils +{ +public: + static std::pair findMatchingCertificate(const std::string &host); + static void init(); + +private: + static std::vector getFilesByExtension(const std::string &extension); + static void untarCertificatesPackages(); + + static Maybe extractModulus(const std::string &path, const std::string &type); + + static std::unordered_map + calculatePublicModulus(const std::vector &certs); + + static std::unordered_map + calculatePrivateModulus(const std::vector &keys); + + static std::unordered_map cert_key_map; +}; +unordered_map ReverseProxyCertUtils::cert_key_map; + +void +RPMSettings::load(cereal::JSONInputArchive &archive_in) +{ + dbgFlow(D_LOCAL_POLICY) << "Loading RP Settings"; + + parseAppsecJSONKey("name", name, archive_in); + parseAppsecJSONKey("host-header", host_hdr, archive_in, "$host"); + parseAppsecJSONKey("dns-resolver", dns_resolver, archive_in, "127.0.0.11"); +} + +const string & +RPMSettings::getName() const +{ + return name; +} + +string +RPMSettings::applySettings(const std::string &server_content) const +{ + string new_server_content = ReverseProxyBuilder::replaceTemplate(server_content, host_header_template, host_hdr); + return ReverseProxyBuilder::replaceTemplate(new_server_content, dns_resolver_template, dns_resolver); +} + +void +ReverseProxyCertUtils::init() +{ + certs_path = getProfileAgentSettingWithDefault("/etc/certs/", "openappsec.reverseProxy.certs"); + + untarCertificatesPackages(); + cert_key_map.clear(); + auto public_modulus_map = calculatePublicModulus(getFilesByExtension(".pem")); + auto private_modulus_map = calculatePrivateModulus(getFilesByExtension(".key")); + for (const auto &public_modulus_entry : public_modulus_map) { + auto public_modulus = public_modulus_entry.second; + if (private_modulus_map.find(public_modulus) != private_modulus_map.end()) { + dbgTrace(D_LOCAL_POLICY) + << "Successfully parsed certificate: " + << public_modulus_entry.first + << " with private key: " + << private_modulus_map[public_modulus]; + + cert_key_map[public_modulus_entry.first] = private_modulus_map[public_modulus]; + } + } +} + +vector +ReverseProxyCertUtils::getFilesByExtension(const string &extension) +{ + auto maybe_files = NGEN::Filesystem::getDirectoryFiles(certs_path); + if (!maybe_files.ok()) return {}; + + auto files = maybe_files.unpack(); + files.erase( + remove_if( + files.begin(), + files.end(), + [&](const string& file) { return file.length() < 4 || file.substr(file.length() - 4) != extension; } + ), + files.end() + ); + + for (const auto &file : files) { + dbgTrace(D_LOCAL_POLICY) << "Found file: " << file; + } + + return files; +} + +pair +ReverseProxyCertUtils::findMatchingCertificate(const string &host) +{ + dbgFlow(D_LOCAL_POLICY) << "Looking for a matching certificate to host: " << host; + + for (const auto &entry : cert_key_map) { + string cert_path = entry.first; + + dbgTrace(D_LOCAL_POLICY) << "Checking match of certificate: " << cert_path; + + // Create a BIO object to read the certificate + BIO* cert_bio = BIO_new_file(cert_path.c_str(), "rb"); + if (!cert_bio) { + dbgWarning(D_LOCAL_POLICY) << "Could not open certificate file: " << cert_path; + continue; + } + + // Load the PEM-encoded public key from the file + X509 *cert = PEM_read_bio_X509(cert_bio, nullptr, nullptr, nullptr); + if (!cert) { + dbgWarning(D_LOCAL_POLICY) << "Could not parse X509 certificate file: " << cert_path; + BIO_free(cert_bio); + continue; + } + + // Get the subject alternative name extension + STACK_OF(GENERAL_NAME)* san_names = static_cast( + X509_get_ext_d2i(cert, NID_subject_alt_name, nullptr, nullptr) + ); + + if (!san_names) { + dbgWarning(D_LOCAL_POLICY) << "No Subject Alternative Name found in the certificate: " << cert_path; + X509_free(cert); + BIO_free(cert_bio); + continue; + } + + // Iterate through the SAN entries + for (int i = 0; i < sk_GENERAL_NAME_num(san_names); ++i) { + GENERAL_NAME* name = sk_GENERAL_NAME_value(san_names, i); + if (name->type == GEN_DNS) { + const char* san = reinterpret_cast(ASN1_STRING_get0_data(name->d.dNSName)); + + if (X509_check_host(cert, host.c_str(), host.length(), 0, nullptr) == 1) { + dbgTrace(D_LOCAL_POLICY) << "Found matching certificate: " << cert_path << ", DNS name: " << san; + sk_GENERAL_NAME_pop_free(san_names, GENERAL_NAME_free); + X509_free(cert); + BIO_free(cert_bio); + return {cert_path, cert_key_map[cert_path]}; + } + } + } + + dbgTrace(D_LOCAL_POLICY) << "Certificate: " << cert_path << " does not match host: " << host; + + // Clean up + sk_GENERAL_NAME_pop_free(san_names, GENERAL_NAME_free); + X509_free(cert); + BIO_free(cert_bio); + } + + return {}; +} + +Maybe +ReverseProxyCertUtils::extractModulus(const string &path, const string &type) +{ + dbgFlow(D_LOCAL_POLICY) << "Started calculating modulus of: " << path << ", type: " << type; + + string modulus_cmd = "openssl " + type + " -noout -modulus -in " + path + "; echo $?"; + auto modulus_maybe = Singleton::Consume::by()->getExecOutput(modulus_cmd); + if (!modulus_maybe.ok()) return genError("Could not complete command, error: " + modulus_maybe.getErr()); + + auto modulus_cmd_output = NGEN::Strings::removeTrailingWhitespaces(modulus_maybe.unpack()); + if (modulus_cmd_output.back() != '0') return genError("Could not extract modulus, error: " + modulus_cmd_output); + + modulus_cmd_output.pop_back(); + + dbgTrace(D_LOCAL_POLICY) << "Extracted modulus for: " << path << ", " << modulus_cmd_output; + + return modulus_cmd_output; +} + +unordered_map +ReverseProxyCertUtils::calculatePublicModulus(const vector &certs) +{ + dbgFlow(D_LOCAL_POLICY) << "Calculating certificates modulus"; + + unordered_map certs_modulus; + for (const string &cert_file_name : certs) { + string cert_path = certs_path + cert_file_name; + auto modulus = extractModulus(cert_path, "x509"); + if (!modulus.ok()) { + dbgWarning(D_LOCAL_POLICY) << modulus.getErr(); + continue; + } + + certs_modulus[cert_path] = modulus.unpack(); + } + + return certs_modulus; +} + +unordered_map +ReverseProxyCertUtils::calculatePrivateModulus(const vector &keys) +{ + unordered_map key_modulus; + for (const string &private_key_file_name : keys) { + string private_key_path = certs_path + private_key_file_name; + auto modulus = extractModulus(private_key_path, "rsa"); + if (!modulus.ok()) { + dbgWarning(D_LOCAL_POLICY) << modulus.getErr(); + continue; + } + + key_modulus[modulus.unpack()] = private_key_path; + } + + return key_modulus; +} + +void +ReverseProxyCertUtils::untarCertificatesPackages() +{ + vector cert_pkgs = getFilesByExtension(".pkg"); + if (cert_pkgs.empty()) return; + + for (const auto &cert_pkg : cert_pkgs) { + dbgTrace(D_LOCAL_POLICY) << "Untaring certificate package: " << cert_pkg; + string untar_cmd = "tar -C " + certs_path + " -xvf " + certs_path + cert_pkg; + auto maybe_tar_res = Singleton::Consume::by()->getExecOutput(untar_cmd); + if (!maybe_tar_res.ok()) { + dbgWarning(D_LOCAL_POLICY) << "Untar package error: " << maybe_tar_res.getErr(); + } + } +} + +string +ReverseProxyBuilder::replaceTemplate( + const string &content, + const boost::regex &nginx_directive_template, + const string &value) +{ + return NGEN::Regex::regexReplace(__FILE__, __LINE__, content, nginx_directive_template, value); +} + +Maybe +ReverseProxyBuilder::getTemplateContent(const string &nginx_conf_template) +{ + ifstream nginx_template_in(nginx_templates_path + nginx_conf_template); + if (!nginx_template_in.is_open()) return genError("Could not open the " + nginx_conf_template + " template"); + + string file_content((istreambuf_iterator(nginx_template_in)), istreambuf_iterator()); + nginx_template_in.close(); + + return file_content; +} + +Maybe +ReverseProxyBuilder::createSSLNginxServer(const string &host, const RPMSettings &rp_settings) +{ + dbgTrace(D_LOCAL_POLICY) << "Creating SSL NGINX server: " << host; + + pair cert_key = ReverseProxyCertUtils::findMatchingCertificate(host); + if (cert_key.first.empty() || cert_key.second.empty()) { + return genError("Cannot find matching certificates to host: " + host); + } + + auto maybe_server_content = getTemplateContent(nginx_ssl_server_template); + if (!maybe_server_content.ok()) return maybe_server_content.passErr(); + + string server_content = replaceTemplate(maybe_server_content.unpack(), host_template, host); + server_content = replaceTemplate(server_content, private_key_template, cert_key.second); + server_content = replaceTemplate(server_content, certificate_template, cert_key.first); + server_content = rp_settings.applySettings(server_content); + + dbgTrace(D_LOCAL_POLICY) << "NGINX SSL Server content: " << server_content; + + string conf_path = conf_base_path + nginx_configuration_path + "/443_" + host + ".conf"; + ofstream server_file(conf_path, ofstream::out | ofstream::trunc); + if (!server_file.is_open()) { + return genError("Could not open the output SSL NGINX configuration file: " + conf_path); + } + + server_file << server_content; + server_file.close(); + + return {}; +} + +Maybe +ReverseProxyBuilder::createHTTPNginxServer(const string &host, const RPMSettings &rp_settings) +{ + dbgFlow(D_LOCAL_POLICY) << "Creating HTTP NGINX server: " << host; + + auto maybe_server_content = getTemplateContent(nginx_http_server_template); + if (!maybe_server_content.ok()) return maybe_server_content.passErr(); + + string server_content = replaceTemplate(maybe_server_content.unpack(), host_template, host); + server_content = rp_settings.applySettings(server_content); + + dbgTrace(D_LOCAL_POLICY) << "NGINX HTTP Server content: " << server_content; + + string http_server_conf_path = conf_base_path + nginx_configuration_path + "80_" + host + ".conf"; + ofstream server_file(http_server_conf_path, ofstream::out | ofstream::trunc); + if (!server_file.is_open()) { + return genError("Could not open the output HTTP NGINX configuration file: " + http_server_conf_path); + } + + server_file << server_content; + server_file.close(); + + return {}; +} + +Maybe +ReverseProxyBuilder::addNginxServerLocation( + string location, + const string &host, + const ParsedRule &rule, + const RPMSettings &rp_settings) +{ + string port = rule.rpmIsHttps() ? string("443") : string("80"); + string location_conf_path = conf_base_path + nginx_configuration_path + port + '_' + host + "_locations/"; + + dbgFlow(D_LOCAL_POLICY) << "Adding a new NGINX location: " << location << " to: " << location_conf_path; + + NGEN::Filesystem::makeDirRecursive(location_conf_path); + + if (location.empty() || location.find_first_not_of('/') == string::npos) + { + location = "/"; + location_conf_path += "root_location.conf"; + } + else + { + string location_conf_basename = location.substr(1, location.length() - 1) + "_location"; + replace(location_conf_basename.begin(), location_conf_basename.end(), '/', '_'); + location_conf_path += location_conf_basename + ".conf"; + } + auto maybe_location_content = getTemplateContent(nginx_location_template); + if (!maybe_location_content.ok()) return maybe_location_content.passErr(); + + string location_content = replaceTemplate(maybe_location_content.unpack(), location_template, location); + location_content = replaceTemplate(location_content, upstream_template, rule.rpmGetUpstream()); + location_content = rp_settings.applySettings(location_content); + + dbgTrace(D_LOCAL_POLICY) << "NGINX server location content: " << location_content; + + ofstream location_file(location_conf_path, ofstream::out | ofstream::trunc); + if (!location_file.is_open()) { + return genError("Could not open the output NGINX location block: " + location_conf_path); + } + + location_file << location_content; + location_file.close(); + + return {}; +} + +Maybe +ReverseProxyBuilder::createNewNginxServer(const string &host, const ParsedRule &rule, const RPMSettings &rp_settings) +{ + dbgFlow(D_LOCAL_POLICY) << "Creating a new NGINX server: " << host << ", SSL: " << rule.rpmIsHttps(); + + if (rule.rpmIsHttps()) { + auto maybe_res = ReverseProxyBuilder::createSSLNginxServer(host, rp_settings); + if (!maybe_res.ok()) { + return genError("Could not create an SSL NGINX server configuration: " + maybe_res.getErr()); + } + } else { + auto maybe_res = ReverseProxyBuilder::createHTTPNginxServer(host, rp_settings); + if (!maybe_res.ok()) { + return genError("Could not create an HTTP NGINX server: " + maybe_res.getErr()); + } + } + + return {}; +} + +Maybe +ReverseProxyBuilder::reloadNginx() +{ + dbgFlow(D_LOCAL_POLICY) << "Reloading NGINX..."; + + auto maybe_nginx_t = Singleton::Consume::by()->getExecOutput( + "nginx -t 2>&1; echo $?" + ); + + if (!maybe_nginx_t.ok()){ + return genError("Could not check NGINX configuration: " + maybe_nginx_t.getErr()); + } + + string nginx_t_output = NGEN::Strings::removeTrailingWhitespaces(maybe_nginx_t.unpack()); + if (nginx_t_output.back() != '0') return genError("Invalid NGINX configuration: " + nginx_t_output); + + auto maybe_nginx_reload = Singleton::Consume::by()->getExecOutput( + "nginx -s reload 2>&1;" + ); + + if (!maybe_nginx_reload.ok()){ + return genError("Could not reload NGINX: " + maybe_nginx_reload.getErr()); + } + + return {}; +} + +void +ReverseProxyBuilder::init() +{ + conf_base_path = getConfigurationWithDefault("/etc/cp/conf/", "Config Component", "configuration path"); + nginx_templates_path = getProfileAgentSettingWithDefault( + "/etc/nginx/nginx-templates/", "openappsec.reverseProxy.nginxTemplates" + ); + + NGEN::Filesystem::deleteDirectory(conf_base_path + nginx_configuration_path, true); + NGEN::Filesystem::makeDir(conf_base_path + nginx_configuration_path); + ReverseProxyCertUtils::init(); +} diff --git a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc index 1ffe81d..699bca8 100755 --- a/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc +++ b/components/security_apps/local_policy_mgmt_gen/rules_config_section.cc @@ -156,6 +156,7 @@ RulesTriggerSection::save(cereal::JSONOutputArchive &out_ar) const RulesConfigRulebase::RulesConfigRulebase( const string &_name, const string &_url, + const string &_port, const string &_uri, vector _practices, vector _parameters, @@ -169,39 +170,19 @@ RulesConfigRulebase::RulesConfigRulebase( try { bool any = _name == "Any" && _url == "Any" && _uri == "Any"; id = any ? "Any" : _url+_uri; - if (_uri != "/") { - context = any ? "All()" : "Any(" - "All(" - "Any(" - "EqualHost(" + _url + ")" - ")," - "EqualListeningPort(80)" + - string(_uri.empty() ? "" : ",BeginWithUri(" + _uri + ")") + - ")," - "All(" - "Any(" - "EqualHost(" + _url + ")" - ")," - "EqualListeningPort(443)" + - string(_uri.empty() ? "" : ",BeginWithUri(" + _uri + ")") + - ")" - ")"; - } else { - context = any ? "All()" : "Any(" - "All(" - "Any(" - "EqualHost(" + _url + ")" - ")," - "EqualListeningPort(80)" - ")," - "All(" - "Any(" - "EqualHost(" + _url + ")" - ")," - "EqualListeningPort(443)" - ")" - ")"; + if (any) { + context ="All()"; + return; } + string host_check = "Any(EqualHost(" + _url + ")),"; + string uri_check = (_uri.empty() || _uri == "/" ) ? "" : ",BeginWithUri(" + _uri + ")"; + auto ports = _port.empty() ? vector({"80", "443"}) : vector({_port}); + context = "Any("; + for (auto &port : ports) { + string check_last = (ports.size() == 1 || port == "443") ? ")" : "),"; + context += "All(" + host_check + "EqualListeningPort(" + port + ")" + uri_check + check_last; + } + context += ")"; } catch (const boost::uuids::entropy_error &e) { dbgWarning(D_LOCAL_POLICY) << "Failed to generate rule UUID. Error: " << e.what(); } @@ -284,6 +265,7 @@ UsersIdentifiersRulebase::UsersIdentifiersRulebase( const string & UsersIdentifiersRulebase::getIdentifier() const { + if (source_identifiers.empty()) return source_identifier; return source_identifiers[0].getIdentifier(); } // LCOV_EXCL_STOP diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index 5a36ff8..bb1b700 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -45,11 +45,7 @@ checkSamlPortal(const string &command_output) Maybe getIDAGaia(const string &command_output) { - if (command_output.find("Portal is running") != string::npos) { - return string("ida_gaia"); - } - - return genError("Current host does not have SAML Portal configured"); + return string("ida_gaia"); } Maybe @@ -72,6 +68,22 @@ checkIDP(shared_ptr file_stream) #if defined(gaia) || defined(smb) +Maybe +checkIsCpviewRunning(const string &command_output) +{ + if (command_output == "true" || command_output == "false") return command_output; + + return genError("cpview is not running"); +} + +Maybe +checkIsCPotelcolGRET64(const string &command_output) +{ + if (command_output == "true" || command_output == "false") return command_output; + + return genError("CPotelcol is not installed or its take is below T64"); +} + Maybe checkHasSDWan(const string &command_output) { @@ -193,6 +205,12 @@ checkIfSdwanRunning(const string &command_output) return genError("Could not determine if sd-wan is running or not"); } +Maybe +getClusterObjectIP(const string &command_output) +{ + return getAttr(command_output, "Cluster object IP was not found"); +} + Maybe getSmbObjectName(const string &command_output) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 7beed3d..f38dac1 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -30,6 +30,14 @@ #ifdef SHELL_CMD_HANDLER #if defined(gaia) || defined(smb) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) +SHELL_CMD_HANDLER("isCpviewRunning", + "pidof cpview_api_service > /dev/null 2>&1 && [ -f $CPDIR/conf/cpview_api_service.version ] " + "&& echo 'true' || echo 'false'", + checkIsCpviewRunning) +SHELL_CMD_HANDLER("isCPotelcolGRET64", + "grep -A 10 '(BUNDLE_CPOTELCOL_AUTOUPDATE' ${CPDIR}/registry/HKLM_registry.data | " + "awk '/SU_Build_Take/{val = substr($2, 2, length($2)-2); if (val >=64) print \"true\"; else print \"false\" }'", + checkIsCPotelcolGRET64) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( "canUpdateSDWanData", @@ -50,12 +58,20 @@ SHELL_CMD_HANDLER( "cat /etc/cp-release | grep -oE 'R[0-9]+(\\.[0-9]+)?'", getGWVersion ) +SHELL_CMD_HANDLER( + "cpProductIntegrationMgmtParentObjectIP", + "obj=\"$(cpsdwan get_data | jq -r .cluster_name)\";" + " awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")" + " { gsub(/[()]/, \"\", $2); print $2; exit; } }'" + " $FWDIR/state/local/FW1/local.gateway_cluster", + getClusterObjectIP +) #endif //gaia || smb #if defined(gaia) SHELL_CMD_HANDLER("hasSupportedBlade", "enabled_blades", checkHasSupportedBlade) SHELL_CMD_HANDLER("hasSamlPortal", "mpclient status saml-vpn", checkSamlPortal) -SHELL_CMD_HANDLER("requiredNanoServices", "mpclient status saml-vpn", getIDAGaia) +SHELL_CMD_HANDLER("requiredNanoServices", "ida_gaia", getIDAGaia) SHELL_CMD_HANDLER( "cpProductIntegrationMgmtParentObjectName", "cat $FWDIR/database/myself_objects.C " diff --git a/components/security_apps/orchestration/downloader/curl_client.cc b/components/security_apps/orchestration/downloader/curl_client.cc index dce531a..01ec1f6 100755 --- a/components/security_apps/orchestration/downloader/curl_client.cc +++ b/components/security_apps/orchestration/downloader/curl_client.cc @@ -278,36 +278,6 @@ HttpsCurl::HttpsCurl(const HttpsCurl &other) : HttpCurl(other), ca_path(other.ca_path) {} -bool -HttpsCurl::downloadOpenAppsecPackages() -{ - char errorstr[CURL_ERROR_SIZE]; - CURL* curl_handle = curl_easy_init(); - if (!curl_handle) return false; - - curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1); - curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 2); - - curl_easy_setopt(curl_handle, CURLOPT_URL, ("https://" + curl_url).c_str()); - curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeResponseCallback); - curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &out_file); - - curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L); - curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errorstr); - - CURLcode res = curl_easy_perform(curl_handle); - if (res == CURLE_OK) { - dbgTrace(D_HTTP_REQUEST) << "CURL HTTP request successfully completed."; - } else { - dbgWarning(D_HTTP_REQUEST) << "CURL result " + string(curl_easy_strerror(res)); - curl_easy_cleanup(curl_handle); - return false; - } - - curl_easy_cleanup(curl_handle); - return true; -} - void HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version) { @@ -347,7 +317,7 @@ HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version) curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeResponseCallback); curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &out_file); curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, timeout); - curl_easy_setopt(curl_handle, CURLOPT_CAINFO, ca_path.c_str()); + if (ca_path != "") curl_easy_setopt(curl_handle, CURLOPT_CAINFO, ca_path.c_str()); headers = curl_slist_append(headers, "Accept: */*"); string auth = string("Authorization: Bearer ") + bearer; headers = curl_slist_append(headers, auth.c_str()); diff --git a/components/security_apps/orchestration/downloader/curl_client.h b/components/security_apps/orchestration/downloader/curl_client.h index fd99dd8..9104f29 100755 --- a/components/security_apps/orchestration/downloader/curl_client.h +++ b/components/security_apps/orchestration/downloader/curl_client.h @@ -105,7 +105,6 @@ public: static CURLcode ssl_ctx_verify_certificate(CURL *curl, void *ssl_ctx, void *opq); static int verify_certificate(int preverify_ok, X509_STORE_CTX *ctx); void setCurlOpts(long timeout = 60L, HTTP_VERSION http_version = HTTP_VERSION::HTTP_VERSION_1_1) override; - bool downloadOpenAppsecPackages(); private: std::string ca_path; diff --git a/components/security_apps/orchestration/downloader/https_client.cc b/components/security_apps/orchestration/downloader/https_client.cc index 8ef61a3..33d9031 100755 --- a/components/security_apps/orchestration/downloader/https_client.cc +++ b/components/security_apps/orchestration/downloader/https_client.cc @@ -592,13 +592,8 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s proxy_config->getProxyCredentials(ProxyProtocol::HTTPS), cert_file_path); - bool connection_ok; - if (url.getBaseURL().unpack() == "downloads.openappsec.io") { - connection_ok = ssl_curl_client.downloadOpenAppsecPackages(); - } else { - ssl_curl_client.setCurlOpts(); - connection_ok = ssl_curl_client.connect(); - } + ssl_curl_client.setCurlOpts(); + bool connection_ok = ssl_curl_client.connect(); if (!connection_ok) { stringstream url_s; diff --git a/components/security_apps/orchestration/health_check/health_check.cc b/components/security_apps/orchestration/health_check/health_check.cc index db81a41..6d61c7d 100755 --- a/components/security_apps/orchestration/health_check/health_check.cc +++ b/components/security_apps/orchestration/health_check/health_check.cc @@ -256,10 +256,23 @@ private: if (!getenv("DOCKER_RPM_ENABLED")) return HealthCheckStatus::IGNORED; static const string standalone_cmd = "/usr/sbin/cpnano -s --docker-rpm; echo $?"; + static int timeout_tolerance = 1; + static HealthCheckStatus health_status = HealthCheckStatus::HEALTHY; + dbgTrace(D_HEALTH_CHECK) << "Checking the standalone docker health status with command: " << standalone_cmd; - auto maybe_result = Singleton::Consume::by()->getExecOutput(standalone_cmd, 1000); + auto maybe_result = Singleton::Consume::by()->getExecOutput(standalone_cmd, 5000); if (!maybe_result.ok()) { + if (maybe_result.getErr().find("Reached timeout") != string::npos) { + dbgWarning(D_HEALTH_CHECK) + << "Reached timeout while querying standalone health status, attempt number: " + << timeout_tolerance; + + return health_status == HealthCheckStatus::UNHEALTHY || timeout_tolerance++ > 3 ? + HealthCheckStatus::UNHEALTHY : + health_status; + } + dbgWarning(D_HEALTH_CHECK) << "Unable to get the standalone docker status. Returning unhealthy status."; return HealthCheckStatus::UNHEALTHY; } @@ -267,10 +280,10 @@ private: auto response = NGEN::Strings::removeTrailingWhitespaces(maybe_result.unpack()); - if (response.back() == '0') return HealthCheckStatus::HEALTHY; - if (response.back() == '1') return HealthCheckStatus::UNHEALTHY; + if (response.back() == '1') return health_status = HealthCheckStatus::UNHEALTHY; - return HealthCheckStatus::DEGRADED; + timeout_tolerance = 1; + return health_status = (response.back() == '0') ? HealthCheckStatus::HEALTHY : HealthCheckStatus::DEGRADED; } bool diff --git a/components/security_apps/orchestration/include/declarative_policy_utils.h b/components/security_apps/orchestration/include/declarative_policy_utils.h index 461ba17..58eb01e 100644 --- a/components/security_apps/orchestration/include/declarative_policy_utils.h +++ b/components/security_apps/orchestration/include/declarative_policy_utils.h @@ -87,10 +87,12 @@ public: private: std::string getCleanChecksum(const std::string &unclean_checksum); + void periodicPolicyLoad(); std::string local_policy_path; std::string curr_version; std::string curr_policy; + std::string curr_checksum; bool should_apply_policy; }; diff --git a/components/security_apps/orchestration/include/get_status_rest.h b/components/security_apps/orchestration/include/get_status_rest.h index d1b4a88..655389f 100755 --- a/components/security_apps/orchestration/include/get_status_rest.h +++ b/components/security_apps/orchestration/include/get_status_rest.h @@ -54,7 +54,6 @@ public: last_update = i_orch_status->getUpdateTime(); last_update_status = i_orch_status->getUpdateStatus(); policy_version = i_orch_status->getPolicyVersion(); - waap_model_version = i_orch_status->getWaapModelVersion(); last_policy_update = i_orch_status->getLastPolicyUpdate(); last_manifest_update = i_orch_status->getLastManifestUpdate(); last_settings_update = i_orch_status->getLastSettingsUpdate(); @@ -73,7 +72,6 @@ private: S2C_LABEL_PARAM(std::string, last_update, "Last update"); S2C_LABEL_PARAM(std::string, last_update_status, "Last update status"); S2C_LABEL_PARAM(std::string, policy_version, "Policy version"); - S2C_LABEL_PARAM(std::string, waap_model_version, "AI model version"); S2C_LABEL_PARAM(std::string, last_policy_update, "Last policy update"); S2C_LABEL_PARAM(std::string, last_manifest_update, "Last manifest update"); S2C_LABEL_PARAM(std::string, last_settings_update, "Last settings update"); diff --git a/components/security_apps/orchestration/include/mock/mock_orchestration_status.h b/components/security_apps/orchestration/include/mock/mock_orchestration_status.h index a848973..740cfa5 100644 --- a/components/security_apps/orchestration/include/mock/mock_orchestration_status.h +++ b/components/security_apps/orchestration/include/mock/mock_orchestration_status.h @@ -45,7 +45,6 @@ public: MOCK_CONST_METHOD0(getUpdateTime, const std::string&()); MOCK_CONST_METHOD0(getLastManifestUpdate, const std::string&()); MOCK_CONST_METHOD0(getPolicyVersion, const std::string&()); - MOCK_CONST_METHOD0(getWaapModelVersion, const std::string&()); MOCK_CONST_METHOD0(getLastPolicyUpdate, const std::string&()); MOCK_CONST_METHOD0(getLastSettingsUpdate, const std::string&()); MOCK_CONST_METHOD0(getUpgradeMode, const std::string&()); diff --git a/components/security_apps/orchestration/include/mock/mock_service_controller.h b/components/security_apps/orchestration/include/mock/mock_service_controller.h index 23f8d16..e32fea9 100755 --- a/components/security_apps/orchestration/include/mock/mock_service_controller.h +++ b/components/security_apps/orchestration/include/mock/mock_service_controller.h @@ -66,7 +66,7 @@ public: typedef std::map ServicePortMap; MOCK_METHOD0(getServiceToPortMap, ServicePortMap()); - MOCK_METHOD2(updateReconfStatus, void(int id, ReconfStatus status)); + MOCK_METHOD3(updateReconfStatus, void(int id, const std::string &service_name, ReconfStatus status)); MOCK_METHOD4( startReconfStatus, void(int id, ReconfStatus status, const std::string &serivce_name, const std::string &service_id) diff --git a/components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc b/components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc index 07eb0b9..6c902ed 100755 --- a/components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc +++ b/components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc @@ -11,7 +11,6 @@ #include "mock/mock_time_get.h" #include "mock/mock_orchestration_tools.h" #include "mock/mock_agent_details.h" -#include "mock/mock_details_resolver.h" #include "mock/mock_mainloop.h" #include "mock/mock_rest_api.h" @@ -39,17 +38,9 @@ public: .WillOnce(DoAll(SaveArg<2>(&routine), Return(1)) ); EXPECT_CALL(mock_tools, readFile(file_path)).WillOnce(Return(start_file_content)); - prepareResolvedDetails(); orchestration_status.init(); } - void - prepareResolvedDetails() - { - map resolved_details({{"AppSecModelVersion", waap_model}}); - EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_details)); - } - string orchestrationStatusFileToString() { @@ -91,8 +82,7 @@ public: const string ®istration_details_architecture = "", const string &agent_id = "None", const string &profile_id = "None", - const string &tenant_id = "None", - const string &waap_model_version = "Advanced model" + const string &tenant_id = "None" ) { return "{\n" @@ -101,7 +91,6 @@ public: " \"Last update\": \"" + last_update + "\",\n" " \"Last manifest update\": \"" + last_manifest_update + "\",\n" " \"Policy version\": \"" + policy_version + "\",\n" - " \"AI model version\": \"" + waap_model_version + "\",\n" " \"Last policy update\": \"" + last_policy_update + "\",\n" " \"Last settings update\": \"" + last_settings_update + "\",\n" " \"Upgrade mode\": \"" + upgrade_mode + "\",\n" @@ -129,14 +118,12 @@ public: ostringstream capture_debug; StrictMock mock_tools; StrictMock mock_agent_details; - StrictMock mock_details_resolver; OrchestrationStatus orchestration_status; I_OrchestrationStatus * i_orchestration_status = Singleton::Consume::from(orchestration_status); string file_path; Maybe start_file_content = genError("No file"); I_MainLoop::Routine routine; - string waap_model = "Advanced model"; }; TEST_F(OrchestrationStatusTest, doNothing) @@ -160,7 +147,6 @@ TEST_F(OrchestrationStatusTest, recoverFields) TEST_F(OrchestrationStatusTest, loadFromFile) { - prepareResolvedDetails(); Maybe status = genError("No file");; CPTestTempfile status_file; file_path = status_file.fname; @@ -228,14 +214,12 @@ TEST_F(OrchestrationStatusTest, recoveryFields) const string agent_id = "AgentId"; const string profile_id = "ProfileId"; const string tenant_id = "TenantId"; - auto fog_addr = Maybe(string("FogDomain")); EXPECT_CALL(mock_agent_details, getAgentId()).WillOnce(Return(agent_id)); EXPECT_CALL(mock_agent_details, getProfileId()).WillOnce(Return(profile_id)); EXPECT_CALL(mock_agent_details, getTenantId()).WillOnce(Return(tenant_id)); EXPECT_CALL(mock_agent_details, getFogDomain()).WillOnce(Return(fog_addr)); - i_orchestration_status->writeStatusToFile(); EXPECT_THAT(capture_debug.str(), HasSubstr("Repairing status fields")); @@ -243,7 +227,6 @@ TEST_F(OrchestrationStatusTest, recoveryFields) EXPECT_EQ(i_orchestration_status->getProfileId(), profile_id); EXPECT_EQ(i_orchestration_status->getTenantId(), tenant_id); EXPECT_EQ(i_orchestration_status->getFogAddress(), fog_addr.unpack()); - EXPECT_EQ(i_orchestration_status->getWaapModelVersion(), waap_model); } TEST_F(OrchestrationStatusTest, updateAllLastUpdatesTypes) @@ -436,7 +419,6 @@ TEST_F(OrchestrationStatusTest, setAllFields) " \"Last update\": \"current time\",\n" " \"Last manifest update\": \"current time\",\n" " \"Policy version\": \"12\",\n" - " \"AI model version\": \"Advanced model\",\n" " \"Last policy update\": \"current time\",\n" " \"Last settings update\": \"current time\",\n" " \"Upgrade mode\": \"Test Mode\",\n" diff --git a/components/security_apps/orchestration/modules/orchestration_status.cc b/components/security_apps/orchestration/modules/orchestration_status.cc index 4a45a20..f2839e6 100755 --- a/components/security_apps/orchestration/modules/orchestration_status.cc +++ b/components/security_apps/orchestration/modules/orchestration_status.cc @@ -108,7 +108,6 @@ public: last_update_attempt = from.last_update_attempt; last_manifest_update = from.last_manifest_update; policy_version = from.policy_version; - waap_model_version = from.waap_model_version; last_policy_update = from.last_policy_update; last_settings_update = from.last_settings_update; upgrade_mode = from.upgrade_mode; @@ -129,7 +128,6 @@ public: const string & getUpdateTime() const { return last_update_time; } const string & getLastManifestUpdate() const { return last_manifest_update; } const string & getPolicyVersion() const { return policy_version; } - const string & getWaapModelVersion() const { return waap_model_version; } const string & getLastPolicyUpdate() const { return last_policy_update; } const string & getLastSettingsUpdate() const { return last_settings_update; } const string & getUpgradeMode() const { return upgrade_mode; } @@ -144,16 +142,6 @@ public: const map & getServicePolicies() const { return service_policies; } const map & getServiceSettings() const { return service_settings; } - void updateWaapModelVersion() { - map details_resolver = - Singleton::Consume::by()->getResolvedDetails(); - if (details_resolver.find("AppSecModelVersion") != details_resolver.end()) { - waap_model_version = details_resolver["AppSecModelVersion"]; - } else { - waap_model_version = "None"; - } - } - void insertServicePolicy(const string &key, const string &value) { @@ -279,7 +267,6 @@ public: last_manifest_update = "None"; last_policy_update = "None"; last_settings_update = "None"; - waap_model_version = "None"; fog_address = "None"; agent_id = "None"; profile_id = "None"; @@ -305,7 +292,6 @@ public: } else { fog_address = "None"; } - updateWaapModelVersion(); } } @@ -318,7 +304,6 @@ public: archive(cereal::make_nvp("Last update", last_update_time)); archive(cereal::make_nvp("Last manifest update", last_manifest_update)); archive(cereal::make_nvp("Policy version", policy_version)); - archive(cereal::make_nvp("AI model version", waap_model_version)); archive(cereal::make_nvp("Last policy update", last_policy_update)); archive(cereal::make_nvp("Last settings update", last_settings_update)); archive(cereal::make_nvp("Upgrade mode", upgrade_mode)); @@ -346,7 +331,6 @@ public: archive.setNextName(nullptr); } - archive(cereal::make_nvp("AI model version", waap_model_version)); archive(cereal::make_nvp("Last policy update", last_policy_update)); archive(cereal::make_nvp("Last settings update", last_settings_update)); @@ -384,7 +368,6 @@ private: string last_update_attempt; string last_manifest_update; string policy_version; - string waap_model_version; string last_policy_update; string last_settings_update; string upgrade_mode; @@ -411,7 +394,6 @@ public: "orchestration", "Orchestration status path" ); - status.updateWaapModelVersion(); auto write_result = orchestration_tools->objectToJsonFile(status, orchestration_status_path); if (!write_result) { @@ -515,7 +497,6 @@ private: const string & getUpdateTime() const override { return status.getUpdateTime(); } const string & getLastManifestUpdate() const override { return status.getLastManifestUpdate(); } const string & getPolicyVersion() const override { return status.getPolicyVersion(); } - const string & getWaapModelVersion() const override { return status.getWaapModelVersion(); } const string & getLastPolicyUpdate() const override { return status.getLastPolicyUpdate(); } const string & getLastSettingsUpdate() const override { return status.getLastSettingsUpdate(); } const string & getUpgradeMode() const override { return status.getUpgradeMode(); } diff --git a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc index 88de151..9f073a6 100755 --- a/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc +++ b/components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc @@ -1797,7 +1797,6 @@ TEST_F(OrchestrationTest, GetRestOrchStatus) " \"Last update\": \"" + test_str + "\",\n" " \"Last update status\": \"" + test_str + "\",\n" " \"Policy version\": \"" + test_str + "\",\n" - " \"AI model version\": \"" + test_str + "\",\n" " \"Last policy update\": \"" + test_str + "\",\n" " \"Last manifest update\": \"" + test_str + "\",\n" " \"Last settings update\": \"" + test_str + "\",\n" @@ -1824,7 +1823,6 @@ TEST_F(OrchestrationTest, GetRestOrchStatus) EXPECT_CALL(mock_status, getUpdateTime()).WillOnce(ReturnRef(test_str)); EXPECT_CALL(mock_status, getLastManifestUpdate()).WillOnce(ReturnRef(test_str)); EXPECT_CALL(mock_status, getPolicyVersion()).WillOnce(ReturnRef(test_str)); - EXPECT_CALL(mock_status, getWaapModelVersion()).WillOnce(ReturnRef(test_str)); EXPECT_CALL(mock_status, getLastPolicyUpdate()).WillOnce(ReturnRef(test_str)); EXPECT_CALL(mock_status, getLastSettingsUpdate()).WillOnce(ReturnRef(test_str)); EXPECT_CALL(mock_status, getUpgradeMode()).WillOnce(ReturnRef(test_str)); diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 73828c2..6f7e925 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -53,32 +53,37 @@ public: { auto service_controller = Singleton::Consume::by(); if (!finished.get()) { - service_controller->updateReconfStatus(id.get(), ReconfStatus::IN_PROGRESS); + service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::IN_PROGRESS); dbgTrace(D_ORCHESTRATOR) - << "Request for service reconfiguration, with id " + << "Request for service reconfiguration is still in progress. ID: " << id.get() - << ", is still in progress."; + << ", Service Name: " + << service_name.get(); return; } if (error.get()) { - service_controller->updateReconfStatus(id.get(), ReconfStatus::FAILED); + service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::FAILED); dbgError(D_ORCHESTRATOR) - << "Request for service reconfiguration, with id " + << "Request for service reconfiguration failed to complete. ID: " << id.get() - << ", failed to complete." + << ", Service Name: " + << service_name.get() + << "." << (error_message.isActive() ? " Error: " + error_message.get() : ""); return; } - service_controller->updateReconfStatus(id.get(), ReconfStatus::SUCCEEDED); + service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::SUCCEEDED); dbgInfo(D_ORCHESTRATOR) - << "Request for service reconfiguration, with id " + << "Request for service reconfiguration successfully accomplished. Reconf ID: " << id.get() - << ", successfully accomplished."; + << ", Service Name: " + << service_name.get(); return; } private: C2S_PARAM(int, id); + C2S_PARAM(string, service_name); C2S_PARAM(bool, error); C2S_PARAM(bool, finished); C2S_OPTIONAL_PARAM(string, error_message); @@ -292,7 +297,7 @@ public: const string & getPolicyVersion() const override; const string & getUpdatePolicyVersion() const override; const string & getPolicyVersions() const override; - void updateReconfStatus(int id, ReconfStatus status) override; + void updateReconfStatus(int id, const string &service_name, ReconfStatus status) override; void startReconfStatus( int id, ReconfStatus status, @@ -780,6 +785,7 @@ ServiceController::Impl::updateServiceConfiguration( string version_value; string send_signal_for_services_err; + changed_policy_files.clear(); for (auto &single_policy : all_security_policies.unpack()) { if (single_policy.first == version_param) { version_value = single_policy.second; @@ -1076,19 +1082,25 @@ ServiceController::Impl::getUpdatePolicyVersion() const } void -ServiceController::Impl::updateReconfStatus(int id, ReconfStatus status) +ServiceController::Impl::updateReconfStatus(int id, const string &service_name, ReconfStatus status) { if (status == ReconfStatus::FAILED) { failed_services.emplace(id, status); } if (services_reconf_status.find(id) == services_reconf_status.end()) { - dbgError(D_ORCHESTRATOR) << "Service reconfiguration monitor received illegal id :" << id; + dbgError(D_ORCHESTRATOR) + << "Unable to find a mapping for reconfiguration ID:" + << id + << ". Service name: " + << service_name; return; } dbgTrace(D_ORCHESTRATOR) << "Updating reconf status for reconfiguration ID " << id + << ", Service name: " + << service_name << ". Status: " << static_cast(status); services_reconf_status[id] = status; diff --git a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc index 522ea39..c9c6693 100755 --- a/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc +++ b/components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc @@ -1928,6 +1928,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf) reconf_status << "{" << " \"id\": 1," + << " \"service_name\": \"max\"," << " \"finished\": true," << " \"error\": false," << " \"error_message\": \"\"" diff --git a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc index 52fcd44..e00604a 100755 --- a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc +++ b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc @@ -22,6 +22,16 @@ DeclarativePolicyUtils::init() RestAction::SET, "apply-policy" ); registerListener(); + char *automatic_load = getenv("autoPolicyLoad"); + if (automatic_load != nullptr && automatic_load == string("true")) { + auto mainloop = Singleton::Consume::by(); + mainloop->addRecurringRoutine( + I_MainLoop::RoutineType::Offline, + chrono::minutes(1), + [&] () { periodicPolicyLoad(); }, + "Automatic Policy Loading" + ); + } } // LCOV_EXCL_START Reason: no test exist @@ -170,3 +180,19 @@ DeclarativePolicyUtils::getUpdate(CheckUpdateRequest &request) curr_version = maybe_new_version.unpack(); return policy_response; } + +void +DeclarativePolicyUtils::periodicPolicyLoad() +{ + auto new_checksum = getLocalPolicyChecksum(); + + if (!new_checksum.ok()) { + dbgWarning(D_ORCHESTRATOR) << "Failed to calculate checksum"; + return; + } + + if (*new_checksum == curr_checksum) return; + + should_apply_policy = true; + curr_checksum = *new_checksum; +} diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index 997a50e..5688df8 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -13,6 +13,7 @@ #include "http_inspection_events.h" #include "Waf2Util.h" #include "generic_rulebase/evaluators/asset_eval.h" +#include "generic_rulebase/parameters_config.h" #include "WaapConfigApi.h" #include "WaapConfigApplication.h" #include "PatternMatcher.h" @@ -49,13 +50,13 @@ public: Maybe extractUri(const string &address) { - size_t protocolPos = address.find("://"); - if (protocolPos == std::string::npos) return genError("Invalid URI format: " + address); + size_t protocol_pos = address.find("://"); + if (protocol_pos == string::npos) return genError("Invalid URI format: " + address); - size_t domainPos = address.find('/', protocolPos + 3); - if (domainPos == std::string::npos) return string(""); + size_t domain_pos = address.find('/', protocol_pos + 3); + if (domain_pos == string::npos) return string(""); - return address.substr(domainPos); + return address.substr(domain_pos); } bool @@ -179,15 +180,35 @@ public: { if (!event.isLastHeader()) return INSPECT; - auto uri_ctx = Singleton::Consume::by()->get(HttpTransactionData::uri_ctx); + auto env = Singleton::Consume::by(); + auto uri_ctx = env->get(HttpTransactionData::uri_ctx); if (!uri_ctx.ok()) { dbgWarning(D_RATE_LIMIT) << "Unable to get URL from context, Not enforcing rate limit"; return ACCEPT; } - string asset_id; auto uri = uri_ctx.unpack(); transform(uri.begin(), uri.end(), uri.begin(), [](unsigned char c) { return tolower(c); }); + + auto maybe_source_identifier = env->get(HttpTransactionData::source_identifier); + if (!maybe_source_identifier.ok()) { + dbgWarning(D_RATE_LIMIT) << "Unable to get source identifier from context, not enforcing rate limit"; + return ACCEPT; + } + + auto &source_identifier = maybe_source_identifier.unpack(); + dbgDebug(D_RATE_LIMIT) << "source identifier value: " << source_identifier; + + auto maybe_source_ip = env->get(HttpTransactionData::client_ip_ctx); + string source_ip = ""; + if (maybe_source_ip.ok()) source_ip = ipAddrToStr(maybe_source_ip.unpack()); + + if (shouldApplyException(uri, source_identifier, source_ip)) { + dbgDebug(D_RATE_LIMIT) << "found accept exception, not enforcing rate limit on this uri: " << uri; + return ACCEPT; + } + + string asset_id; auto maybe_rule = findRateLimitRule(uri, asset_id); if (!maybe_rule.ok()) { dbgDebug(D_RATE_LIMIT) << "Not Enforcing Rate Limit: " << maybe_rule.getErr(); @@ -205,16 +226,6 @@ public: << (rule.getRateLimitScope() == "Minute" ? 60 : 1) << " seconds"; - auto maybe_source_identifier = - Singleton::Consume::by()->get(HttpTransactionData::source_identifier); - if (!maybe_source_identifier.ok()) { - dbgWarning(D_RATE_LIMIT) << "Unable to get source identifier from context, not enforcing rate limit"; - return ACCEPT; - } - - auto &source_identifier = maybe_source_identifier.unpack(); - dbgDebug(D_RATE_LIMIT) << "source identifier value: " << source_identifier; - string unique_key = asset_id + ":" + source_identifier + ":" + uri; if (unique_key.back() == '/') unique_key.pop_back(); @@ -224,7 +235,7 @@ public: return ACCEPT; } - if (verdict == RateLimitVedict::DROP_AND_LOG) sendLog(uri, source_identifier, rule); + if (verdict == RateLimitVedict::DROP_AND_LOG) sendLog(uri, source_identifier, source_ip, rule); if (mode == "Active") { dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit"; @@ -242,7 +253,7 @@ public: } RateLimitVedict - decide(const std::string &key) { + decide(const string &key) { if (redis == nullptr) { dbgDebug(D_RATE_LIMIT) << "there is no connection to the redis at the moment, unable to enforce rate limit"; @@ -288,7 +299,7 @@ public: } void - sendLog(const string &uri, const string &source_identifier, const RateLimitRule& rule) + sendLog(const string &uri, const string &source_identifier, const string &source_ip, const RateLimitRule &rule) { set rate_limit_triggers_set; for (const auto &trigger : rule.getRateLimitTriggers()) { @@ -296,7 +307,7 @@ public: } ScopedContext ctx; - ctx.registerValue>(TriggerMatcher::ctx_key, rate_limit_triggers_set); + ctx.registerValue>(TriggerMatcher::ctx_key, rate_limit_triggers_set); auto log_trigger = getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log"); if (!log_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) { @@ -336,25 +347,44 @@ public: << LogField("securityAction", (mode == "Active" ? "Prevent" : "Detect")) << LogField("waapIncidentType", "Rate Limit"); - auto http_method = - Singleton::Consume::by()->get(HttpTransactionData::method_ctx); + auto env = Singleton::Consume::by(); + auto http_method = env->get(HttpTransactionData::method_ctx); if (http_method.ok()) log << LogField("httpMethod", http_method.unpack()); - auto http_host = - Singleton::Consume::by()->get(HttpTransactionData::host_name_ctx); + auto http_host = env->get(HttpTransactionData::host_name_ctx); if (http_host.ok()) log << LogField("httpHostName", http_host.unpack()); - auto source_ip = - Singleton::Consume::by()->get(HttpTransactionData::client_ip_ctx); - if (source_ip.ok()) log << LogField("sourceIP", ipAddrToStr(source_ip.unpack())); + if (!source_ip.empty()) log << LogField("sourceIP", source_ip); - auto proxy_ip = - Singleton::Consume::by()->get(HttpTransactionData::proxy_ip_ctx); - if (proxy_ip.ok() && source_ip.ok() && ipAddrToStr(source_ip.unpack()) != proxy_ip.unpack()) { - log << LogField("proxyIP", static_cast(proxy_ip.unpack())); + auto proxy_ip = env->get(HttpTransactionData::proxy_ip_ctx); + if (proxy_ip.ok() && !source_ip.empty() && source_ip != proxy_ip.unpack()) { + log << LogField("proxyIP", static_cast(proxy_ip.unpack())); } } + bool + shouldApplyException(const string &uri, const string &source_identifier, const string &source_ip) + { + dbgTrace(D_RATE_LIMIT) << "matching exceptions"; + unordered_map> exceptions_dict; + + // collect sourceip, sourceIdentifier, url + if (!source_ip.empty()) exceptions_dict["sourceIP"].insert(source_ip); + exceptions_dict["sourceIdentifier"].insert(source_identifier); + exceptions_dict["url"].insert(uri); + + auto behaviors = Singleton::Consume::by()->getBehavior(exceptions_dict); + for (auto const &behavior : behaviors) { + if (behavior == action_accept) { + dbgTrace(D_RATE_LIMIT) << "matched exceptions for " << uri << " should accept"; + return true; + } + } + + dbgTrace(D_RATE_LIMIT) << "No accept exceptions found for this uri and source ip"; + return false; + } + string ipAddrToStr(const IPAddr& ip_address) const { @@ -368,14 +398,21 @@ public: { disconnectRedis(); - const string &redis_ip = getConfigurationWithDefault("127.0.0.1", "connection", "Redis IP"); - int redis_port = getConfigurationWithDefault(6379, "connection", "Redis Port"); + redisOptions options; + memset(&options, 0, sizeof(redisOptions)); + REDIS_OPTIONS_SET_TCP( + &options, + "127.0.0.1", + getConfigurationWithDefault(6379, "connection", "Redis Port") + ); timeval timeout; timeout.tv_sec = 0; timeout.tv_usec = getConfigurationWithDefault(30000, "connection", "Redis Timeout"); + options.connect_timeout = &timeout; + options.command_timeout = &timeout; - redisContext* context = redisConnectWithTimeout(redis_ip.c_str(), redis_port, timeout); + redisContext* context = redisConnectWithOptions(&options); if (context != nullptr && context->err) { dbgDebug(D_RATE_LIMIT) << "Error connecting to Redis: " diff --git a/components/security_apps/waap/include/i_transaction.h b/components/security_apps/waap/include/i_transaction.h index 6f731bc..1af9157 100755 --- a/components/security_apps/waap/include/i_transaction.h +++ b/components/security_apps/waap/include/i_transaction.h @@ -126,7 +126,6 @@ public: virtual void add_request_body_chunk(const char* data, int data_len) = 0; virtual void end_request_body() = 0; virtual void end_request() = 0; - // Response virtual void start_response(int response_status, int http_version) = 0; virtual void start_response_hdrs() = 0; diff --git a/components/security_apps/waap/include/i_waapConfig.h b/components/security_apps/waap/include/i_waapConfig.h index b8a1b4f..5d1340b 100755 --- a/components/security_apps/waap/include/i_waapConfig.h +++ b/components/security_apps/waap/include/i_waapConfig.h @@ -23,7 +23,6 @@ #include "../waap_clib/UserLimitsPolicy.h" #include "../waap_clib/RateLimiting.h" #include "../waap_clib/SecurityHeadersPolicy.h" - #include enum class BlockingLevel { diff --git a/components/security_apps/waap/waap_clib/CMakeLists.txt b/components/security_apps/waap/waap_clib/CMakeLists.txt index 93d7132..3e363eb 100755 --- a/components/security_apps/waap/waap_clib/CMakeLists.txt +++ b/components/security_apps/waap/waap_clib/CMakeLists.txt @@ -84,6 +84,7 @@ add_library(waap_clib WaapSampleValue.cc ParserGql.cc ParserPercentEncode.cc + ParserPairs.cc ) add_definitions("-Wno-unused-function") diff --git a/components/security_apps/waap/waap_clib/ContentTypeParser.cc b/components/security_apps/waap/waap_clib/ContentTypeParser.cc index 9c8fba3..fbacc0d 100755 --- a/components/security_apps/waap/waap_clib/ContentTypeParser.cc +++ b/components/security_apps/waap/waap_clib/ContentTypeParser.cc @@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_CONTENT_TYPE); const std::string ContentTypeParser::m_parserName = "contentTypeParser"; -int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) +int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) { dbgTrace(D_WAAP_PARSER_CONTENT_TYPE) << "ContentTypeParser::onKv(): " << std::string(v, v_len); assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH); @@ -42,10 +42,12 @@ int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v return 0; // ok } -ContentTypeParser::ContentTypeParser() -:ctParserState(CTP_STATE_CONTENT_TYPE), m_rcvr(*this), m_hvp(m_rcvr), m_error(false) -{ -} +ContentTypeParser::ContentTypeParser() : + ctParserState(CTP_STATE_CONTENT_TYPE), + m_rcvr(*this), + m_hvp(m_rcvr), + m_error(false) +{} size_t ContentTypeParser::push(const char *data, size_t data_len) { diff --git a/components/security_apps/waap/waap_clib/ContentTypeParser.h b/components/security_apps/waap/waap_clib/ContentTypeParser.h index b500cc2..02a27d0 100755 --- a/components/security_apps/waap/waap_clib/ContentTypeParser.h +++ b/components/security_apps/waap/waap_clib/ContentTypeParser.h @@ -25,7 +25,7 @@ class ContentTypeParser : public ParserBase, private IParserReceiver { CTP_STATE_CONTENT_TYPE_PARAMS } ctParserState; private: - virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags); + virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth); public: ContentTypeParser(); diff --git a/components/security_apps/waap/waap_clib/DeepAnalyzer.cc b/components/security_apps/waap/waap_clib/DeepAnalyzer.cc index b63175f..8584220 100755 --- a/components/security_apps/waap/waap_clib/DeepAnalyzer.cc +++ b/components/security_apps/waap/waap_clib/DeepAnalyzer.cc @@ -107,7 +107,7 @@ AnalysisResult DeepAnalyzer::Impl::analyzeData(const D2InputData& data, const IW (shouldExcept ? "true" : "false"); analysis.threatLevel = threat; - analysis.shouldBlock = shouldBlock && !shouldExcept; + analysis.shouldBlock = shouldBlock; return analysis; } diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index cba668c..0ab840f 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -24,6 +24,7 @@ #include "ParserBinary.h" #include "ParserMultipartForm.h" #include "ParserPercentEncode.h" +#include "ParserPairs.h" #include "ParserDelimiter.h" #include "WaapAssetState.h" #include "Waf2Regex.h" @@ -34,17 +35,17 @@ USE_DEBUG_FLAG(D_WAAP_DEEP_PARSER); USE_DEBUG_FLAG(D_WAAP_ULIMITS); +USE_DEBUG_FLAG(D_WAAP_STREAMING_PARSING); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); -#define DONE_PARSING 0 -#define FAILED_PARSING -1 +#define DONE_PARSING 0 +#define FAILED_PARSING -1 #define CONTINUE_PARSING 1 -#define MAX_DEPTH 7 +#define MAX_DEPTH 7 DeepParser::DeepParser( - std::shared_ptr pWaapAssetState, - IParserReceiver& receiver, - IWaf2Transaction* pTransaction) -: + std::shared_ptr pWaapAssetState, IParserReceiver &receiver, IWaf2Transaction *pTransaction +) : m_key("deep_parser"), m_pWaapAssetState(pWaapAssetState), m_pTransaction(pTransaction), @@ -58,20 +59,20 @@ DeepParser::DeepParser( m_localMaxObjectDepth(0), m_globalMaxObjectDepthReached(false), m_is_wbxml(false) -{ -} +{} DeepParser::~DeepParser() -{ -} +{} -void DeepParser::setWaapAssetState(std::shared_ptr pWaapAssetState) +void +DeepParser::setWaapAssetState(std::shared_ptr pWaapAssetState) { m_pWaapAssetState = pWaapAssetState; } -void DeepParser::clear() +void +DeepParser::clear() { m_depth = 0; m_splitRefs = 0; @@ -82,17 +83,19 @@ void DeepParser::clear() m_multipart_boundary = ""; } -size_t DeepParser::depth() const +size_t +DeepParser::depth() const { return m_depth; } // Called when another key/value pair is ready -int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) +int +DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) { - int rc = 0; m_depth++; + dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): k='" << std::string(k, (int)k_len) @@ -101,8 +104,11 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i << "'; depth=" << m_depth << "; flags=" - << flags; - + << flags + << " parser_depth: " + << parser_depth + << " v_len = " + << v_len; // Decide whether to push/pop the value in the keystack. bool shouldUpdateKeyStack = (flags & BUFFERED_RECEIVER_F_UNNAMED) == 0; @@ -118,14 +124,21 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i if (shouldUpdateKeyStack) { m_key.push(k, k_len); } - rc = m_receiver.onKv(m_key.c_str(), strlen(m_key.c_str()), cur_val.data(), cur_val.size(), flags); + rc = m_receiver.onKv( + m_key.c_str(), + strlen(m_key.c_str()), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); m_depth--; return rc; } size_t currDepth = 0; if (!isGlobalMaxObjectDepthReached()) { - for (const auto& parser : m_parsersDeque) { + for (const auto &parser : m_parsersDeque) { if (shouldEnforceDepthLimit(parser)) { currDepth += parser->depth(); } @@ -137,20 +150,26 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i } if (currDepth > getGlobalMaxObjectDepth()) { setGlobalMaxObjectDepthReached(); - dbgTrace(D_WAAP_ULIMITS) << "[USER LIMITS] DeepParser::onKv(): Object depth limit exceeded " << - currDepth << "/" << getGlobalMaxObjectDepth() << - " no. of parsers: " << m_parsersDeque.size(); + dbgTrace(D_WAAP_DEEP_PARSER) + << "[USER LIMITS] DeepParser::onKv(): Object depth limit exceeded " + << currDepth + << "/" + << getGlobalMaxObjectDepth() + << " no. of parsers: " + << m_parsersDeque.size(); return DONE_PARSING; - } - else { - dbgTrace(D_WAAP_ULIMITS) << "[USER LIMITS] DeepParser::onKv(): current object depth " << - currDepth << "/" << getGlobalMaxObjectDepth() << - " no. of parsers: " << m_parsersDeque.size(); + } else { + dbgTrace(D_WAAP_DEEP_PARSER) + << "[USER LIMITS] DeepParser::onKv(): current object depth " + << currDepth + << "/" + << getGlobalMaxObjectDepth() + << " no. of parsers: " + << m_parsersDeque.size(); } // Ignore when both key and value are empty - if (k_len == 0 && v_len == 0) - { + if (k_len == 0 && v_len == 0) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): ignoring empty KV pair."; m_depth--; return DONE_PARSING; @@ -169,14 +188,22 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i bool isBodyPayload = (m_key.first().size() == 4 && m_key.first() == "body"); // If csrf/antibot cookie - send to Waf2Transaction for collection of cookie value. - if( m_depth == 1 && isCookiePayload && - (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) { + if (m_depth == 1 && isCookiePayload && (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) { std::string cur_val = std::string(v, v_len); dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): found: " << m_key.str() << "cookie - sending to Waf2Transaction to collect cookie value."; - rc = m_receiver.onKv(m_key.c_str(), strlen(m_key.c_str()), cur_val.data(), cur_val.size(), flags); + rc = + m_receiver.onKv( + m_key.c_str(), + strlen(m_key.c_str()), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); + if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } @@ -185,11 +212,19 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i } // If csrf header - send to Waf2Transaction for collection of cookie value. - if( m_depth == 1 && isHeaderPayload && m_key.str() == "x-chkp-csrf-token") { + if (m_depth == 1 && isHeaderPayload && m_key.str() == "x-chkp-csrf-token") { std::string cur_val = std::string(v, v_len); dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): found csrf header - sending to Waf2Transaction to collect cookie value."; - rc = m_receiver.onKv(m_key.c_str(), strlen(m_key.c_str()), cur_val.data(), cur_val.size(), flags); + rc = m_receiver.onKv( + m_key.c_str(), + strlen(m_key.c_str()), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); + if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } @@ -198,11 +233,19 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i } // If csrf body - send to Waf2Transaction for collection of cookie value. - if(isBodyPayload && m_key.str() == "x-chkp-csrf-token") { + if (isBodyPayload && m_key.str() == "x-chkp-csrf-token") { std::string cur_val = std::string(v, v_len); - dbgTrace(D_WAAP_DEEP_PARSER) - << "DeepParser::onKv(): found csrf form data - sending to Waf2Transaction to collect cookie value."; - rc = m_receiver.onKv(m_key.c_str(), strlen(m_key.c_str()), cur_val.data(), cur_val.size(), flags); + dbgTrace(D_WAAP_DEEP_PARSER + ) << "DeepParser::onKv(): found csrf form data - sending to Waf2Transaction to collect cookie value."; + rc = m_receiver.onKv( + m_key.c_str(), + strlen(m_key.c_str()), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); + if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } @@ -210,17 +253,12 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i return rc; } - // only report kv_pairs collected from the 1st recursion level // (and before b64 decoding, which is important since we don't want to see ".#base64" in parameter // names in this report. if (m_depth == 1) { - if ((k_len > 0 || v_len > 0) && - !isHeaderPayload && - !isUrlPayload && - !isRefererPayload && - !isRefererParamPayload && - !isCookiePayload) { + if ((k_len > 0 || v_len > 0) && !isHeaderPayload && !isUrlPayload && !isRefererPayload && + !isRefererParamPayload && !isCookiePayload) { dbgTrace(D_WAAP_DEEP_PARSER) << " kv_pairs.push_back"; kv_pairs.push_back(std::make_pair(std::string(k, k_len), std::string(v, v_len))); } @@ -234,9 +272,7 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i bool base64ParamFound = false; dbgTrace(D_WAAP_DEEP_PARSER) << " ===Processing potential base64==="; std::string decoded_val, decoded_key; - base64_variants base64_status = Waap::Util::b64Test (cur_val, - decoded_key, - decoded_val); + base64_variants base64_status = Waap::Util::b64Test(cur_val, decoded_key, decoded_val); dbgTrace(D_WAAP_DEEP_PARSER) << " status = " @@ -247,26 +283,34 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i << decoded_val; switch (base64_status) { - case SINGLE_B64_CHUNK_CONVERT: - cur_val = decoded_val; - base64ParamFound = true; - break; - case KEY_VALUE_B64_PAIR: - // going deep with new pair in case value is not empty - if (decoded_val.size() > 0) { + case SINGLE_B64_CHUNK_CONVERT: cur_val = decoded_val; base64ParamFound = true; - rc = onKv(decoded_key.c_str(), decoded_key.size(), cur_val.data(), cur_val.size(), flags); - dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc; - if (rc != CONTINUE_PARSING) { - return rc; + break; + case KEY_VALUE_B64_PAIR: + // going deep with new pair in case value is not empty + if (decoded_val.size() > 0) { + cur_val = decoded_val; + base64ParamFound = true; + rc = onKv( + decoded_key.c_str(), + decoded_key.size(), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); + + dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc; + if (rc != CONTINUE_PARSING) { + return rc; + } } - } - break; - case CONTINUE_AS_IS: - break; - default: - break; + break; + case CONTINUE_AS_IS: + break; + default: + break; } if (base64ParamFound) { @@ -279,14 +323,15 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i // Escape HTML entities such as   before running heuristic stats analyzer std::string cur_val_html_escaped = orig_val; - cur_val_html_escaped.erase(escape_html(cur_val_html_escaped.begin(), cur_val_html_escaped.end()), - cur_val_html_escaped.end()); + cur_val_html_escaped.erase( + escape_html(cur_val_html_escaped.begin(), cur_val_html_escaped.end()), cur_val_html_escaped.end() + ); // Calculate various statistics over currently processed value ValueStatsAnalyzer valueStats(cur_val_html_escaped); + dbgTrace(D_WAAP_DEEP_PARSER) << "ValueStats:\n " << valueStats.textual; - if (valueStats.canSplitPipe || valueStats.canSplitSemicolon) - { + if (valueStats.canSplitPipe || valueStats.canSplitSemicolon) { std::string key = IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction); m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction); } @@ -295,27 +340,58 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i Waap::Util::decodeUtf16Value(valueStats, cur_val); // First buffer in stream - - if (flags & BUFFERED_RECEIVER_F_FIRST) - { - createInternalParser(k, k_len, orig_val, + int offset; + if (flags & BUFFERED_RECEIVER_F_FIRST) { + offset = createInternalParser( + k, + k_len, + orig_val, valueStats, isBodyPayload, isRefererPayload, isRefererParamPayload, isUrlPayload, isUrlParamPayload, - flags); + flags, + parser_depth + ); + } else { + offset = 0; + } + + if (isDebugRequired(TRACE, D_WAAP_STREAMING_PARSING)) { + printParserDeque(); + } + dbgTrace(D_WAAP_STREAMING_PARSING) + << "\n\toffset = " + << offset + << "\n\tm_parsersDeque.empty() = " + << m_parsersDeque.empty() + << "\n\tm_parsersDeque.size() = " + << m_parsersDeque.size() + << "\n\tparser_depth = " + << parser_depth << "\n\tdepth = " + << m_depth; + // defends on parsers' queue for case when ParserRaw created from Waf2Transaction and not placed to m_parsersDeque + if (!m_parsersDeque.empty()) { + dbgTrace(D_WAAP_STREAMING_PARSING) << "m_parsersDeque.size() = " << m_parsersDeque.size(); + if (m_parsersDeque.size() > parser_depth) { + dbgTrace(D_WAAP_STREAMING_PARSING) + << "m_parsersDeque.at(parser_depth-1)->getRecursionFlag() = " + << m_parsersDeque.at(parser_depth)->getRecursionFlag(); + } } // If there's a parser in parsers stack, push the value to the top parser - if (!m_parsersDeque.empty() && !m_parsersDeque.front()->getRecursionFlag()) - { + if (!m_parsersDeque.empty() + && offset >= 0 + && m_parsersDeque.size() > parser_depth + &&!m_parsersDeque.at(parser_depth)->getRecursionFlag() + ) { ScopedContext ctx; - ctx.registerValue("waap_transaction", m_pTransaction); - rc = pushValueToTopParser(cur_val, flags, base64ParamFound); - if (rc != CONTINUE_PARSING) - { + ctx.registerValue("waap_transaction", m_pTransaction); + rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth); + if (rc != CONTINUE_PARSING) { if (shouldUpdateKeyStack) { m_key.pop("deep parser key"); } @@ -327,20 +403,49 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i } - // Parse buffer + if (rc == CONTINUE_PARSING) { + // Try to eliminate m_multipart_boundary to allow other parser to work instead of multipart + if (m_depth == 1 + && isBodyPayload + && !m_multipart_boundary.empty() + && !Waap::Util::testUrlBareUtf8Evasion(cur_val) + && !valueStats.hasSpace + && valueStats.hasCharAmpersand + && valueStats.hasTwoCharsEqual + && !isBinaryData() + ) { + m_multipart_boundary = ""; + rc = parseAfterMisleadingMultipartBoundaryCleaned( + k, + k_len, + orig_val, + valueStats, + isBodyPayload, + isRefererPayload, + isRefererParamPayload, + isUrlPayload, + isUrlParamPayload, + flags, + parser_depth, + base64ParamFound + ); + if (rc != CONTINUE_PARSING) { + return rc; + } + } + } + dbgTrace(D_WAAP_DEEP_PARSER) + << "rc = " + << rc; + // Parse buffer // Note: API report does not include output of "PIPE" and similar extracted stuff. // However, it does include output of URLEncode, MIME, JSON, XML, HTML ... // Also, do not report API for data collected from headers (including the cookie header) - if (m_splitRefs == 0 && - !isHeaderPayload && - !isRefererPayload && - !isRefererParamPayload && - !isUrlPayload && + if (m_splitRefs == 0 && !isHeaderPayload && !isRefererPayload && !isRefererParamPayload && !isUrlPayload && !isCookiePayload) { // A bit ugly (need to rethink/refactor!): remove #.base64 temporarily while adding entry to API report. - if (base64ParamFound) - { + if (base64ParamFound) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): temporarily removing the #base64 prefix from the key."; m_key.pop("#base64", false); @@ -350,22 +455,19 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i // A bit ugly: add back #.base64 after adding entry to API report, so it is reported // correctly if WAF suspicion found... - if (base64ParamFound) - { - dbgTrace(D_WAAP_DEEP_PARSER) << - "DeepParser::onKv(): returning temporarily removed #base64 prefix to the key."; + if (base64ParamFound) { + dbgTrace(D_WAAP_DEEP_PARSER) + << "DeepParser::onKv(): returning temporarily removed #base64 prefix to the key."; m_key.push("#base64", 7, false); } } - if (isUrlPayload) - { + if (isUrlPayload) { valueStats.canSplitPipe = false; valueStats.canSplitSemicolon = false; } - rc = parseBuffer(valueStats, orig_val, base64ParamFound, shouldUpdateKeyStack); - if (rc != CONTINUE_PARSING) - { + rc = parseBuffer(valueStats, orig_val, base64ParamFound, shouldUpdateKeyStack, parser_depth); + if (rc != CONTINUE_PARSING) { return rc; } @@ -376,7 +478,14 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i << " value = " << decoded_val; - rc = onKv(decoded_key.c_str(), decoded_key.size(), decoded_val.data(), decoded_val.size(), flags); + rc = onKv( + decoded_key.c_str(), + decoded_key.size(), + decoded_val.data(), + decoded_val.size(), + flags, + parser_depth + ); dbgTrace(D_WAAP_DEEP_PARSER) << " After processing potential JSON rc = " << rc; if (rc != CONTINUE_PARSING) { @@ -388,20 +497,22 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i // Send key/value pair to the Signature scanner - if (m_key.size() > 0 || cur_val.size() > 0) - { - if (m_deepParserFlag) - { - rc = m_receiver.onKv(m_key.c_str(), strlen(m_key.c_str()), cur_val.data(), cur_val.size(), flags); - } - else - { - rc = m_receiver.onKv(k, k_len, cur_val.data(), cur_val.size(), flags); + if (m_key.size() > 0 || cur_val.size() > 0) { + if (m_deepParserFlag) { + rc = m_receiver.onKv( + m_key.c_str(), + strlen(m_key.c_str()), + cur_val.data(), + cur_val.size(), + flags, + parser_depth + ); + } else { + rc = m_receiver.onKv(k, k_len, cur_val.data(), cur_val.size(), flags, parser_depth); } } - if (base64ParamFound) - { + if (base64ParamFound) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; m_key.pop("#base64", false); } @@ -412,214 +523,23 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i return rc; } - -int DeepParser::parseBuffer(ValueStatsAnalyzer& valueStats, const std::string& cur_val, bool base64ParamFound, - bool shouldUpdateKeyStack) +int +DeepParser::parseBuffer( + ValueStatsAnalyzer &valueStats, + const std::string &cur_val, + bool base64ParamFound, + bool shouldUpdateKeyStack, + size_t parser_depth +) { dbgFlow(D_WAAP_DEEP_PARSER) << "cur_val='" << cur_val << "'"; - - // Detect and decode stuff in relative urls like /a.php?blah=cow&... - // TODO:: when removing cur_val -> PAY ATTENTION THAT THIS CODE ASSUMES - // cur_val is ZERO TERMINATED STRING! - if (valueStats.hasCharSlash && valueStats.hasCharEqual && cur_val.length() > 1 && cur_val[0] == '/') - { - const char* p = cur_val.c_str() + 1; - // Read path part until it either hits '?' or '/' - while (isalpha(*p) || isdigit(*p) || *p == '.' || *p == '-' || *p == '_') - { - p++; - } - - if (*p == '?' || *p == '/') - { - if (*p == '?') - { - // '?' character is found - p++; - } - else - { - // Path seem to be starting correctly, hitting the '/' character. - // Skip the path to find the '?' character - p = strchr(p, '?'); - if (p) { - // '?' character is found - p++; - } - } - - if (p) - { - // Value starts as url, and contains '?' character: urldecode the rest. - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): relative url -> parsing urlencode"; - // Avoid using any output of this and deeper subparsers for API structure report. - Ref ref(m_splitRefs); - BufferedReceiver rcvr(*this); - size_t buff_len = cur_val.length() - (p - &cur_val[0]); - ParserUrlEncode up(rcvr, '&', checkUrlEncoded(p, buff_len)); - up.push(p, buff_len); - up.finish(); - - if (base64ParamFound) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; - m_key.pop("#base64", false); - } - - if (shouldUpdateKeyStack) { - m_key.pop("deep parser key"); - } - m_depth--; - - if (!up.error()) - { - return DONE_PARSING; - } - } - else - { - return CONTINUE_PARSING; - } - } - } - - // Detect and decode stuff like URLs that start with http:// or https:// - if (valueStats.hasCharColon && valueStats.hasCharSlash && cur_val.length() > 7) - { - const char* p = cur_val.c_str(); - - if (*p++ == 'h' && *p++ == 't' && *p++ == 't' && *p++ == 'p') - { - // value starts with "http" - if (*p == 's') - { - // starts with "https" - p++; - } - - if (*p++ == ':' && *p++ == '/' && *p++ == '/') - { - // cur_val starts with "http://" or "https://" - // first, ensure that domain name is valid (to eliminate false detections of URLs) - while (isalpha(*p) || isdigit(*p) || *p == '.' || *p == '-' || *p == '_') - { - p++; - } - - if (*p == '/') - { - // domain name is seemingly valid, and we hit '/' character - p++; - // skip the path to find the '?' character - p = strchr(p, '?'); - - if (p) - { - // Value starts as url, and contains '?' character: urldecode the rest. - p++; - - // Avoid using any output of this and deeper subparsers for API structure report. - Ref ref(m_splitRefs); - - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): full url -> parsing urlencode"; - BufferedReceiver rcvr(*this); - size_t buff_len = cur_val.length() - (p - &cur_val[0]); - ParserUrlEncode up(rcvr, '&', checkUrlEncoded(p, buff_len)); - up.push(p, buff_len); - up.finish(); - - if (base64ParamFound) - { - dbgTrace(D_WAAP_DEEP_PARSER) - << "DeepParser::onKv(): removing the #base64 prefix from the key."; - m_key.pop("#base64", false); - } - - if (shouldUpdateKeyStack) { - m_key.pop("deep parser key"); - } - m_depth--; - - if (!up.error()) - { - return DONE_PARSING; - } - } - else - { - return CONTINUE_PARSING; - } - } - } - } - } - - bool isUrlBareUtf8Evasion = valueStats.isUrlEncoded && Waap::Util::testUrlBareUtf8Evasion(cur_val); - - bool isUrlBadUtf8Evasion = valueStats.isUrlEncoded && Waap::Util::testUrlBadUtf8Evasion(cur_val); - - bool isUrlEncodedPairs = valueStats.hasCharAmpersand && valueStats.hasTwoCharsEqual; - - if (valueStats.isUrlEncoded && !isUrlEncodedPairs && !isUrlBareUtf8Evasion && !isUrlBadUtf8Evasion) { - // Single UrlEncoded (percent-encoded) value detected, not urlencoded pairs - dbgTrace(D_WAAP_DEEP_PARSER) << - "DeepParser::onKv(): urlencoded single value detected, decoding percent encoding"; - std::string decodedVal = cur_val; - Waap::Util::decodePercentEncoding(decodedVal, true); - onKv("", 0, decodedVal.data(), decodedVal.size(), BUFFERED_RECEIVER_F_BOTH | BUFFERED_RECEIVER_F_UNNAMED); - - if (base64ParamFound) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; - m_key.pop("#base64", false); - } - - if (shouldUpdateKeyStack) { - m_key.pop("deep parser key"); - } - m_depth--; - - return DONE_PARSING; - } - else if (!valueStats.hasSpace && isUrlEncodedPairs && !isUrlBareUtf8Evasion && !isBinaryData()) - { - // If there are 1 or more '&' characters, or 2 or more '=' characters - assume its - // URLEncoded data and apply URL decode to the value. - // This case applies even to samples like "a=b&c=d" where no percent-encoding is present. - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): urlencoded pairs value detected, parsing urlencode pairs"; - // Avoid using any output of this and deeper subparsers for API structure report. - Ref ref(m_splitRefs); - BufferedReceiver rcvr(*this); - ParserUrlEncode up(rcvr, '&', checkUrlEncoded(cur_val.data(), cur_val.size())); - up.push(cur_val.data(), cur_val.size()); - up.finish(); - - if (base64ParamFound) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; - m_key.pop("#base64", false); - } - - if (shouldUpdateKeyStack) { - m_key.pop("deep parser key"); - } - m_depth--; - - if (!up.error()) - { - return DONE_PARSING; - } - } + // TODO: SplitRegex should be replaced by streaming solution, probably, ParserDelimiter in this case // detect and decode stuff like "a=b;c=d;e=f;klm" - if (valueStats.canSplitSemicolon && - (valueStats.hasCharSemicolon) && - cur_val.length() > 0 && - splitByRegex(cur_val, m_pWaapAssetState->getSignatures()->semicolon_split_re, "sem")) - { + if (valueStats.canSplitSemicolon && valueStats.hasCharSemicolon && cur_val.length() > 0 && + splitByRegex(cur_val, m_pWaapAssetState->getSignatures()->semicolon_split_re, "sem", parser_depth)) { if (base64ParamFound) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; m_key.pop("#base64", false); - } if (shouldUpdateKeyStack) { @@ -629,17 +549,13 @@ int DeepParser::parseBuffer(ValueStatsAnalyzer& valueStats, const std::string& c m_depth--; return DONE_PARSING; } - + // TODO: SplitRegex should be replaced by streaming solution, probably, ParserDelimiter in this case // detect and decode stuff like "abc|def|klm" - if (valueStats.canSplitPipe && - (valueStats.hasCharPipe) && - cur_val.length() > 0 && - splitByRegex(cur_val, m_pWaapAssetState->getSignatures()->pipe_split_re, "pipe")) - { + if (valueStats.canSplitPipe && valueStats.hasCharPipe && cur_val.length() > 0 && + splitByRegex(cur_val, m_pWaapAssetState->getSignatures()->pipe_split_re, "pipe", parser_depth)) { // split done - do not send the unsplit string to the scanner - if (base64ParamFound) - { + if (base64ParamFound) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): removing the #base64 prefix from the key."; m_key.pop("#base64", false); } @@ -654,75 +570,122 @@ int DeepParser::parseBuffer(ValueStatsAnalyzer& valueStats, const std::string& c return CONTINUE_PARSING; } -int DeepParser::pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound) +int +DeepParser::pushValueToTopParser( + std::string &cur_val, + int flags, + bool base64ParamFound, + int offset, + size_t parser_depth +) { - std::shared_ptr topParser = m_parsersDeque.front(); + std::shared_ptr actualParser = m_parsersDeque.at(parser_depth); + dbgTrace(D_WAAP_STREAMING_PARSING) + << "Actual parser name = " + << actualParser->name() + << " \tparser_depth=" + << parser_depth + << " \tName by parser depth = " + << m_parsersDeque.at(parser_depth)->name() + << " \toffset = " + << offset + << " \tflags = " + << flags; - if (!topParser->error()) - { + if (isDebugRequired(TRACE, D_WAAP_STREAMING_PARSING)) { + printParserDeque(); + } + + if (!actualParser->error()) { m_deepParserFlag = true; - m_parsersDeque.front()->setRecursionFlag(); + m_parsersDeque.at(parser_depth)->setRecursionFlag(); // Push current buffer to the top parser // This might generate one or more recursive calls back to DeepParser::onKv() dbgTrace(D_WAAP_DEEP_PARSER) - << "DeepParser::onKv(): push " + << "DeepParser::pushValueToTopParser():" << cur_val.size() << " bytes parser " - << topParser->name(); - topParser->push(cur_val.c_str(), cur_val.length()); + << actualParser->name(); + actualParser->push(cur_val.c_str() + offset, cur_val.length() - offset); // Last buffer in stream - if (flags & BUFFERED_RECEIVER_F_LAST) - { - // Tell the top parser that the stream is finished - // This might still generate one or more recursive calls back to DeepParser::onKv() - topParser->finish(); + if (flags & BUFFERED_RECEIVER_F_LAST) { + actualParser->finish(); } - m_parsersDeque.front()->clearRecursionFlag(); + m_parsersDeque.at(parser_depth)->clearRecursionFlag(); m_deepParserFlag = false; - } - else - { + } else { dbgTrace(D_WAAP_DEEP_PARSER) - << "DeepParser::onKv(): skip push " + << "DeepParser::pushValueToTopParser():" << cur_val.size() << " bytes to parser " - << topParser->name() + << actualParser->name() << " (parser is in error state)"; } + // TODO - must ensure that its removal correct!!!!! // Last buffer in stream - if (!m_parsersDeque.empty() && flags & BUFFERED_RECEIVER_F_LAST) - { + if (!m_parsersDeque.empty() && flags & BUFFERED_RECEIVER_F_LAST) { // Remove the top parser from the stack - m_parsersDeque.pop_front(); + m_parsersDeque.pop_back(); + dbgTrace(D_WAAP_DEEP_PARSER) + << "DeepParser::pushValueToTopParser(): " + << " Remove the top parser from the stack" + << " parser_depth = " + << parser_depth + << " flags = " + << flags; } - if (base64ParamFound) - { + if (base64ParamFound) { dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): temporarily removing the #base64 prefix from the key."; m_key.pop("#base64", false); } - if (!topParser->error()) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): parser " << topParser->name() << " is still valid."; + if (!actualParser->error()) { + dbgTrace(D_WAAP_DEEP_PARSER) + << "DeepParser::pushValueToTopParser(): parser " + << actualParser->name() + << " is still valid on depth = " + << parser_depth + << "DeepParser::pushValueToTopParser(): " + << " return DONE_PARSING"; return DONE_PARSING; // do not send the parsed source to the scanner } - + dbgTrace(D_WAAP_DEEP_PARSER) + << "DeepParser::pushValueToTopParser(): " + << " return CONTINUE_PARSING"; return CONTINUE_PARSING; } -class StubParserReceiver : public IParserReceiver { +class StubParserReceiver : public IParserReceiver +{ int - onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) + onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) { return 0; } }; +static bool +checkIfdelimeteredPattern(const std::string &pattern, char delim) +{ + bool is_empty = true; + bool has_eq_sign = false; + for (auto &ch : pattern) { + if (ch == '=') has_eq_sign = true; + is_empty = false; + if (ch == delim) { + if (!has_eq_sign) return false; + is_empty = true; + has_eq_sign = false; + } + } + return has_eq_sign || is_empty; +} + static bool validateJson(const char *v, size_t v_len) { @@ -733,115 +696,316 @@ validateJson(const char *v, size_t v_len) return !jsParser.error(); } -void DeepParser::createInternalParser(const char *k, size_t k_len, std::string& cur_val, +void +DeepParser::printParserDeque() +{ + if (isDebugRequired(TRACE, D_WAAP_STREAMING_PARSING)) { + dbgTrace(D_WAAP_STREAMING_PARSING) << "---- Printing parser queue: -----"; + for (auto it = m_parsersDeque.begin(); it != m_parsersDeque.end(); ++it) { + std::shared_ptr tmp = *it; + dbgTrace(D_WAAP_STREAMING_PARSING) << "\t\t" << tmp->name(); + } + dbgTrace(D_WAAP_STREAMING_PARSING) << "---- End of parsers queue -----"; + } +} + +// getShiftInUrlEncodedBuffer receives potential encoded URL and calculates offset where URL query is starting +// i.e. in case of input like "http[s]://domain[:port]/uri_path?param=value&..." offset will point to +// 1st character of of query i.e. "param=value&..." +// In case input doesn't comply URI format, negative value will be returned +// This function also supports old notation where semicolon used instead of ampersand +int +DeepParser::getShiftInUrlEncodedBuffer(const ValueStatsAnalyzer &valueStats, std::string &cur_val) +{ + dbgTrace(D_WAAP_DEEP_PARSER) << "getShiftInUrlEncodedBuffer(): " << cur_val; + bool continue_flag = false; + int offset = -1; + const char *end = cur_val.c_str() + cur_val.size(); + const char *p = cur_val.c_str(); + + if (valueStats.hasCharSlash && valueStats.hasCharEqual && cur_val.length() > 1 && cur_val[0] == '/') { + p++; + continue_flag = true; + offset = 1; + + // Read path part until it either hits '?' or '/' + while (p < end && (isalpha(*p) || isdigit(*p) || *p == '.' || *p == '-' || *p == '_')) { + p++; + offset++; + } + } + + if (offset < 0) p = cur_val.c_str(); + if (valueStats.hasCharColon && valueStats.hasCharSlash && cur_val.length() > 7) { + if (*p++ == 'h' && *p++ == 't' && *p++ == 't' && *p++ == 'p') { + // value starts with "http" + offset = 4; + if (*p == 's') { + // starts with "https" + p++; + offset++; + } + + if (*p++ == ':' && *p++ == '/' && *p++ == '/') { + // cur_val starts with "http://" or "https://" + // first, ensure that domain name is valid (to eliminate false detections of URLs) + // added '@' and ':' to comply format domain name in URL (user_context@domain.name:port_num) + offset +=3; + while ( + p < end + && (isalpha(*p) || isdigit(*p) || *p == '.' || *p == '-' || *p == '_' || *p == ':' || *p == '@') + ) { + p++; + offset++; + } + if (*p != '/') return -1; + continue_flag = true; + } + } + } + + if (continue_flag) { + // domain name is seemingly valid, and we hit '/' character + // skip the path to find the '?' character + // in contradiction to canonical definition allowed chars ';' in the path of URL to avoid some false positives + p = strchr(p, '?'); + const char *start_point = cur_val.c_str(); + + if (p) { + int range = cur_val.length(); + int shift = p - start_point; + // Value starts as url, and contains '?' character: urldecode the rest. + if (shift < range) { + offset = shift; + if (*p == '?') offset++; + } + } else { + offset = -1; + } + } else { + offset = -1; + } + return offset; +} +int +DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( + const char *k, + size_t k_len, + std::string &cur_val, const ValueStatsAnalyzer &valueStats, bool isBodyPayload, bool isRefererPayload, bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, - int flags) + int flags, + size_t parser_depth, + bool base64ParamFound) +{ + int offset = -1; + int rc = 0; + bool shouldUpdateKeyStack = (flags & BUFFERED_RECEIVER_F_UNNAMED) == 0; + if (flags & BUFFERED_RECEIVER_F_FIRST) { + offset = createInternalParser( + k, + k_len, + cur_val, + valueStats, + isBodyPayload, + isRefererPayload, + isRefererParamPayload, + isUrlPayload, + isUrlParamPayload, + flags, + parser_depth + ); + } else { + offset = 0; + } + + if (isDebugRequired(TRACE, D_WAAP_STREAMING_PARSING)) { + printParserDeque(); + } + + dbgTrace(D_WAAP_STREAMING_PARSING) + << "\n\toffset = " + << offset + << "\n\tm_parsersDeque.empty() = " + << m_parsersDeque.empty() + << "\n\tm_parsersDeque.size() = " + << m_parsersDeque.size() + << "\n\tparser_depth = " + << parser_depth << "\n\tdepth = " + << m_depth; + // defends on parsers' queue for case when ParserRaw created from Waf2Transaction and not placed to m_parsersDeque + if (!m_parsersDeque.empty()) { + dbgTrace(D_WAAP_STREAMING_PARSING) << "\n\tm_parsersDeque.size() = " << m_parsersDeque.size(); + if (m_parsersDeque.size() > parser_depth) { + dbgTrace(D_WAAP_STREAMING_PARSING) + << "m_parsersDeque.at(parser_depth-1)->getRecursionFlag() = " + << m_parsersDeque.at(parser_depth)->getRecursionFlag(); + } + } + + // If there's a parser in parsers stack, push the value to the actual parser + if (!m_parsersDeque.empty() + && offset >= 0 + && m_parsersDeque.size() > parser_depth + &&!m_parsersDeque.at(parser_depth)->getRecursionFlag() + ) { + ScopedContext ctx; + ctx.registerValue("waap_transaction", m_pTransaction); + rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth); + if (rc != CONTINUE_PARSING) { + if (shouldUpdateKeyStack) { + m_key.pop("deep parser key"); + } + + + m_depth--; + return rc; + } + } + + + return rc; +} + +static bool err = false; +static const SingleRegex json_detector_re("\\A[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", err, "json_detector"); +static const SingleRegex json_quoteless_detector_re("^[{\\[][[,0-9nul\\]]+", err, "json_quoteless_detector"); + +//intended to keep and process all types of leftovers detected as separate cases for parsing +int +DeepParser::createUrlParserForJson( +const char *k, +size_t k_len, +std::string &cur_val, +const ValueStatsAnalyzer &valueStats, +bool isBodyPayload, +bool isRefererPayload, +bool isRefererParamPayload, +bool isUrlPayload, +bool isUrlParamPayload, +int flags, +size_t parser_depth +) { + int ret_val = -1; + std::string decoded_key, decoded_val; + dbgTrace(D_WAAP_DEEP_PARSER) + << "Last try create parsers for value: >>>" + << cur_val + << "\n\tm_parsersDeque.size() = " + << m_parsersDeque.size() + << "\n\tparser_depth = " + << parser_depth + << "\n\tdepth = " + << m_depth + << "\n\tflags: " + << flags + << "\n\tparser_depth: " + << parser_depth; + + if (Waap::Util::detectJSONasParameter(cur_val, decoded_key, decoded_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) + << "Detected param=JSON," + << " still starting to parse an Url-encoded-like data due to possible tail"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + ret_val = 0; + } + return ret_val; +} + + +int +DeepParser::createInternalParser( + const char *k, + size_t k_len, + std::string &cur_val, + const ValueStatsAnalyzer &valueStats, + bool isBodyPayload, + bool isRefererPayload, + bool isRefererParamPayload, + bool isUrlPayload, + bool isUrlParamPayload, + int flags, + size_t parser_depth +) { dbgTrace(D_WAAP_DEEP_PARSER) << "Starting create parsers for value: >>>" << cur_val - << "<<<"; - dbgTrace(D_WAAP_DEEP_PARSER) - << "Stats:\n " - << valueStats.textual; - bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, - isCommaType = false, isAmperType = false; + << "<<<\n\tm_parsersDeque.empty() = " + << m_parsersDeque.empty() + << "\n\tm_parsersDeque.size() = " + << m_parsersDeque.size() + << "\n\tparser_depth = " + << parser_depth + << "\n\tdepth = " + << m_depth + << "\n\tflags: " + << flags + << "\n\tparser_depth: " + << parser_depth; + bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false, + isAmperType = false; bool isKeyValDelimited = false; bool isHtmlType = false; bool isBinaryType = false; + int offset = -1; auto pWaapAssetState = m_pTransaction->getAssetState(); std::shared_ptr signatures = m_pWaapAssetState->getSignatures(); - if (pWaapAssetState != nullptr) - { + if (pWaapAssetState != nullptr) { // Find out learned type std::set paramTypes = pWaapAssetState->m_filtersMngr->getParameterTypes( - IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction)); + IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction) + ); dbgTrace(D_WAAP_DEEP_PARSER) << "ParamTypes (count=" << paramTypes.size() << "):"; for (const auto ¶mType : paramTypes) { dbgTrace(D_WAAP_DEEP_PARSER) << "ParamType: '" << paramType << "'"; } - if (!paramTypes.empty()) - { + if (!paramTypes.empty()) { std::set sampleType = m_pWaapAssetState->getSampleType(cur_val); boost::smatch match; if (paramTypes.find("ampersand_delimiter") != paramTypes.end()) { - isKeyValDelimited = NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - cur_val, - match, - signatures->ampersand_delimited_key_val_re - ); + isKeyValDelimited = checkIfdelimeteredPattern(cur_val, '&'); isAmperType = sampleType.find("ampersand_delimiter") != sampleType.end(); } else if (paramTypes.find("pipes") != paramTypes.end()) { - isKeyValDelimited = NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - cur_val, - match, - signatures->pipes_delimited_key_val_re - ); + isKeyValDelimited = checkIfdelimeteredPattern(cur_val, '|'); isPipesType = sampleType.find("pipes") != sampleType.end(); } else if (paramTypes.find("semicolon_delimiter") != paramTypes.end()) { - isKeyValDelimited = NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - cur_val, - match, - signatures->semicolon_delimited_key_val_re - ); + isKeyValDelimited = checkIfdelimeteredPattern(cur_val, ';'); isSemicolonType = sampleType.find("semicolon_delimiter") != sampleType.end(); } else if (paramTypes.find("asterisk_delimiter") != paramTypes.end()) { - isKeyValDelimited = NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - cur_val, - match, - signatures->asterisk_delimited_key_val_re - ); + isKeyValDelimited = checkIfdelimeteredPattern(cur_val, '*'); isAsteriskType = sampleType.find("asterisk_delimiter") != sampleType.end(); } else if (paramTypes.find("comma_delimiter") != paramTypes.end()) { - isKeyValDelimited = NGEN::Regex::regexMatch( - __FILE__, - __LINE__, - cur_val, - match, - signatures->comma_delimited_key_val_re - ); + isKeyValDelimited = checkIfdelimeteredPattern(cur_val, ','); isCommaType = sampleType.find("comma_delimiter") != sampleType.end(); } - if (paramTypes.find("html_input") != paramTypes.end()) - { + if (paramTypes.find("html_input") != paramTypes.end()) { std::set sampleType = m_pWaapAssetState->getSampleType(cur_val); - if (sampleType.find("html_input") != sampleType.end()) - { + if (sampleType.find("html_input") != sampleType.end()) { dbgTrace(D_WAAP_DEEP_PARSER) << "html_input sample type learned and validated"; isHtmlType = true; } } - if (paramTypes.find("binary_input") != paramTypes.end()) - { + if (paramTypes.find("binary_input") != paramTypes.end()) { std::set sampleType = m_pWaapAssetState->getSampleType(cur_val); - if (sampleType.find("binary_input") != sampleType.end()) - { + if (sampleType.find("binary_input") != sampleType.end()) { dbgTrace(D_WAAP_DEEP_PARSER) << "binary_input sample type learned and validated"; isBinaryType = true; } @@ -868,174 +1032,308 @@ void DeepParser::createInternalParser(const char *k, size_t k_len, std::string& // size (you can find the value of MAX_VALUE_SIZE defined in ParserBase.cc). Waap::Util::ContentType requestContentType = m_pTransaction->getContentType(); bool isPotentialGqlQuery = false; + if (flags == BUFFERED_RECEIVER_F_BOTH) { // TODO:: should we limit ourselves to the 64k buffer? static std::string strQuery("query"); bool isParamQuery = strQuery.size() == k_len && std::equal(k, k + k_len, strQuery.begin()); - isPotentialGqlQuery |= isParamQuery && m_depth == 1 && (isUrlParamPayload || isRefererParamPayload); - isPotentialGqlQuery |= m_depth == 1 && isBodyPayload && requestContentType == Waap::Util::CONTENT_TYPE_GQL; - isPotentialGqlQuery |= isParamQuery && m_depth == 2 && isBodyPayload && - requestContentType == Waap::Util::CONTENT_TYPE_JSON; + isPotentialGqlQuery |= isParamQuery + && m_depth == 1 + && (isUrlParamPayload || isRefererParamPayload); + isPotentialGqlQuery |= m_depth == 1 + && isBodyPayload + && requestContentType == Waap::Util::CONTENT_TYPE_GQL; + isPotentialGqlQuery |= isParamQuery + && m_depth == 2 + && isBodyPayload + && requestContentType == Waap::Util::CONTENT_TYPE_JSON; } - dbgTrace(D_WAAP_DEEP_PARSER) - << "isPotentialGqlQuery=" - << isPotentialGqlQuery - << ";isTopData=" - << isTopData - << ";depth=" - << m_depth - << (m_parsersDeque.empty() ? "" : ";topParserName=" + m_parsersDeque.front()->name()); + << "\n\tm_parsersDeque.empty() = " + << m_parsersDeque.empty() + << "\n\tm_parsersDeque.size() = " + << m_parsersDeque.size() + << "\n\tparser_depth = " + << parser_depth + << "\n\tdepth = " + << m_depth; + if (parser_depth > 0) { + dbgTrace(D_WAAP_DEEP_PARSER) + << "isPotentialGqlQuery=" + << isPotentialGqlQuery + << ";isTopData=" + << isTopData + << ";depth=" + << m_depth + << (m_parsersDeque.empty() ? "" : ";actualParserName=" + m_parsersDeque.at(parser_depth - 1)->name()); + } // Add zero or one parser on top of the parsers stack // Note that this function must not add more than one parser // because only the topmost parser will run on the value. // Normally, DeepParser will take care of recursively run other parsers. - if (valueStats.isUrlEncoded && - !Waap::Util::testUrlBareUtf8Evasion(cur_val)) { - if (!valueStats.hasSpace && - valueStats.hasCharAmpersand && - valueStats.hasTwoCharsEqual && - !isBinaryData()) { - dbgTrace(D_WAAP_DEEP_PARSER) << " Starting to parse an Url-encoded data"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } else if (!Waap::Util::testUrlBadUtf8Evasion(cur_val)) { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an percent decoding"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } - } else if (isHtmlType && - !isRefererPayload && - !isUrlPayload) { + + if (isHtmlType + && !isRefererPayload + && !isUrlPayload + ) { // HTML detected dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an HTML file"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } else if (cur_val.size() > 0 && - signatures->php_serialize_identifier.hasMatch(cur_val)) { + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else if (cur_val.size() > 0 && signatures->php_serialize_identifier.hasMatch(cur_val)) { // PHP value detected dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse phpSerializedData"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } else if (isPotentialGqlQuery && - cur_val.size() > 0 && - !validateJson(cur_val.data(), cur_val.size())) { + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else if (isPotentialGqlQuery + && cur_val.size() > 0 + && !validateJson(cur_val.data(), cur_val.size()) + ) { // Graphql value detected dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse graphql"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } else if (cur_val.length() > 0 && (cur_val[0] == '[' || cur_val[0] == '{')) { + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else if (cur_val.length() > 0 + && (cur_val[0] == '[' || cur_val[0] == '{') + ) { boost::smatch confulence_match; - - if (NGEN::Regex::regexMatch(__FILE__, __LINE__, cur_val, confulence_match, signatures->confluence_macro_re)) - { + dbgTrace(D_WAAP_DEEP_PARSER) << "attempt to find confluence of JSON by '{' or '['"; + if (NGEN::Regex::regexMatch(__FILE__, __LINE__, cur_val, confulence_match, signatures->confluence_macro_re)) { dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a confluence macro"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } - else - { - // JSON value detected - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file"; - // Send openApiReceiver as secondary receiver, but only if the JSON is passed in body and on the top level. + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else { + dbgTrace(D_WAAP_DEEP_PARSER) << "attempt to find JSON by '{' or '['"; + bool percent_encoded_doublequote_detected = cur_val.find("%22") != std::string::npos; + if (json_detector_re.hasMatch(cur_val) + && (valueStats.hasDoubleQuote + || json_quoteless_detector_re.hasMatch(cur_val) + || percent_encoded_doublequote_detected)) { + // JSON value detected + if (percent_encoded_doublequote_detected && !valueStats.hasDoubleQuote) { + // We have JSOn but it %-encoded, first start percent decoding for it. Very narrow case + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file from percent decoding"; + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1) + ); + offset = 0; + } else { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file"; + // Send openApiReceiver as secondary receiver, + // but only if the JSON is passed in body and on the top level. - - m_parsersDeque.push_front(std::make_shared>(*this)); + m_parsersDeque.push_back( + std::make_shared>( + *this, + NULL, + parser_depth + 1 +)); + offset = 0; + } + } } } - else if (cur_val.length() > 4 && - (cur_val[0] == '<') && - !isRefererPayload && - !isRefererParamPayload && - !isUrlPayload && - !isUrlParamPayload && - !startsWithHtmlTagName(cur_val.c_str() + 1)) - { - // XML detected. Note: XML must be at a minimum 4 bytes long to be valid. - // Also, XML is not scanned in payload coming from URL or URL parameters, or if the - // payload starts with one of known HTML tags. - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an XML file"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } - else if (m_depth == 1 && - isBodyPayload && - !m_multipart_boundary.empty()) { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a multipart file"; - m_parsersDeque.push_front(std::make_shared>( - *this, - m_multipart_boundary.c_str(), - m_multipart_boundary.length() + if (offset < 0) { + if (cur_val.length() > 4 + && (cur_val[0] == '<') + && !isRefererPayload + && !isRefererParamPayload + && !isUrlPayload + && !isUrlParamPayload + && !startsWithHtmlTagName(cur_val.c_str() + 1) + ) { + // XML detected. Note: XML must be at a minimum 4 bytes long to be valid. + // Also, XML is not scanned in payload coming from URL or URL parameters, or if the + // payload starts with one of known HTML tags. + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an XML file"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + } else if (m_depth == 1 && isBodyPayload && !m_multipart_boundary.empty()) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a multipart file"; + m_parsersDeque.push_back(std::make_shared>( + *this, parser_depth + 1, m_multipart_boundary.c_str(), m_multipart_boundary.length() )); - } - else if (isTopData && (isBinaryType || m_pWaapAssetState->isBinarySampleType(cur_val))) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file"; - m_parsersDeque.push_front(std::make_shared>(*this)); - } - else if (isPipesType) { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse pipes, positional: " << isKeyValDelimited; - if (isKeyValDelimited) - { - m_parsersDeque.push_front(std::make_shared>(*this, '|')); - } - else - { - m_parsersDeque.push_front(std::make_shared>(*this, '|', "pipe")); + offset = 0; + } else if (isTopData && (isBinaryType || m_pWaapAssetState->isBinarySampleType(cur_val))) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; } } - else if (isSemicolonType) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a semicolon, positional: " << isKeyValDelimited; - if (isKeyValDelimited) - { - m_parsersDeque.push_front(std::make_shared>(*this, ';')); - } - else - { - m_parsersDeque.push_front(std::make_shared>(*this, ';', "sem")); + if (offset < 0) { + if (isPipesType) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse pipes, positional: " << isKeyValDelimited; + if (isKeyValDelimited) { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '|') + ); + offset = 0; + } else { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '|', "pipe") + ); + offset = 0; + } + } else if (isSemicolonType) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a semicolon, positional: " << isKeyValDelimited; + if (isKeyValDelimited) { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, ';') + ); + offset = 0; + } else { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, ';', "sem") + ); + offset = 0; + } + } else if (isAsteriskType) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an asterisk, positional: " << isKeyValDelimited; + if (isKeyValDelimited) { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '*') + ); + offset = 0; + } else { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '*', "asterisk") + ); + offset = 0; + } + } else if (isCommaType) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a comma, positional: " << isKeyValDelimited; + if (isKeyValDelimited) { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, ',') + ); + offset = 0; + } else { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, ',', "comma") + ); + offset = 0; + } + } else if (isAmperType) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a ampersand, positional: " << isKeyValDelimited; + if (isKeyValDelimited) { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '&') + ); + offset = 0; + } else { + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1, '&', "amp") + ); + offset = 0; + } + // the following block is added to allow streaming parsing (instead of unstreamed parser usage from + // DeepParser::parseBuffer - code was removed from there + // In case we have some kind of URI (with or without protocol/port), getShiftInUrlEncodedBuffer method will + // detect where URI path is started and based on this ParserUrlEncode parser will be created for sub-buffer + // else ParserPercentEncode parser is invoked + } else if ( + valueStats.hasCharSlash + && (valueStats.hasCharColon || valueStats.hasCharEqual) + && !valueStats.hasCharLess + ) { + offset = getShiftInUrlEncodedBuffer(valueStats, cur_val); + dbgTrace(D_WAAP_DEEP_PARSER) + << "offset = " + << offset + << " cur_val.size = " + << cur_val.size() + << " cur_val.len = " + << cur_val.length(); + int delta = offset - cur_val.size(); + if (offset >= 0 && delta <= 0) { + dbgTrace(D_WAAP_DEEP_PARSER) << " Starting to parse an Url-encoded data after removing prefix"; + m_parsersDeque.push_back( + std::make_shared>( + *this, + parser_depth + 1, + '&', + valueStats.isUrlEncoded) + ); + } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)"; + if (!valueStats.hasSpace + && valueStats.hasCharAmpersand + && valueStats.hasTwoCharsEqual + && !isBinaryData() + ) { + dbgTrace(D_WAAP_DEEP_PARSER) << " Starting to parse an Url-encoded data - pairs detected"; + m_parsersDeque.push_back( + std::make_shared>( + *this, + parser_depth + 1, + '&', + valueStats.isUrlEncoded) + ); + offset = 0; + return offset; + } else if (valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an percent decoding"; + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1) + ); + offset = 0; + return offset; + } + } + } else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)"; + if (!valueStats.hasSpace + && valueStats.hasCharAmpersand + && valueStats.hasTwoCharsEqual + && !isBinaryData() + ) { + dbgTrace(D_WAAP_DEEP_PARSER) << " Starting to parse an Url-encoded data - pairs detected"; + m_parsersDeque.push_back( + std::make_shared>( + *this, + parser_depth + 1, + '&', + valueStats.isUrlEncoded) + ); + offset = 0; + return offset; + } else if (valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an percent decoding"; + m_parsersDeque.push_back( + std::make_shared>(*this, parser_depth + 1) + ); + offset = 0; + return offset; + } } } - else if (isAsteriskType) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an asterisk, positional: " << isKeyValDelimited; - if (isKeyValDelimited) - { - m_parsersDeque.push_front(std::make_shared>(*this, '*')); - } - else - { - m_parsersDeque.push_front(std::make_shared>(*this, '*', "asterisk")); - } - } - else if (isCommaType) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a comma, positional: " << isKeyValDelimited; - if (isKeyValDelimited) - { - m_parsersDeque.push_front(std::make_shared>(*this, ',')); - } - else - { - m_parsersDeque.push_front(std::make_shared>(*this, ',', "comma")); - } - } - else if (isAmperType) - { - dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a ampersand, positional: " << isKeyValDelimited; - if (isKeyValDelimited) - { - m_parsersDeque.push_front(std::make_shared>(*this, '&')); - } - else - { - m_parsersDeque.push_front(std::make_shared>(*this, '&', "amp")); - } + if (offset < 0) { + offset = createUrlParserForJson( + k, + k_len, + cur_val, + valueStats, + isBodyPayload, + isRefererPayload, + isRefererParamPayload, + isUrlPayload, + isUrlParamPayload, + flags, + parser_depth + ); } + return offset; } -void DeepParser::apiProcessKey(const char* v, size_t v_len) +void +DeepParser::apiProcessKey(const char *v, size_t v_len) { // Build dot-formatted full keyword name std::string kwType = m_key.first(); std::string kwFullName = m_key.str(); - if (v_len == 0 && kwFullName.size() == 0) - { + if (v_len == 0 && kwFullName.size() == 0) { return; } @@ -1043,7 +1341,8 @@ void DeepParser::apiProcessKey(const char* v, size_t v_len) } // TODO:: maybe convert this splitter to Parser-derived class? -bool DeepParser::splitByRegex(const std::string& val, const Regex& r, const char* keyPrefix) +bool +DeepParser::splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix, size_t parser_depth) { bool splitDone = false; std::vector matches; @@ -1051,19 +1350,16 @@ bool DeepParser::splitByRegex(const std::string& val, const Regex& r, const char dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::splitByRegex(): splitting '" - << val - << "' keyPrefix='" + << val << "' keyPrefix='" << keyPrefix << "' into " << matches.size() << "u pieces ..."; size_t splitIndex = 0; - for (size_t i = 0; i < matches.size(); ++i) - { - for (size_t j = 0; j < matches[i].groups.size(); ++j) - { - RegexMatch::MatchGroup& g = matches[i].groups[j]; + for (size_t i = 0; i < matches.size(); ++i) { + for (size_t j = 0; j < matches[i].groups.size(); ++j) { + RegexMatch::MatchGroup &g = matches[i].groups[j]; char nbuf[64]; snprintf(nbuf, sizeof(nbuf), "%s", keyPrefix); dbgTrace(D_WAAP_DEEP_PARSER) @@ -1082,7 +1378,14 @@ bool DeepParser::splitByRegex(const std::string& val, const Regex& r, const char m_splitTypesStack.push(std::make_tuple(m_depth, splitIndex++, std::string(keyPrefix))); if (!g.value.empty()) { // Send non-empty split parts to deeper scanning - onKv(nbuf, strlen(nbuf), g.value.data(), g.value.size(), BUFFERED_RECEIVER_F_BOTH); + onKv( + nbuf, + strlen(nbuf), + g.value.data(), + g.value.size(), + BUFFERED_RECEIVER_F_BOTH, + parser_depth + ); } // Forget most recent split type m_splitTypesStack.pop(); @@ -1099,17 +1402,20 @@ bool DeepParser::splitByRegex(const std::string& val, const Regex& r, const char return splitDone; } -void DeepParser::setMultipartBoundary(const std::string &boundary) +void +DeepParser::setMultipartBoundary(const std::string &boundary) { m_multipart_boundary = boundary; } -const std::string &DeepParser::getMultipartBoundary() const +const std::string & +DeepParser::getMultipartBoundary() const { return m_multipart_boundary; } -bool DeepParser::isBinaryData() const +bool +DeepParser::isBinaryData() const { for (const auto &parser : m_parsersDeque) { if (parser->name() == "binary") return true; @@ -1117,58 +1423,62 @@ bool DeepParser::isBinaryData() const return false; } -const std::string DeepParser::getLastParser() const +const std::string +DeepParser::getActualParser(size_t parser_depth) const { if (m_parsersDeque.empty()) { return ""; - } - else { - return m_parsersDeque.front()->name(); + } else { + return m_parsersDeque.at(parser_depth)->name(); } } -bool DeepParser::isWBXmlData() const +bool +DeepParser::isWBXmlData() const { return m_is_wbxml; } -Maybe DeepParser::getSplitType() const +Maybe +DeepParser::getSplitType() const { dbgTrace(D_WAAP_DEEP_PARSER) << "getSplitType: enter. current m_depth=" << m_depth; if (!m_splitTypesStack.empty()) { dbgTrace(D_WAAP_DEEP_PARSER) - << "getSplitType: stack top: (depth=" << std::get<0>(m_splitTypesStack.top()) - << ", splitIndex=" << std::get<1>(m_splitTypesStack.top()) - << ", splitType='" << std::get<2>(m_splitTypesStack.top()) + << "getSplitType: stack top: (depth=" + << std::get<0>(m_splitTypesStack.top()) + << ", splitIndex=" + << std::get<1>(m_splitTypesStack.top()) + << ", splitType='" + << std::get<2>(m_splitTypesStack.top()) << "')"; - } - else { + } else { dbgTrace(D_WAAP_DEEP_PARSER) << "getSplitType: stack is empty"; } // Return only immediate split type. Ignore additional levels of parsers inside splitted item, and ignore // any first item in the splitted value (ex. "id;ls" -> "id" is first item in split list and hence ignored) - if ( - m_splitTypesStack.empty() || - std::get<0>(m_splitTypesStack.top()) != m_depth || - std::get<1>(m_splitTypesStack.top()) == 0 - ) { + if (m_splitTypesStack.empty() + || std::get<0>(m_splitTypesStack.top()) != m_depth + || std::get<1>(m_splitTypesStack.top()) == 0 + ) { dbgTrace(D_WAAP_DEEP_PARSER) << "getSplitType: returning empty string"; return genError("should not be split"); } return std::get<2>(m_splitTypesStack.top()); } -bool DeepParser::isGlobalMaxObjectDepthReached() const +bool +DeepParser::isGlobalMaxObjectDepthReached() const { return m_globalMaxObjectDepthReached; } -bool DeepParser::shouldEnforceDepthLimit(const std::shared_ptr& parser) const +bool +DeepParser::shouldEnforceDepthLimit(const std::shared_ptr &parser) const { - const std::string& name = parser->name(); - if ((name == ParserJson::m_parserName) || - (name == ParserXML::m_parserName)) { + const std::string &name = parser->name(); + if ((name == ParserJson::m_parserName) || (name == ParserXML::m_parserName)) { return true; } return false; diff --git a/components/security_apps/waap/waap_clib/DeepParser.h b/components/security_apps/waap/waap_clib/DeepParser.h index ff80d6a..6cf5504 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.h +++ b/components/security_apps/waap/waap_clib/DeepParser.h @@ -31,7 +31,7 @@ public: void setWaapAssetState(std::shared_ptr pWaapAssetState); // This callback receives input key/value pairs, dissects, decodes and deep-scans these, recursively // finally, it calls onDetected() on each detected parameter. - virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags); + virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth); void clear(); void showStats(std::string& buff, const ValueStatsAnalyzer& valueStats); @@ -44,7 +44,7 @@ public: void setMultipartBoundary(const std::string &boundary); const std::string &getMultipartBoundary() const; bool isBinaryData() const; - const std::string getLastParser() const; + const std::string getActualParser(size_t parser_depth) const; bool isWBXmlData() const; Maybe getSplitType() const; std::vector > kv_pairs; @@ -94,6 +94,7 @@ public: std::vector m_keywordInfo; KeyStack m_key; + int getShiftInUrlEncodedBuffer(const ValueStatsAnalyzer &valueStats, std::string &cur_val); private: class Ref @@ -115,18 +116,60 @@ private: // Split a value by given regexp. Return true if split, false otherwise. // note: This function calls onKv(), and the call can be recursive! // TODO:: maybe convert this splitter to Parser-derived class?! - bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix); - void createInternalParser(const char *k, size_t k_len, std::string& cur_val, + bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix, size_t parser_depth); + + int createInternalParser( + const char *k, + size_t k_len, + std::string &cur_val, const ValueStatsAnalyzer &valueStats, bool isBodyPayload, bool isRefererPayload, bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, - int flags); - int pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound); - int parseBuffer(ValueStatsAnalyzer& valueStats, const std::string &cur_val, bool base64ParamFound, - bool shouldUpdateKeyStack); + int flags, + size_t parser_depth + ); + + int createUrlParserForJson( + const char *k, + size_t k_len, + std::string &cur_val, + const ValueStatsAnalyzer &valueStats, + bool isBodyPayload, + bool isRefererPayload, + bool isRefererParamPayload, + bool isUrlPayload, + bool isUrlParamPayload, + int flags, + size_t parser_depth + ); + + void printParserDeque(); + + int parseAfterMisleadingMultipartBoundaryCleaned( + const char *k, + size_t k_len, + std::string &cur_val, + const ValueStatsAnalyzer &valueStats, + bool isBodyPayload, + bool isRefererPayload, + bool isRefererParamPayload, + bool isUrlPayload, + bool isUrlParamPayload, + int flags, + size_t parser_depth, + bool base64ParamFound + ); + int pushValueToTopParser(std::string &cur_val, int flags, bool base64ParamFound, int offset, size_t parser_depth); + int parseBuffer( + ValueStatsAnalyzer &valueStats, + const std::string &cur_val, + bool base64ParamFound, + bool shouldUpdateKeyStack, + size_t parser_depth + ); bool shouldEnforceDepthLimit(const std::shared_ptr& parser) const; void setLocalMaxObjectDepth(size_t depth) { m_localMaxObjectDepth = depth; } void setGlobalMaxObjectDepthReached() { m_globalMaxObjectDepthReached = true; } diff --git a/components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc b/components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc index 8c8fee1..3439c8a 100644 --- a/components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc +++ b/components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc @@ -19,9 +19,15 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PHPSERIALIZE); const std::string PHPSerializedDataParser::m_parserName = "PHPSerializedDataParser"; -PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver) -: m_state(), m_outReceiver(outReceiver), m_keyStack("php_serialized") +PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth) : + m_state(), + m_outReceiver(outReceiver), + m_keyStack("php_serialized"), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER_PHPSERIALIZE) + << "parser_depth=" + << parser_depth; } size_t @@ -344,7 +350,7 @@ size_t PHPSerializedDataParser::handleValue (const char &c) // else will parse it normaly. dbgTrace(D_WAAP_PARSER_PHPSERIALIZE) << "PHPSerializedDataParser::push(): End of Class object" << " sending class object data to PHPSerializedDataParser"; - PHPSerializedDataParser psdp(m_outReceiver); + PHPSerializedDataParser psdp(m_outReceiver, m_parser_depth); psdp.push(m_value.c_str(), m_value.length()); if(psdp.error()) { diff --git a/components/security_apps/waap/waap_clib/PHPSerializedDataParser.h b/components/security_apps/waap/waap_clib/PHPSerializedDataParser.h index 5afede1..8ee8665 100644 --- a/components/security_apps/waap/waap_clib/PHPSerializedDataParser.h +++ b/components/security_apps/waap/waap_clib/PHPSerializedDataParser.h @@ -20,7 +20,7 @@ class PHPSerializedDataParser : public ParserBase { public: - PHPSerializedDataParser(IParserStreamReceiver &outReceiver); + PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth); size_t push(const char* buf, size_t len); void finish(); virtual const std::string &name() const; @@ -84,6 +84,6 @@ private: IParserStreamReceiver &m_outReceiver; KeyStack m_keyStack; std::stack m_stack; - + size_t m_parser_depth; static const std::string m_parserName; }; diff --git a/components/security_apps/waap/waap_clib/ParserBase.cc b/components/security_apps/waap/waap_clib/ParserBase.cc index 514b3ca..12ba64a 100755 --- a/components/security_apps/waap/waap_clib/ParserBase.cc +++ b/components/security_apps/waap/waap_clib/ParserBase.cc @@ -13,19 +13,38 @@ #include "ParserBase.h" #include +#include "debug.h" + +USE_DEBUG_FLAG(D_WAAP_PARSER); // Max size for key and value that can be stored in memory (per thread) #define MAX_KEY_SIZE 64*1024 #define MAX_VALUE_SIZE 64*1024 -BufferedReceiver::BufferedReceiver(IParserReceiver &receiver) -:m_receiver(receiver), -m_flags(BUFFERED_RECEIVER_F_FIRST) +BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) : + m_receiver(receiver), + m_flags(BUFFERED_RECEIVER_F_FIRST), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER) + << "parser_depth=" + << parser_depth; } int BufferedReceiver::onKey(const char *k, size_t k_len) { + dbgTrace(D_WAAP_PARSER) + << "Entering BufferedReceiver::onKey with values: \n" + << "\tkey = " + << k + << "\n\tk_len = " + << k_len + << "\n\tm_key = " + << m_key + << "\n\t m_key_size = " + << m_key.size() + << "\n\t parser_depth = " + << m_parser_depth; if (m_key.size() + k_len < MAX_KEY_SIZE) { m_key += std::string(k, k_len); } @@ -36,7 +55,18 @@ int BufferedReceiver::onKey(const char *k, size_t k_len) int BufferedReceiver::onValue(const char *v, size_t v_len) { int rc = 0; - + dbgTrace(D_WAAP_PARSER) + << "Entering BufferedReceiver::onValue with values: \n" + << "\tvalue = " + << v + << "\n\tv_len = " + << v_len + << "\n\tm_value = " + << m_value + << "\n\t m_value_size = " + << m_value.size() + << "\n\t parser_depth = " + << m_parser_depth; while (v_len > 0) { // Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size()); @@ -48,7 +78,18 @@ int BufferedReceiver::onValue(const char *v, size_t v_len) // Only push full buffers to the m_receiver if (m_value.size() == MAX_VALUE_SIZE) { // The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag - int tempRc= m_receiver.onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags); + dbgTrace(D_WAAP_PARSER) + << " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag" + << "calling onKv() while m_flags = " + << m_flags; + int tempRc= m_receiver.onKv( + m_key.data(), + m_key.size(), + m_value.data(), + m_value.size(), + m_flags, + m_parser_depth + ); if (tempRc != 0) { rc = tempRc; } @@ -68,16 +109,19 @@ int BufferedReceiver::onKvDone() // Call onKv on the remainder of the buffer not yet pushed to the receiver // This must be called even if m_value is empty in order to signal the BUFFERED_RECEIVER_F_LAST flag to the // receiver! - int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags); + dbgTrace(D_WAAP_PARSER) + << " Call onKv on the remainder of the buffer not yet pushed to the receiver " + << "calling onKv()"; + int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags, m_parser_depth); // Reset the object's state to allow reuse for other parsers clear(); return rc; } -int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) +int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) { - return m_receiver.onKv(k, k_len, v, v_len, flags); + return m_receiver.onKv(k, k_len, v, v_len, flags, parser_depth); } void BufferedReceiver::clear() diff --git a/components/security_apps/waap/waap_clib/ParserBase.h b/components/security_apps/waap/waap_clib/ParserBase.h index d49b3cd..4d7d9e3 100755 --- a/components/security_apps/waap/waap_clib/ParserBase.h +++ b/components/security_apps/waap/waap_clib/ParserBase.h @@ -33,7 +33,7 @@ // Interface for receiver classes that accept full key/value pairs struct IParserReceiver { - virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) = 0; + virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) = 0; }; struct IParserReceiver2 { @@ -80,11 +80,11 @@ struct IParserStreamReceiver : public IParserReceiver { // This will in many cases cause sub-parsers to also work in zero-copy style too! class BufferedReceiver : public IParserStreamReceiver { public: - BufferedReceiver(IParserReceiver &receiver); + BufferedReceiver(IParserReceiver &receiver, size_t parser_depth=0); virtual int onKey(const char *k, size_t k_len); virtual int onValue(const char *v, size_t v_len); virtual int onKvDone(); - virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags); + virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth); virtual void clear(); // Helper methods to access accumulated key and value (read-only) @@ -97,6 +97,8 @@ private: // Accumulated key/value pair std::string m_key; std::string m_value; + size_t m_parser_depth; + }; // Base class for various streaming parsers that accept data stream in multiple pieces through @@ -123,10 +125,11 @@ class BufferedParser : public ParserBase { public: template - explicit BufferedParser(IParserReceiver &receiver, _Args... _args) + explicit BufferedParser(IParserReceiver &receiver, size_t parser_depth, _Args... _args) : - m_bufferedReceiver(receiver), - m_parser(m_bufferedReceiver, _args...) // pass any extra arguments to specific parser's constructor + m_bufferedReceiver(receiver, parser_depth), + // pass any extra arguments to specific parser's constructor + m_parser(m_bufferedReceiver, parser_depth, _args...) {} virtual ~BufferedParser() {} virtual size_t push(const char *data, size_t data_len) { return m_parser.push(data, data_len); } diff --git a/components/security_apps/waap/waap_clib/ParserBinary.cc b/components/security_apps/waap/waap_clib/ParserBinary.cc index e9b2052..371a3d2 100755 --- a/components/security_apps/waap/waap_clib/ParserBinary.cc +++ b/components/security_apps/waap/waap_clib/ParserBinary.cc @@ -18,13 +18,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_BINARY); #define MIN_TEXT_SIZE 10 -ParserBinary::ParserBinary(IParserStreamReceiver& receiver) : +ParserBinary::ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth) : m_parserName("binary"), m_receiver(receiver), m_state(s_start), m_textFromLastBuffer(), - m_textCharCount(0) + m_textCharCount(0), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER_BINARY) + << "parser_depth=" + << parser_depth; } ParserBinary::~ParserBinary() diff --git a/components/security_apps/waap/waap_clib/ParserBinary.h b/components/security_apps/waap/waap_clib/ParserBinary.h index 81a1c14..f78b9fb 100755 --- a/components/security_apps/waap/waap_clib/ParserBinary.h +++ b/components/security_apps/waap/waap_clib/ParserBinary.h @@ -19,7 +19,7 @@ class ParserBinary : public ParserBase { public: - ParserBinary(IParserStreamReceiver& receiver); + ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth); virtual ~ParserBinary(); virtual size_t push(const char* data, size_t data_len); virtual void finish(); @@ -39,7 +39,7 @@ private: state m_state; std::string m_textFromLastBuffer; size_t m_textCharCount; - + size_t m_parser_depth; void flush(); }; diff --git a/components/security_apps/waap/waap_clib/ParserConfluence.cc b/components/security_apps/waap/waap_clib/ParserConfluence.cc index 3796938..3aab98b 100755 --- a/components/security_apps/waap/waap_clib/ParserConfluence.cc +++ b/components/security_apps/waap/waap_clib/ParserConfluence.cc @@ -16,12 +16,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_CONFLUENCE); -ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver) : +ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth) : m_parserName("confluence"), m_state(s_start), m_receiver(receiver), - m_name() + m_name(), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER_CONFLUENCE) + << "parser_depth=" + << parser_depth; + } ParserConfluence::~ParserConfluence() diff --git a/components/security_apps/waap/waap_clib/ParserConfluence.h b/components/security_apps/waap/waap_clib/ParserConfluence.h index f2af07a..8129948 100755 --- a/components/security_apps/waap/waap_clib/ParserConfluence.h +++ b/components/security_apps/waap/waap_clib/ParserConfluence.h @@ -19,7 +19,7 @@ class ParserConfluence : public ParserBase { public: - ParserConfluence(IParserStreamReceiver& receiver); + ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth); virtual ~ParserConfluence(); virtual size_t push(const char* data, size_t data_len); @@ -43,6 +43,7 @@ private: state m_state; IParserStreamReceiver& m_receiver; std::string m_name; + size_t m_parser_depth; }; #endif diff --git a/components/security_apps/waap/waap_clib/ParserDelimiter.cc b/components/security_apps/waap/waap_clib/ParserDelimiter.cc index 64ab0b1..1800ea0 100755 --- a/components/security_apps/waap/waap_clib/ParserDelimiter.cc +++ b/components/security_apps/waap/waap_clib/ParserDelimiter.cc @@ -16,15 +16,22 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_DELIMITER); -ParserDelimiter::ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName) - : ParserBase(), +ParserDelimiter::ParserDelimiter( + IParserStreamReceiver& receiver, + size_t parser_depth, + char delim, + const std::string& delimName + ) : ParserBase(), m_state(s_start), m_receiver(receiver), m_delim(delim), m_delim_name(delimName), - m_found_delim(false) + m_found_delim(false), + m_parser_depth(parser_depth) { - + dbgTrace(D_WAAP_PARSER_DELIMITER) + << "parsing delimiter: parser depth=" + << parser_depth; } ParserDelimiter::~ParserDelimiter() diff --git a/components/security_apps/waap/waap_clib/ParserDelimiter.h b/components/security_apps/waap/waap_clib/ParserDelimiter.h index c463576..b1903a0 100755 --- a/components/security_apps/waap/waap_clib/ParserDelimiter.h +++ b/components/security_apps/waap/waap_clib/ParserDelimiter.h @@ -19,7 +19,7 @@ class ParserDelimiter : public ParserBase { public: - ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName); + ParserDelimiter(IParserStreamReceiver& receiver, size_t parser_depth, char delim, const std::string& delimName); virtual ~ParserDelimiter(); virtual size_t push(const char* data, size_t data_len); @@ -45,6 +45,7 @@ private: char m_delim; std::string m_delim_name; bool m_found_delim; + size_t m_parser_depth; }; diff --git a/components/security_apps/waap/waap_clib/ParserGql.cc b/components/security_apps/waap/waap_clib/ParserGql.cc index 294cfba..0b75d18 100644 --- a/components/security_apps/waap/waap_clib/ParserGql.cc +++ b/components/security_apps/waap/waap_clib/ParserGql.cc @@ -21,12 +21,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_GQL); const std::string ParserGql::m_parserName = "gqlParser"; -ParserGql::ParserGql(IParserReceiver& receiver) : +ParserGql::ParserGql(IParserReceiver &receiver, size_t parser_depth) : m_receiver(receiver), m_error(false), - m_curNameValues(0) + m_curNameValues(0), + m_parser_depth(parser_depth) { dbgFlow(D_WAAP_PARSER_GQL); + dbgTrace(D_WAAP_PARSER_GQL) << "parser_depth=" << parser_depth; } ParserGql::~ParserGql() { @@ -56,7 +58,9 @@ size_t ParserGql::push(const char* buf, size_t len) { // Handle corner case of last name visited without value: don't forget to output that name too if (m_curNameValues == 0 && !m_curNodeName.empty()) { dbgTrace(D_WAAP_PARSER_GQL) << "handle last name: '" << m_curNodeName << "'"; - if (m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH) != 0) { + if (m_receiver.onKv( + m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth + ) != 0) { m_error = true; } } @@ -81,7 +85,9 @@ bool ParserGql::visitValue(const char *value) { dbgTrace(D_WAAP_PARSER_GQL) << "'" << value << "'"; m_curNameValues++; - return m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH); + return m_receiver.onKv( + m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH, m_parser_depth + ); } bool ParserGql::visitName(const facebook::graphql::ast::Name &node) @@ -89,7 +95,9 @@ bool ParserGql::visitName(const facebook::graphql::ast::Name &node) dbgTrace(D_WAAP_PARSER_GQL) << node.getValue() << "'"; bool ret = true; if (m_curNameValues == 0 && !m_curNodeName.empty()) { - ret = m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH); + ret = m_receiver.onKv( + m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth + ); } // wait for next name m_curNodeName = std::string(node.getValue()); diff --git a/components/security_apps/waap/waap_clib/ParserGql.h b/components/security_apps/waap/waap_clib/ParserGql.h index 0215e1c..1616bf5 100644 --- a/components/security_apps/waap/waap_clib/ParserGql.h +++ b/components/security_apps/waap/waap_clib/ParserGql.h @@ -25,7 +25,7 @@ class ParserGql : public ParserBase, public facebook::graphql::ast::visitor::AstVisitor { public: - ParserGql(IParserReceiver &receiver); + ParserGql(IParserReceiver &receiver, size_t parser_depth); virtual ~ParserGql(); size_t push(const char *data, size_t data_len); void finish(); @@ -51,6 +51,7 @@ private: bool visitEnumValue(const facebook::graphql::ast::EnumValue &node) override; public: static const std::string m_parserName; + size_t m_parser_depth; }; #endif // __PARSER_JQL_H diff --git a/components/security_apps/waap/waap_clib/ParserHTML.cc b/components/security_apps/waap/waap_clib/ParserHTML.cc index e23641e..b74f242 100755 --- a/components/security_apps/waap/waap_clib/ParserHTML.cc +++ b/components/security_apps/waap/waap_clib/ParserHTML.cc @@ -38,16 +38,24 @@ void ParserHTML::onStartElement( if (attr_value == NULL) { attr_value = (const xmlChar*)""; } - - dbgTrace(D_WAAP_PARSER_HTML) << "\tHTML ATTR: elem='" << (char*)localname << "', " << attr_localname << - "='" << std::string((char*)attr_value) << "'"; - p->m_key.push((const char*)attr_localname, xmlStrlen(attr_localname)); + + dbgTrace(D_WAAP_PARSER_HTML) + << "\tHTML ATTR: elem='" + << (char *)localname + << "', " + << attr_localname + << "='" + << std::string((char *)attr_value) + << "'"; + p->m_key.push((const char *)attr_localname, xmlStrlen(attr_localname)); if (p->m_receiver.onKv( - p->m_key.first().c_str(), - p->m_key.first().size(), - (const char*)attr_value, strlen((const char*)attr_value), - BUFFERED_RECEIVER_F_BOTH - ) != 0) { + p->m_key.first().c_str(), + p->m_key.first().size(), + (const char *)attr_value, + strlen((const char *)attr_value), + BUFFERED_RECEIVER_F_BOTH, + p->m_parser_depth + ) != 0) { p->m_state = s_error; } p->m_key.pop("HTML end attribute"); @@ -73,8 +81,8 @@ ParserHTML::onEndElement( dbgTrace(D_WAAP_PARSER_HTML) << "HTML CLOSE: '" << localname << "'"; if (p->m_elemTrackStack.empty()) { - dbgWarning(D_WAAP_PARSER_HTML) << - "HTML closing tag and elem track stack is empty. This is probably sign of a bug!"; + dbgWarning(D_WAAP_PARSER_HTML) + << "HTML closing tag and elem track stack is empty. This is probably sign of a bug!"; return; } @@ -111,8 +119,10 @@ ParserHTML::onEndElement( p->m_key.pop("HTML end element"); } -void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) { - ParserHTML* p = (ParserHTML*)ctx; +void +ParserHTML::onCharacters(void *ctx, const xmlChar *ch, int len) +{ + ParserHTML *p = (ParserHTML *)ctx; if (p->m_elemTrackStack.empty()) { dbgWarning(D_WAAP_PARSER_HTML) << "HTML text and elem track stack is empty. This is probably sign of a bug!"; @@ -141,7 +151,9 @@ void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) { elemTrackInfo.value += val; } -static void onError(void* ctx, const char* msg, ...) { +static void +onError(void *ctx, const char *msg, ...) +{ static const size_t TMP_BUF_SIZE = 4096; char string[TMP_BUF_SIZE]; va_list arg_ptr; @@ -152,9 +164,19 @@ static void onError(void* ctx, const char* msg, ...) { dbgTrace(D_WAAP_PARSER_HTML) << "LIBXML (html) onError: " << std::string(string); } -ParserHTML::ParserHTML(IParserStreamReceiver& receiver) - :m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("html_parser"), m_pushParserCtxPtr(NULL) { - dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::ParserHTML()"; +ParserHTML::ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth) : + m_receiver(receiver), + m_state(s_start), + m_bufLen(0), + m_key("html_parser"), + m_pushParserCtxPtr(NULL), + m_parser_depth(parser_depth) +{ + dbgTrace(D_WAAP_PARSER_HTML) + << "ParserHTML::ParserHTML()" + << "parser_depth=" + << parser_depth; + // TODO:: is zeroing this really needed? memset(m_buf, 0, sizeof(m_buf)); @@ -173,7 +195,8 @@ ParserHTML::ParserHTML(IParserStreamReceiver& receiver) m_key.push("html", 4); } -ParserHTML::~ParserHTML() { +ParserHTML::~ParserHTML() +{ // Cleanup HTML dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::~ParserHTML()"; @@ -182,9 +205,15 @@ ParserHTML::~ParserHTML() { } } -bool ParserHTML::filterErrors(xmlErrorPtr xmlError) { - dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): xmlError " << xmlError->code << ": '" << - xmlError->message << "'"; +bool +ParserHTML::filterErrors(xmlErrorPtr xmlError) +{ + dbgDebug(D_WAAP_PARSER_HTML) + << "ParserHTML::filterErrors(): xmlError " + << xmlError->code + << ": '" + << xmlError->message + << "'"; // Ignore specific error: "HTML declaration allowed only at the start of the document". // This includes the case of "multiple HTML declarations" we've seen sent by some SOAP clients. @@ -193,15 +222,21 @@ bool ParserHTML::filterErrors(xmlErrorPtr xmlError) { // Ignoring this error prevents the WAAP code from thinking the HTML is "broken" and from scanning the HTML // source as-is, in effect preventing false alarm on that HTML source. if (xmlError->code == XML_ERR_RESERVED_XML_NAME || xmlError->code == XML_ERR_UNDECLARED_ENTITY) { - dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): ignoring the '" << xmlError->code << ": " << - xmlError->message << "' html parser error."; + dbgDebug(D_WAAP_PARSER_HTML) + << "ParserHTML::filterErrors(): ignoring the '" + << xmlError->code + << ": " + << xmlError->message + << "' html parser error."; return false; } return true; } -size_t ParserHTML::push(const char* data, size_t data_len) { +size_t +ParserHTML::push(const char *data, size_t data_len) +{ size_t i = 0; char c; @@ -212,8 +247,12 @@ size_t ParserHTML::push(const char* data, size_t data_len) { if (htmlParseChunk(m_pushParserCtxPtr, m_buf, 0, 1)) { xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr); if (xmlError && filterErrors(xmlError)) { - dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code << ": '" << - xmlError->message << "'"; + dbgDebug(D_WAAP_PARSER_HTML) + << "ParserHTML::push(): xmlError: code=" + << xmlError->code + << ": '" + << xmlError->message + << "'"; m_state = s_error; // error return -1; } @@ -232,8 +271,13 @@ size_t ParserHTML::push(const char* data, size_t data_len) { // fall through // CP_FALL_THROUGH; case s_accumulate_first_bytes: - dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_accumulate_first_bytes. c='" << data[i] << - "'; m_bufLen=" << m_bufLen << "; i=" << i; + dbgTrace(D_WAAP_PARSER_HTML) + << "ParserHTML::push(): s_accumulate_first_bytes. c='" + << data[i] + << "'; m_bufLen=" + << m_bufLen + << "; i=" + << i; m_buf[m_bufLen] = c; m_bufLen++; if (c == '?') { @@ -244,13 +288,19 @@ size_t ParserHTML::push(const char* data, size_t data_len) { } break; - case s_start_parsing: - dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_start_parsing. sending len=" << m_bufLen << ": '" << - std::string(m_buf, m_bufLen) << "'; i=" << i; - // Create HTML SAX (push parser) context - // It is important to buffer at least first 4 bytes of input stream so libxml can determine text encoding! - m_pushParserCtxPtr = htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL, - XML_CHAR_ENCODING_UTF8); + case s_start_parsing: + dbgTrace(D_WAAP_PARSER_HTML) + << "ParserHTML::push(): s_start_parsing. sending len=" + << m_bufLen + << ": '" + << std::string(m_buf, m_bufLen) + << "'; i=" + << i; + // Create HTML SAX (push parser) context + // It is important to buffer at least first 4 bytes of input stream so libxml can determine text + // encoding! + m_pushParserCtxPtr = + htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL, XML_CHAR_ENCODING_UTF8); // Enable "permissive mode" for HTML SAX parser. // In this mode, the libxml parser doesn't stop on errors, but still reports them! @@ -261,14 +311,23 @@ size_t ParserHTML::push(const char* data, size_t data_len) { // fall through // CP_FALL_THROUGH; case s_parsing: - dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_parsing. sending len=" << (int)(data_len - i) << - ": '" << std::string(data + i, data_len - i) << "'; i=" << i; + dbgTrace(D_WAAP_PARSER_HTML) + << "ParserHTML::push(): s_parsing. sending len=" + << (int)(data_len - i) + << ": '" + << std::string(data + i, data_len - i) + << "'; i=" + << i; if (m_pushParserCtxPtr) { if (htmlParseChunk(m_pushParserCtxPtr, data + i, data_len - i, 0)) { xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr); if (xmlError && filterErrors(xmlError)) { - dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code << - ": '" << xmlError->message << "'"; + dbgDebug(D_WAAP_PARSER_HTML) + << "ParserHTML::push(): xmlError: code=" + << xmlError->code + << ": '" + << xmlError->message + << "'"; m_state = s_error; // error return 0; } @@ -290,15 +349,20 @@ size_t ParserHTML::push(const char* data, size_t data_len) { return i; } -void ParserHTML::finish() { +void +ParserHTML::finish() +{ push(NULL, 0); } const std::string & -ParserHTML::name() const { +ParserHTML::name() const +{ return m_parserName; } -bool ParserHTML::error() const { +bool +ParserHTML::error() const +{ return m_state == s_error; } diff --git a/components/security_apps/waap/waap_clib/ParserHTML.h b/components/security_apps/waap/waap_clib/ParserHTML.h index 2459141..692ad7b 100755 --- a/components/security_apps/waap/waap_clib/ParserHTML.h +++ b/components/security_apps/waap/waap_clib/ParserHTML.h @@ -24,7 +24,7 @@ class ParserHTML : public ParserBase { public: - ParserHTML(IParserStreamReceiver &receiver); + ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth); virtual ~ParserHTML(); size_t push(const char *data, size_t data_len); void finish(); @@ -81,4 +81,5 @@ private: htmlParserCtxtPtr m_pushParserCtxPtr; static const std::string m_parserName; + size_t m_parser_depth; }; diff --git a/components/security_apps/waap/waap_clib/ParserJson.cc b/components/security_apps/waap/waap_clib/ParserJson.cc index 91e1190..3959c7a 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.cc +++ b/components/security_apps/waap/waap_clib/ParserJson.cc @@ -20,17 +20,21 @@ #include USE_DEBUG_FLAG(D_WAAP_PARSER_JSON); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); const std::string ParserJson::m_parserName = "jsonParser"; -int ParserJson::cb_null() { +int +ParserJson::cb_null() +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_null():"; + if (m_receiver2) { m_receiver2->onKvt(m_key.c_str(), m_key.size(), "null", 4, DataType::EMPTY); } - if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH)) { + if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) { return 0; } @@ -41,20 +45,22 @@ int ParserJson::cb_null() { return 1; } -int ParserJson::cb_boolean(int boolean) { +int +ParserJson::cb_boolean(int boolean) +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_boolean(): " << boolean; + if (m_receiver2) { m_receiver2->onKvt(m_key.c_str(), m_key.size(), NULL, boolean, DataType::BOOLEAN); } if (boolean) { - if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH)) { + if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) { return 0; } - } - else { - if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH)) { + } else { + if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) { return 0; } } @@ -65,14 +71,16 @@ int ParserJson::cb_boolean(int boolean) { return 1; } -int ParserJson::cb_number(const char* s, yajl_size_t slen) { +int +ParserJson::cb_number(const char *s, yajl_size_t slen) +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_number(): '" << std::string(s, slen) << "'"; if (m_receiver2) { m_receiver2->onKvt(m_key.c_str(), m_key.size(), s, slen, DataType::NUMBER); } - if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH)) { + if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) { return 0; } @@ -82,14 +90,19 @@ int ParserJson::cb_number(const char* s, yajl_size_t slen) { return 1; } -int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) { - dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char*)s, slen) << "'"; +int +ParserJson::cb_string(const unsigned char *s, yajl_size_t slen) +{ + dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char *)s, slen) << "'"; if (m_receiver2) { m_receiver2->onKvt(m_key.c_str(), m_key.size(), (const char*)s, slen, DataType::STRING); } - if (m_receiver.onKv(m_key.c_str(), m_key.size(), (const char*)s, slen, BUFFERED_RECEIVER_F_BOTH)) { + + if (m_receiver.onKv( + m_key.c_str(), m_key.size(), (const char *)s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth + )) { return 0; } @@ -99,8 +112,10 @@ int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) { return 1; } -int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) { - dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char*)s, slen) << "'"; +int +ParserJson::cb_map_key(const unsigned char *s, yajl_size_t slen) +{ + dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char *)s, slen) << "'"; m_key.push((char*)s, slen); @@ -111,7 +126,9 @@ int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) { return 1; } -int ParserJson::cb_start_map() { +int +ParserJson::cb_start_map() +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_map():"; if (m_receiver2) { @@ -122,7 +139,9 @@ int ParserJson::cb_start_map() { return 1; } -int ParserJson::cb_end_map() { +int +ParserJson::cb_end_map() +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_map():"; if (m_receiver2) { @@ -140,7 +159,9 @@ int ParserJson::cb_end_map() { return 1; } -int ParserJson::cb_start_array() { +int +ParserJson::cb_start_array() +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_array():"; if (m_receiver2) { @@ -151,7 +172,9 @@ int ParserJson::cb_start_array() { return 1; } -int ParserJson::cb_end_array() { +int +ParserJson::cb_end_array() +{ dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_array():"; if (m_receiver2) { @@ -162,6 +185,7 @@ int ParserJson::cb_end_array() { m_depthStack.pop_back(); } + if (!m_depthStack.empty() && m_depthStack.back() == js_map) { m_key.pop("json end array"); } @@ -169,51 +193,78 @@ int ParserJson::cb_end_array() { } // Static functions to be called from C and forward the calls to respective class cb_* methods -int ParserJson::p_null(void* ctx) +int +ParserJson::p_null(void *ctx) { return ((ParserJson*)ctx)->cb_null(); } -int ParserJson::p_boolean(void* ctx, int boolean) + +int +ParserJson::p_boolean(void *ctx, int boolean) { return ((ParserJson*)ctx)->cb_boolean(boolean); } -int ParserJson::p_number(void* ctx, const char* s, yajl_size_t slen) + +int +ParserJson::p_number(void *ctx, const char *s, yajl_size_t slen) { return ((ParserJson*)ctx)->cb_number(s, slen); } -int ParserJson::p_string(void* ctx, const unsigned char* s, yajl_size_t slen) + +int +ParserJson::p_string(void *ctx, const unsigned char *s, yajl_size_t slen) { return ((ParserJson*)ctx)->cb_string(s, slen); } -int ParserJson::p_map_key(void* ctx, const unsigned char* s, yajl_size_t slen) + +int +ParserJson::p_map_key(void *ctx, const unsigned char *s, yajl_size_t slen) { return ((ParserJson*)ctx)->cb_map_key(s, slen); } -int ParserJson::p_start_map(void* ctx) + +int +ParserJson::p_start_map(void *ctx) { return ((ParserJson*)ctx)->cb_start_map(); } -int ParserJson::p_end_map(void* ctx) + +int +ParserJson::p_end_map(void *ctx) { return ((ParserJson*)ctx)->cb_end_map(); } -int ParserJson::p_start_array(void* ctx) + +int +ParserJson::p_start_array(void *ctx) { return ((ParserJson*)ctx)->cb_start_array(); } -int ParserJson::p_end_array(void* ctx) + +int +ParserJson::p_end_array(void *ctx) { return ((ParserJson*)ctx)->cb_end_array(); } -ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) : +ParserJson::ParserJson( + IParserReceiver &receiver, + bool should_collect_oas, + size_t parser_depth, + IParserReceiver2 *receiver2) + : m_receiver(receiver), m_receiver2(receiver2), m_state(s_start), m_bufLen(0), m_key("json_parser"), - m_jsonHandler(NULL) + m_jsonHandler(NULL), + is_map_empty(false), + should_collect_for_oa_schema_updater(should_collect_oas), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER_JSON) << "parser_depth= " << parser_depth; + // TODO:: do we really want to clear this? memset(m_buf, 0, sizeof(m_buf)); @@ -232,7 +283,7 @@ ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) : }; m_jsonHandler = yajl_alloc(&callbacks, NULL, this); - + if (m_jsonHandler == NULL) { dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::ParserJson(): yajl_alloc() failed. Switching to s_error state."; m_state = s_error; @@ -249,7 +300,8 @@ ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) : m_key.push("json", 4); } -ParserJson::~ParserJson() { +ParserJson::~ParserJson() +{ // Cleanup JSON dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::~ParserJson():"; @@ -258,7 +310,9 @@ ParserJson::~ParserJson() { } } -size_t ParserJson::push(const char* buf, size_t len) { +size_t +ParserJson::push(const char *buf, size_t len) +{ size_t i = 0; char c; @@ -280,6 +334,7 @@ size_t ParserJson::push(const char* buf, size_t len) { while (i < len) { c = buf[i]; + switch (m_state) { case s_start: dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start"; @@ -288,8 +343,12 @@ size_t ParserJson::push(const char* buf, size_t len) { // fallthrough // CP_FALL_THROUGH; case s_accumulate_first_bytes: - dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_accumulate_first_bytes. i=" << i << - " c='" << buf[i] << "'"; + dbgTrace(D_WAAP_PARSER_JSON) + << "ParserJson::push(): s_accumulate_first_bytes. i=" + << i + << " c='" + << buf[i] + << "'"; m_buf[m_bufLen] = c; m_bufLen++; if (m_bufLen == FIRST_JSON_BUFFER_SIZE) { @@ -298,15 +357,23 @@ size_t ParserJson::push(const char* buf, size_t len) { break; case s_start_parsing: - dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start_parsing. sending len=" << - (int)m_bufLen << ": '" << std::string(m_buf, m_bufLen) << "'"; + dbgTrace(D_WAAP_PARSER_JSON) + << "ParserJson::push(): s_start_parsing. sending len=" + << (int)m_bufLen + << ": '" + << std::string(m_buf, m_bufLen) + << "'"; m_state = s_parsing; // fallthrough // CP_FALL_THROUGH; case s_parsing: - dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_parsing. sending len=" << (int)(len - i) << ": '" << - std::string(buf + i, len - i) << "'"; + dbgTrace(D_WAAP_PARSER_JSON) + << "ParserJson::push(): s_parsing. sending len=" + << (int)(len - i) + << ": '" + << std::string(buf + i, len - i) + << "'"; if (m_bufLen > 0) { // Send accumulated bytes (if any) if (yajl_parse(m_jsonHandler, (unsigned char*)m_buf, m_bufLen) != yajl_status_ok) { @@ -333,15 +400,20 @@ size_t ParserJson::push(const char* buf, size_t len) { return len; } -void ParserJson::finish() { +void +ParserJson::finish() +{ push(NULL, 0); } const std::string & -ParserJson::name() const { +ParserJson::name() const +{ return m_parserName; } -bool ParserJson::error() const { +bool +ParserJson::error() const +{ return m_state == s_error; } diff --git a/components/security_apps/waap/waap_clib/ParserJson.h b/components/security_apps/waap/waap_clib/ParserJson.h index 6c14b9f..f113831 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.h +++ b/components/security_apps/waap/waap_clib/ParserJson.h @@ -27,7 +27,11 @@ typedef size_t yajl_size_t; class ParserJson : public ParserBase { public: - ParserJson(IParserReceiver &receiver, IParserReceiver2 *receiver2=NULL); + ParserJson( + IParserReceiver &receiver, + bool should_collect_for_oa_schema_updater=false, + size_t parser_depth=0, + IParserReceiver2 *receiver2=NULL); virtual ~ParserJson(); size_t push(const char *data, size_t data_len); void finish(); @@ -80,6 +84,10 @@ private: KeyStack m_key; std::vector m_depthStack; yajl_handle m_jsonHandler; + bool is_map_empty; + bool should_collect_for_oa_schema_updater; + + size_t m_parser_depth; public: static const std::string m_parserName; }; diff --git a/components/security_apps/waap/waap_clib/ParserMultipartForm.cc b/components/security_apps/waap/waap_clib/ParserMultipartForm.cc index e56957a..db32b2e 100755 --- a/components/security_apps/waap/waap_clib/ParserMultipartForm.cc +++ b/components/security_apps/waap/waap_clib/ParserMultipartForm.cc @@ -26,7 +26,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_MULTIPART_FORM); const std::string ParserMultipartForm::m_parserName = "ParserMultipartForm"; -int ParserMultipartForm::HdrValueAnalyzer::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) +int ParserMultipartForm::HdrValueAnalyzer::onKv( + const char* k, + size_t k_len, + const char* v, + size_t v_len, + int flags, + size_t parser_depth + ) { dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "HdrValueAnalyzer::onKv(): k='%.*s' v='%.*s'" << (int)k_len << v; assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH); @@ -43,10 +50,8 @@ void ParserMultipartForm::HdrValueAnalyzer::clear() { } ParserMultipartForm::ParserMultipartForm( - IParserStreamReceiver& receiver, - const char* boundary, - size_t boundary_len) -: + IParserStreamReceiver &receiver, size_t parser_depth, const char *boundary, size_t boundary_len +) : m_receiver(receiver), m_partIdx(0), state(s_start), @@ -55,9 +60,12 @@ ParserMultipartForm::ParserMultipartForm( lookbehind(NULL), multipart_boundary(NULL), m_headerValueParser(NULL), - m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer) + m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer), + m_parser_depth(parser_depth) { - dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "ParserMultipartForm::ParserMultipartForm()"; + dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) + << "ParserMultipartForm::ParserMultipartForm() parser_depth=" + << parser_depth; boundary_len += 2; // two hyphens will be prepended to boundary string provided multipart_boundary = (char*)malloc(boundary_len + boundary_len + 9); diff --git a/components/security_apps/waap/waap_clib/ParserMultipartForm.h b/components/security_apps/waap/waap_clib/ParserMultipartForm.h index aa1492a..7ec8e9f 100755 --- a/components/security_apps/waap/waap_clib/ParserMultipartForm.h +++ b/components/security_apps/waap/waap_clib/ParserMultipartForm.h @@ -18,18 +18,25 @@ #include "ParserHdrValue.h" #include -class ParserMultipartForm : public ParserBase, boost::noncopyable { +class ParserMultipartForm : public ParserBase, boost::noncopyable +{ public: - class HdrValueAnalyzer : public IParserReceiver { + class HdrValueAnalyzer : public IParserReceiver + { public: - int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags); + int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth); void clear(); const std::string &getPartName() const { return m_partName; } private: std::string m_partName; }; - ParserMultipartForm(IParserStreamReceiver &receiver, const char *boundary, size_t boundary_len); + ParserMultipartForm( + IParserStreamReceiver &receiver, + size_t parser_depth, + const char *boundary, + size_t boundary_len + ); virtual ~ParserMultipartForm(); size_t push(const char *buf, size_t len); void finish(); @@ -37,7 +44,8 @@ public: virtual bool error() const; virtual size_t depth() { return 1; } private: - enum state { + enum state + { s_start, s_start_boundary, s_key_start, @@ -88,6 +96,7 @@ private: std::string m_partName; // Part name static const std::string m_parserName; + size_t m_parser_depth; }; #endif // __PARSER_MULTIPART_FORM_H__1c7eb4fa diff --git a/components/security_apps/waap/waap_clib/ParserPairs.cc b/components/security_apps/waap/waap_clib/ParserPairs.cc new file mode 100644 index 0000000..a881c1f --- /dev/null +++ b/components/security_apps/waap/waap_clib/ParserPairs.cc @@ -0,0 +1,479 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ParserPairs.h" +#include "Waf2Util.h" +#include "debug.h" + +USE_DEBUG_FLAG(D_WAAP_PARSER_PAIRS); +USE_DEBUG_FLAG(D_WAAP); + +const std::string ParserPairs::m_parserName = "ParserPairs"; + +ParserPairs::ParserPairs( + IParserStreamReceiver &receiver, + size_t parser_depth, + char separatorChar, + bool should_decode_per, + bool should_decode_plus_sign +) : + m_receiver(receiver), + m_state(s_start), + m_escapedLen(0), + m_separatorChar(separatorChar), + m_escapedCharCandidate(0), + should_decode_percent(should_decode_per), + should_decode_plus(should_decode_plus_sign), + m_parser_depth(parser_depth), + m_bracket_counter(0) +{ + dbgTrace(D_WAAP) + << "should_decode_percent=" + << should_decode_per + << "parser_depth=" + << parser_depth; + + // TODO:: is there a need for this? + memset(m_escaped, 0, sizeof(m_escaped)); +} + +ParserPairs::~ParserPairs() +{} + +size_t +ParserPairs::push(const char *buf, size_t len) +{ + size_t i = 0; + size_t mark = 0; + char c; + int is_last = 0; + + dbgTrace(D_WAAP) << "ParserPairs::push(): starting (len=" << len << ")"; + + if (len == 0) { + dbgTrace(D_WAAP) << "ParserPairs::push(): end of data signal! m_state=" << m_state; + // flush unescaped data collected (if any) + if (m_escapedLen > 0) { + if (m_state == s_key_start) { + if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + } else if (m_state == s_value_start) { + if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + } + m_escapedLen = 0; + } + + if (m_receiver.onKvDone() != 0) { + m_state = s_error; + return i; + } + + return 0; + } + + while (i < len) { + c = buf[i]; + is_last = (i == (len - 1)); + + // Checking valid char urlencode + if (c < 32) { + // Bitwise operation checking since `char` can be either signed on unsigned depending on arch + if (!isspace(c) && ((c & 0x80) == 0x00)) { + dbgDebug(D_WAAP_PARSER_PAIRS) + << "invalid URL encoding character: " + << c + << " decimal value is " + << c + 0; + m_state = s_error; + return i; + } + } + + dbgTrace(D_WAAP_PARSER_PAIRS) + << "ParserPairs::push(): state=" + << m_state + << "; ch='" + << c + << "' m_bracket_counter = " + << m_bracket_counter; + + switch (m_state) { + case s_start: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_start"; + // m_state = s_key_start; + + // fallthrough // + CP_FALL_THROUGH; + } + case s_key_start: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_start"; + mark = i; + m_state = s_key; + + // fallthrough // + CP_FALL_THROUGH; + } + case s_key: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key"; + + // skip leading spaces in the key + if (isspace(c)) { + m_state = s_key_start; // skip the space character without including it in the output + break; + } + + if (c == '%' && should_decode_percent) { + if (i - mark > 0) { + if (m_receiver.onKey(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + } + m_state = s_key_escaped1; + break; + } else if (c == '+' && should_decode_plus) { + // convert plus character to space + if (i - mark > 0) { + if (m_receiver.onKey(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + mark = i; + } + m_escaped[m_escapedLen] = ' '; + m_escapedLen++; + if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) { + if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + } + m_state = s_key_start; + break; + } else { + // flush unescaped data collected (if any) + if (m_escapedLen > 0) { + if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + mark = i; + } + } + if (c == m_separatorChar) { + // this happens when there is a key without value. Example: ?p&a=b&k&%61&blah + // in this case we emit the key, but not the value, and send onKvDone to cause + // the receiver to process the pair: key will be provided with no value. + if (m_receiver.onKey(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + if (m_receiver.onKvDone() != 0) { + m_state = s_error; + return i; + } + m_state = s_key_start; + break; + } + if (c == '=') { + if (m_receiver.onKey(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + m_state = s_value_start; + break; + } + if (is_last) { + if (m_receiver.onKey(buf + mark, (i - mark) + 1) != 0) { + m_state = s_error; + return i; + } + } + break; + } + case s_key_escaped1: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped1"; + bool valid; + unsigned char v = from_hex(c, valid); + if (!valid) { // character right after the '%' is not a valid hex char. + // dump escaped chars + if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + // return the '%' character back to the output. + if (m_receiver.onKey("%", 1) != 0) { + return i; + } + + // If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences + if (c != '%') { + // pass the non-hex character back to the output too. + if (m_receiver.onKey(&c, 1) != 0) { + return i; + } + + // otherwise (the character is not '%'), switch back to the s_key state + m_state = s_key_start; + } + break; + } + + m_escapedCharCandidate = c; + m_escaped[m_escapedLen] = v << 4; + m_state = s_key_escaped2; + break; + } + case s_key_escaped2: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped2"; + bool valid; + unsigned char v = from_hex(c, valid); + if (!valid) { + // This situation (2nd character is not valid hex) is not treated right now. + // In this case, v will be equal to 0 and output character will be invalid one. + + // dump escaped chars + if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + + // return the '%' character back to the output. + if (m_receiver.onKey("%", 1) != 0) { + return i; + } + // add the character that was thought to be escaped value + if (m_receiver.onKey(&m_escapedCharCandidate, 1)) { + return i; + } + + // re parse the character as a key (i is incremented back to current value) + i--; + m_state = s_key_start; + break; + } + m_escapedCharCandidate = 0; + m_escaped[m_escapedLen] |= v; + m_escapedLen++; + if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) { + if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + } + m_state = s_key_start; + break; + } + case s_value_start: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_start"; + mark = i; + m_state = s_value; + + // fallthrough // + CP_FALL_THROUGH; + } + case s_value: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value"; + if (c == '%' && should_decode_percent) { + if (i - mark > 0) { + if (m_receiver.onValue(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + } + m_state = s_value_escaped1; + break; + } else if (c == '+' && should_decode_plus) { + // convert plus character to space + if (i - mark > 0) { + if (m_receiver.onValue(buf + mark, i - mark) != 0) { + m_state = s_error; + return i; + } + } + m_escaped[m_escapedLen] = ' '; + m_escapedLen++; + if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) { + if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + } + m_state = s_value_start; + break; + } else { + // flush unescaped data collected (if any) + if (c == '{' || c == '[') m_bracket_counter++; + if (c == '}' || c == ']') m_bracket_counter--; + if (m_escapedLen > 0) { + if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + mark = i; + } + } + if (c == m_separatorChar) { + if (m_bracket_counter) { + if (m_escapedLen > 0) { + if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + mark = i; + } + } else { + if (m_receiver.onValue(buf + mark, i - mark) != 0) { + dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() s_value : failed on value"; + m_state = s_error; + return i; + } + if (m_receiver.onKvDone() != 0) { + dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() : s_value : failed on KV"; + m_state = s_error; + return i; + } + m_state = s_key_start; + break; + } + } + if (is_last) { + if (m_receiver.onValue(buf + mark, (i - mark) + 1) != 0) { + m_state = s_error; + return i; + } + } + break; + } + case s_value_escaped1: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped1"; + bool valid; + unsigned char v = from_hex(c, valid); + if (!valid) { // character right after the '%' is not a valid hex char. + // dump escaped chars + if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + // return the '%' character back to the output. + if (m_receiver.onValue("%", 1) != 0) { + return i; + } + + // If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences) + if (c != '%') { + // pass the non-hex character back to the output too. + if (m_receiver.onValue(&c, 1) != 0) { + return i; + } + + // otherwise (the character is not '%'), switch back to the s_value state + m_state = s_value_start; + } + break; + } + m_escapedCharCandidate = c; + m_escaped[m_escapedLen] = v << 4; + m_state = s_value_escaped2; + break; + } + case s_value_escaped2: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped2"; + bool valid; + unsigned char v = from_hex(c, valid); + if (!valid) { + // This situation (2nd character is not valid hex) is not treated right now. + // In this case, v will be equal to 0 and output character will be invalid one. + + // dump escaped chars + if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + + // return the '%' character back to the output. + if (m_receiver.onValue("%", 1) != 0) { + return i; + } + // add the character that was thought to be escaped value + if (m_receiver.onValue(&m_escapedCharCandidate, 1)) { + return i; + } + + // re parse the character as a key (i is incremented back to current value) + i--; + m_state = s_value_start; + break; + } + m_escapedCharCandidate = 0; + m_escaped[m_escapedLen] |= v; + m_escapedLen++; + if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) { + if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { + m_state = s_error; + return i; + } + m_escapedLen = 0; + } + m_state = s_value_start; + break; + } + case s_error: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_error"; + return 0; + } + default: { + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): URL parser unrecoverable error"; + m_state = s_error; + return 0; + } + } // end of switch() + ++i; + } + + dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): finished: len=" << len; + return len; +} + +void +ParserPairs::finish() +{ + push(NULL, 0); +} + +const std::string & +ParserPairs::name() const +{ + return m_parserName; +} + +bool +ParserPairs::error() const +{ + return m_state == s_error; +} diff --git a/components/security_apps/waap/waap_clib/ParserPairs.h b/components/security_apps/waap/waap_clib/ParserPairs.h new file mode 100755 index 0000000..f9533a5 --- /dev/null +++ b/components/security_apps/waap/waap_clib/ParserPairs.h @@ -0,0 +1,65 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __PARSER_PAIRS_H__ +#define __PARSER_PAIRS_H__ + +#include "ParserBase.h" +#include + +#define MAX_PAIRS_ESCAPED_SIZE 16 + +class ParserPairs : public ParserBase { +public: + ParserPairs( + IParserStreamReceiver &receiver, + size_t parser_depth, + char separatorChar = '&', + bool should_decode_per = false, + bool should_decode_plus_sign = false); + virtual ~ParserPairs(); + size_t push(const char *data, size_t data_len); + void finish(); + virtual const std::string &name() const; + bool error() const; + virtual size_t depth() { return 1; } + +private: + enum state { + s_start, + s_key_start, + s_key, + s_key_escaped1, + s_key_escaped2, + s_value_start, + s_value, + s_value_escaped1, + s_value_escaped2, + s_end, + s_error + }; + + IParserStreamReceiver &m_receiver; + enum state m_state; + unsigned char m_escapedLen; // count of characters loaded in m_escaped[] buffer + char m_escaped[MAX_PAIRS_ESCAPED_SIZE]; + char m_separatorChar; + char m_escapedCharCandidate; + bool should_decode_percent; + bool should_decode_plus; + static const std::string m_parserName; + size_t m_parser_depth; + int m_bracket_counter; +}; + +#endif // __PARSER_PAIRS_H__ diff --git a/components/security_apps/waap/waap_clib/ParserPercentEncode.cc b/components/security_apps/waap/waap_clib/ParserPercentEncode.cc index 1f733c2..08619df 100644 --- a/components/security_apps/waap/waap_clib/ParserPercentEncode.cc +++ b/components/security_apps/waap/waap_clib/ParserPercentEncode.cc @@ -19,12 +19,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PERCENT); const std::string ParserPercentEncode::m_parserName = "ParserPercentEncode"; -ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver) : +ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth) : m_receiver(receiver), m_state(s_start), m_escapedLen(0), - m_escapedCharCandidate(0) + m_escapedCharCandidate(0), + m_parser_depth(parser_depth) { + dbgTrace(D_WAAP_PARSER_PERCENT) + << "parser_depth=" + << parser_depth; + memset(m_escaped, 0, sizeof(m_escaped)); } @@ -51,7 +56,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << m_escaped - << "<<<"; + << "<<< and m_escapedLen = " + << m_escapedLen; if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { m_state = s_error; return i; @@ -97,7 +103,6 @@ ParserPercentEncode::push(const char *buf, size_t len) { dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): s_start"; - // fallthrough // CP_FALL_THROUGH; } @@ -107,7 +112,6 @@ ParserPercentEncode::push(const char *buf, size_t len) << "ParserPercentEncode::push(): s_value_start"; pointer_in_buffer = i; m_state = s_value; - // fallthrough // CP_FALL_THROUGH; } @@ -120,9 +124,10 @@ ParserPercentEncode::push(const char *buf, size_t len) if (i - pointer_in_buffer > 0) { dbgTrace(D_WAAP_PARSER_PERCENT) - << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" + << "ParserPercentEncode::push(): call onValue with buffer = >>>" << (buf + pointer_in_buffer) - << "<<<"; + << "<<< of size " + << i - pointer_in_buffer; if (m_receiver.onValue(buf + pointer_in_buffer, i - pointer_in_buffer) != 0) { m_state = s_error; @@ -140,7 +145,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << m_escaped - << "<<<"; + << "<<< and m_escapedLen = " + << m_escapedLen; if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { m_state = s_error; @@ -155,7 +161,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << (buf + pointer_in_buffer) - << "<<<"; + << "<<< of size " + << (i - pointer_in_buffer) + 1; if (m_receiver.onValue(buf + pointer_in_buffer, (i - pointer_in_buffer) + 1) != 0) { m_state = s_error; @@ -177,7 +184,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << m_escaped - << "<<<"; + << "<<< and m_escapedLen = " + << m_escapedLen; if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) { @@ -186,7 +194,7 @@ ParserPercentEncode::push(const char *buf, size_t len) } m_escapedLen = 0; // return the '%' character back to the output. - dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" + dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with \"%\" = >>>" << "%" << "<<<"; if (m_receiver.onValue("%", 1) != 0) @@ -199,7 +207,7 @@ ParserPercentEncode::push(const char *buf, size_t len) { // pass the non-hex character back to the output too. dbgTrace(D_WAAP_PARSER_PERCENT) - << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" + << "ParserPercentEncode::push(): call onValue with current char = >>>" << c << "<<<"; if (m_receiver.onValue(&c, 1) != 0) @@ -232,7 +240,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << m_escaped - << "<<<"; + << "<<< and m_escapedLen = " + << m_escapedLen; if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) { @@ -243,7 +252,7 @@ ParserPercentEncode::push(const char *buf, size_t len) // return the '%' character back to the output. dbgTrace(D_WAAP_PARSER_PERCENT) - << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" + << "ParserPercentEncode::push(): call onValue with \"%\" >>>" << "%" << "<<<"; if (m_receiver.onValue("%", 1) != 0) @@ -252,7 +261,7 @@ ParserPercentEncode::push(const char *buf, size_t len) } // add the character that was thought to be escaped value dbgTrace(D_WAAP_PARSER_PERCENT) - << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" + << "ParserPercentEncode::push(): call onValue with m_escapedCharCandicate = >>>" << m_escapedCharCandidate << "<<<"; if (m_receiver.onValue(&m_escapedCharCandidate, 1)) @@ -273,7 +282,8 @@ ParserPercentEncode::push(const char *buf, size_t len) dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>" << m_escaped - << "<<<"; + << "<<< and m_escapedLen = " + << m_escapedLen; if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { m_state = s_error; diff --git a/components/security_apps/waap/waap_clib/ParserPercentEncode.h b/components/security_apps/waap/waap_clib/ParserPercentEncode.h index 67c5218..fff7b6b 100644 --- a/components/security_apps/waap/waap_clib/ParserPercentEncode.h +++ b/components/security_apps/waap/waap_clib/ParserPercentEncode.h @@ -22,7 +22,7 @@ class ParserPercentEncode : public ParserBase { public: - ParserPercentEncode(IParserStreamReceiver &receiver); + ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth); virtual ~ParserPercentEncode(); size_t push(const char *data, size_t data_len); void finish(); @@ -53,6 +53,7 @@ private: char m_escaped[MAX_PERCENT_ENCODED_SIZE]; char m_escapedCharCandidate; static const std::string m_parserName; + size_t m_parser_depth; }; #endif diff --git a/components/security_apps/waap/waap_clib/ParserRaw.cc b/components/security_apps/waap/waap_clib/ParserRaw.cc index 7903e3a..0aeb0d2 100755 --- a/components/security_apps/waap/waap_clib/ParserRaw.cc +++ b/components/security_apps/waap/waap_clib/ParserRaw.cc @@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_RAW); const std::string ParserRaw::m_parserName = "ParserRaw"; -ParserRaw::ParserRaw(IParserStreamReceiver &receiver, const std::string &key) +ParserRaw::ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key) :m_receiver(receiver), m_key(key), m_state(s_start) { } diff --git a/components/security_apps/waap/waap_clib/ParserRaw.h b/components/security_apps/waap/waap_clib/ParserRaw.h index db1f001..0356ff9 100755 --- a/components/security_apps/waap/waap_clib/ParserRaw.h +++ b/components/security_apps/waap/waap_clib/ParserRaw.h @@ -19,7 +19,7 @@ class ParserRaw : public ParserBase { public: - ParserRaw(IParserStreamReceiver &receiver, const std::string &key); + ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key); virtual ~ParserRaw(); size_t push(const char *data, size_t data_len); void finish(); diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc index 4aa0706..506ef1c 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.cc +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.cc @@ -16,27 +16,37 @@ #include "debug.h" USE_DEBUG_FLAG(D_WAAP_PARSER_URLENCODE); +USE_DEBUG_FLAG(D_WAAP); const std::string ParserUrlEncode::m_parserName = "ParserUrlEncode"; -ParserUrlEncode::ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar, bool should_decode_per) - : +ParserUrlEncode::ParserUrlEncode( + IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar, bool should_decode_per +) : m_receiver(receiver), m_state(s_start), m_escapedLen(0), m_separatorChar(separatorChar), m_escapedCharCandidate(0), - should_decode_percent(should_decode_per) + should_decode_percent(should_decode_per), + m_parser_depth(parser_depth) { - dbgTrace(D_WAAP_PARSER_URLENCODE) << "should_decode_per=" << should_decode_per; + dbgTrace(D_WAAP) + << "should_decode_percent=" + << should_decode_per + << "parser_depth=" + << parser_depth; + // TODO:: is there a need for this? memset(m_escaped, 0, sizeof(m_escaped)); } -ParserUrlEncode::~ParserUrlEncode() { -} +ParserUrlEncode::~ParserUrlEncode() +{} -size_t ParserUrlEncode::push(const char *buf, size_t len) { +size_t +ParserUrlEncode::push(const char *buf, size_t len) +{ size_t i = 0; size_t mark = 0; char c; @@ -53,8 +63,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { m_state = s_error; return i; } - } - else if (m_state == s_value_start) { + } else if (m_state == s_value_start) { if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { m_state = s_error; return i; @@ -76,8 +85,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { is_last = (i == (len - 1)); // Checking valid char urlencode - if (c < 32) - { + if (c < 32) { dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c; m_state = s_error; return i; @@ -119,8 +127,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { } m_state = s_key_escaped1; break; - } - else if (c == '+') { + } else if (c == '+') { // convert plus character to space if (i - mark > 0) { if (m_receiver.onKey(buf + mark, i - mark) != 0) { @@ -140,8 +147,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { } m_state = s_key_start; break; - } - else { + } else { // flush unescaped data collected (if any) if (m_escapedLen > 0) { if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) { @@ -201,7 +207,6 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { // If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences if (c != '%') { - // pass the non-hex character back to the output too. if (m_receiver.onKey(&c, 1) != 0) { return i; @@ -279,8 +284,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { } m_state = s_value_escaped1; break; - } - else if (c == '+') { + } else if (c == '+') { // convert plus character to space if (i - mark > 0) { if (m_receiver.onValue(buf + mark, i - mark) != 0) { @@ -299,8 +303,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { } m_state = s_value_start; break; - } - else { + } else { // flush unescaped data collected (if any) if (m_escapedLen > 0) { if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) { @@ -425,15 +428,20 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) { return len; } -void ParserUrlEncode::finish() { +void +ParserUrlEncode::finish() +{ push(NULL, 0); } const std::string & -ParserUrlEncode::name() const { +ParserUrlEncode::name() const +{ return m_parserName; } -bool ParserUrlEncode::error() const { +bool +ParserUrlEncode::error() const +{ return m_state == s_error; } diff --git a/components/security_apps/waap/waap_clib/ParserUrlEncode.h b/components/security_apps/waap/waap_clib/ParserUrlEncode.h index 71e3d05..7b1cbc0 100755 --- a/components/security_apps/waap/waap_clib/ParserUrlEncode.h +++ b/components/security_apps/waap/waap_clib/ParserUrlEncode.h @@ -21,7 +21,11 @@ class ParserUrlEncode : public ParserBase { public: - ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar = '&', bool should_decode_per = true); + ParserUrlEncode( + IParserStreamReceiver &receiver, + size_t parser_depth, + char separatorChar = '&', + bool should_decode_per = true); virtual ~ParserUrlEncode(); size_t push(const char *data, size_t data_len); void finish(); @@ -52,6 +56,7 @@ private: char m_escapedCharCandidate; bool should_decode_percent; static const std::string m_parserName; + size_t m_parser_depth; }; #endif // __PARSER_URL_ENCODE_H__29ebe806 diff --git a/components/security_apps/waap/waap_clib/ParserXML.cc b/components/security_apps/waap/waap_clib/ParserXML.cc index 5cafc94..ae69d65 100755 --- a/components/security_apps/waap/waap_clib/ParserXML.cc +++ b/components/security_apps/waap/waap_clib/ParserXML.cc @@ -60,9 +60,11 @@ void ParserXML::onStartElementNs( if (p->m_receiver.onKv( p->m_key.c_str(), p->m_key.size(), - (const char*)attr_value_begin, attr_value_end - attr_value_begin, - BUFFERED_RECEIVER_F_BOTH - ) != 0) { + (const char *)attr_value_begin, + attr_value_end - attr_value_begin, + BUFFERED_RECEIVER_F_BOTH, + p->m_parser_depth + ) != 0) { p->m_state = s_error; } p->m_key.pop("XML end attribute"); @@ -195,9 +197,17 @@ static void onError(void* ctx, const char* msg, ...) { dbgTrace(D_WAAP_PARSER_XML) << "LIBXML (xml) onError: " << std::string(string); } -ParserXML::ParserXML(IParserStreamReceiver& receiver) - :m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("xml_parser"), m_pushParserCtxPtr(NULL) { - dbgTrace(D_WAAP_PARSER_XML) << "ParserXML::ParserXML()"; +ParserXML::ParserXML(IParserStreamReceiver &receiver, size_t parser_depth) : + m_receiver(receiver), + m_state(s_start), + m_bufLen(0), + m_key("xml_parser"), + m_pushParserCtxPtr(NULL), + m_parser_depth(parser_depth) +{ + dbgTrace(D_WAAP_PARSER_XML) + << "ParserXML::ParserXML() parser_depth=" + << parser_depth; // TODO:: is zeroing this really needed? memset(m_buf, 0, sizeof(m_buf)); diff --git a/components/security_apps/waap/waap_clib/ParserXML.h b/components/security_apps/waap/waap_clib/ParserXML.h index beb6745..7c0f7b9 100755 --- a/components/security_apps/waap/waap_clib/ParserXML.h +++ b/components/security_apps/waap/waap_clib/ParserXML.h @@ -24,7 +24,7 @@ class ParserXML : public ParserBase { public: - ParserXML(IParserStreamReceiver &receiver); + ParserXML(IParserStreamReceiver &receiver, size_t parser_depth); virtual ~ParserXML(); size_t push(const char *data, size_t data_len); void finish(); @@ -94,6 +94,7 @@ private: std::vector m_elemTrackStack; xmlSAXHandler m_saxHandler; xmlParserCtxtPtr m_pushParserCtxPtr; + size_t m_parser_depth; public: static const std::string m_parserName; }; diff --git a/components/security_apps/waap/waap_clib/Serializator.cc b/components/security_apps/waap/waap_clib/Serializator.cc index fd96b27..cd49cb6 100755 --- a/components/security_apps/waap/waap_clib/Serializator.cc +++ b/components/security_apps/waap/waap_clib/Serializator.cc @@ -96,7 +96,6 @@ Maybe RestGetFile::genJson() const return genError("Failed to compress data"); } data = string((const char *)res.output, res.num_output_bytes); - json = data; if (res.output) free(res.output); @@ -179,10 +178,54 @@ void SerializeToFileBase::saveData() serialize(ss); + string data = ss.str(); + + auto compression_stream = initCompressionStream(); + CompressionResult res = compressData( + compression_stream, + CompressionType::GZIP, + data.size(), + reinterpret_cast(data.c_str()), + true + ); + finiCompressionStream(compression_stream); + if (!res.ok) { + dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data"; + } else { + ss.str(string((const char *)res.output, res.num_output_bytes)); + } + + filestream << ss.str(); filestream.close(); } +string decompress(string fileContent) { + if (!isGZipped(fileContent)) { + dbgTrace(D_WAAP) << "file note zipped"; + return fileContent; + } + auto compression_stream = initCompressionStream(); + + DecompressionResult res = decompressData( + compression_stream, + fileContent.size(), + reinterpret_cast(fileContent.c_str()) + ); + + finiCompressionStream(compression_stream); + + if (res.ok) { + string decompressedData = string((const char *)res.output, res.num_output_bytes); + if (res.output) free(res.output); + res.output = nullptr; + res.num_output_bytes = 0; + return decompressedData; + } + + return fileContent; +} + void SerializeToFileBase::loadFromFile(string filePath) { dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loadFromFile() file: " << filePath; @@ -239,8 +282,8 @@ void SerializeToFileBase::loadFromFile(string filePath) delete[] buffer; - - stringstream ss(dataObfuscated); + stringstream ss; + ss << decompress(dataObfuscated); try { diff --git a/components/security_apps/waap/waap_clib/Signatures.cc b/components/security_apps/waap/waap_clib/Signatures.cc index c1c9a37..82a05fc 100755 --- a/components/security_apps/waap/waap_clib/Signatures.cc +++ b/components/security_apps/waap/waap_clib/Signatures.cc @@ -85,13 +85,6 @@ static filtered_parameters_t to_filtermap(const picojson::value::object& JsObj) return result; } -std::string genDelimitedKeyValPattern(const std::string& delim) -{ - std::string pattern = "^([^" + delim + "]+?=[^" + delim + "]+?" + delim + ")+" - "([^" + delim + "]+?=[^" + delim + "]+?)" + delim + "?$"; - return pattern; -} - Signatures::Signatures(const std::string& filepath) : sigsSource(loadSource(filepath)), error(false), @@ -207,11 +200,6 @@ Signatures::Signatures(const std::string& filepath) : html_regex("(<(?>body|head)\\b.*>(?>.|[\\r\\n]){0,400}){2}|.+\\|)+.+}"), - pipes_delimited_key_val_re(genDelimitedKeyValPattern("\\|")), - semicolon_delimited_key_val_re(genDelimitedKeyValPattern(";")), - asterisk_delimited_key_val_re(genDelimitedKeyValPattern("\\*")), - comma_delimited_key_val_re(genDelimitedKeyValPattern(",")), - ampersand_delimited_key_val_re(genDelimitedKeyValPattern("&")), headers_re(to_regexmap(sigsSource["headers_re"].get(), error)), format_magic_binary_re(sigsSource["format_magic_binary_re"].get(), error, "format_magic_binary_re"), params_type_re(to_regexmap(sigsSource["format_types_regex_list"].get(), error)), diff --git a/components/security_apps/waap/waap_clib/Signatures.h b/components/security_apps/waap/waap_clib/Signatures.h index 0312fb2..ab7bffd 100755 --- a/components/security_apps/waap/waap_clib/Signatures.h +++ b/components/security_apps/waap/waap_clib/Signatures.h @@ -61,11 +61,6 @@ public: const Regex html_regex; const Regex uri_parser_regex; const boost::regex confluence_macro_re; - const boost::regex pipes_delimited_key_val_re; - const boost::regex semicolon_delimited_key_val_re; - const boost::regex asterisk_delimited_key_val_re; - const boost::regex comma_delimited_key_val_re; - const boost::regex ampersand_delimited_key_val_re; #if 0 // Removed by Pavel's request. Leaving here in case he'll want to add this back... const std::set cookie_ignored_keywords; const std::set cookie_ignored_patterns; diff --git a/components/security_apps/waap/waap_clib/WaapConfigApi.cc b/components/security_apps/waap/waap_clib/WaapConfigApi.cc index c2502e6..6b94e4e 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApi.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigApi.cc @@ -59,7 +59,6 @@ WaapConfigAPI::clearAssetsCount() void WaapConfigAPI::load(cereal::JSONInputArchive& ar) { // order has affect - we need to call base last because of triggers and overrides - WaapConfigBase::load(ar); assets_ids_aggregation.insert(m_assetId); } diff --git a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc index 6a0de36..2f98558 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigApplication.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigApplication.cc @@ -74,7 +74,6 @@ void WaapConfigApplication::load(cereal::JSONInputArchive& ar) { // order has affect - we need to call base last because of triggers and overrides - loadOpenRedirectPolicy(ar); loadErrorDisclosurePolicy(ar); loadCsrfPolicy(ar); diff --git a/components/security_apps/waap/waap_clib/WaapConfigBase.cc b/components/security_apps/waap/waap_clib/WaapConfigBase.cc index 910083d..7cf80e7 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigBase.cc +++ b/components/security_apps/waap/waap_clib/WaapConfigBase.cc @@ -24,6 +24,8 @@ USE_DEBUG_FLAG(D_WAAP_ULIMITS); USE_DEBUG_FLAG(D_WAAP); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); + using boost::algorithm::to_lower_copy; using namespace std; @@ -253,12 +255,12 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar) } + const std::vector & WaapConfigBase::get_applicationUrls() const { return m_applicationUrls; } - void WaapConfigBase::loadErrorDisclosurePolicy(cereal::JSONInputArchive& ar) { std::string failMessage = "Failed to load the WAAP Information Disclosure policy"; @@ -432,6 +434,7 @@ const std::shared_ptr& WaapConfigBase::get_OpenRedir } + const std::shared_ptr& WaapConfigBase::get_ErrorDisclosurePolicy() const { return m_errorDisclosurePolicy; diff --git a/components/security_apps/waap/waap_clib/WaapConfigBase.h b/components/security_apps/waap/waap_clib/WaapConfigBase.h index 55d3fd1..18dea20 100755 --- a/components/security_apps/waap/waap_clib/WaapConfigBase.h +++ b/components/security_apps/waap/waap_clib/WaapConfigBase.h @@ -99,6 +99,7 @@ private: std::vector m_applicationUrls; std::shared_ptr m_errorDisclosurePolicy; std::string m_schemaValidationPoicyStatusMessage; + std::string m_schemaUpdaterPoicyStatusMessage; std::shared_ptr m_csrfPolicy; std::shared_ptr m_rateLimitingPolicy; std::shared_ptr m_errorLimitingPolicy; diff --git a/components/security_apps/waap/waap_clib/WaapOverride.cc b/components/security_apps/waap/waap_clib/WaapOverride.cc index 87af2ae..30e691b 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.cc +++ b/components/security_apps/waap/waap_clib/WaapOverride.cc @@ -31,7 +31,7 @@ bool Match::operator==(const Match &other) const } Behavior::Behavior() -: m_action(""), m_log(""), m_sourceIdentifier("") +: m_id(""), m_action(""), m_log(""), m_sourceIdentifier("") { } @@ -40,6 +40,11 @@ bool Behavior::operator==(const Behavior &other) const return (m_action == other.m_action) && (m_log == other.m_log) && (m_sourceIdentifier == other.m_sourceIdentifier); } +const std::string & Behavior::getParentId() const +{ + return m_id; +} + const std::string & Behavior::getAction() const { return m_action; @@ -55,6 +60,11 @@ const std::string& Behavior::getSourceIdentifier() const return m_sourceIdentifier; } +void Behavior::setParentId(const std::string& id) +{ + m_id = id; +} + bool Rule::operator==(const Rule &other) const { return (m_match == other.m_match) && @@ -70,7 +80,9 @@ bool Policy::operator==(const Policy &other) const State::State() : bForceBlock(false), + forceBlockIds(), bForceException(false), + forceExceptionIds(), bIgnoreLog(false), bSourceIdentifierOverride(false), sSourceIdentifierMatch("") diff --git a/components/security_apps/waap/waap_clib/WaapOverride.h b/components/security_apps/waap/waap_clib/WaapOverride.h index 028dc54..2d315ff 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.h +++ b/components/security_apps/waap/waap_clib/WaapOverride.h @@ -175,10 +175,13 @@ public: } } + const std::string &getParentId() const; const std::string &getAction() const; const std::string &getLog() const; const std::string &getSourceIdentifier() const; + void setParentId(const std::string& id); private: + std::string m_id; std::string m_action; std::string m_log; std::string m_sourceIdentifier; @@ -195,20 +198,20 @@ public: } catch (const cereal::Exception &e) { - dbgTrace(D_WAAP_OVERRIDE) << "An override rule has no id."; + dbgDebug(D_WAAP_OVERRIDE) << "An override rule has no id."; m_id.clear(); } - ar(cereal::make_nvp("parsedMatch", m_match)); ar(cereal::make_nvp("parsedBehavior", m_behaviors)); m_isChangingRequestData = false; - for (std::vector::const_iterator it = m_behaviors.begin(); + for (std::vector::iterator it = m_behaviors.begin(); it != m_behaviors.end(); ++it) { - const Behavior& behavior = *it; + Behavior& behavior = *it; + behavior.setParentId(m_id); if (!behavior.getSourceIdentifier().empty()) // this rule changes data in request itself { m_isChangingRequestData = true; @@ -223,8 +226,9 @@ public: { if (m_match.match(testFunctor)) { // extend matchedBehaviors list with all behaviors on this rule - dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched. Adding " << m_behaviors.size() << " new behaviors:"; std::string overrideId = getId(); + dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched id: " << overrideId << + ". Adding " << m_behaviors.size() << " new behaviors:"; if (!overrideId.empty()) { matchedOverrideIds.insert(overrideId); } @@ -308,8 +312,10 @@ private: struct State { // whether to force block regardless of stage2 response (and even if bSendRequest and/or bSendResponse are false) bool bForceBlock; + std::set forceBlockIds; // exception (allow) was matched, so this request won't be blocked. bool bForceException; + std::set forceExceptionIds; // overrides decision in case log should be ignored bool bIgnoreLog; // user identfier override to be applied @@ -335,10 +341,12 @@ struct State { if (matchedBehavior.getAction() == "accept") { dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceException due to override behavior."; bForceException = true; + forceExceptionIds.insert(matchedBehavior.getParentId()); } else if (matchedBehavior.getAction() == "reject") { dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceBlock due to override behavior."; bForceBlock = true; + forceBlockIds.insert(matchedBehavior.getParentId()); } if (matchedBehavior.getLog() == "ignore") diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc index b5de64f..3dadfbb 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc @@ -15,81 +15,86 @@ #include "debug.h" USE_DEBUG_FLAG(D_WAAP); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); namespace Waap { -ResponseInspectReasons::ResponseInspectReasons() -: -openRedirect(false), -errorDisclosure(false), -errorLimiter(false), -rateLimiting(false), -collectResponseForLog(false), -applyOverride(false) -{ -} + ResponseInspectReasons::ResponseInspectReasons() + : + openRedirect(false), + errorDisclosure(false), + errorLimiter(false), + rateLimiting(false), + collectResponseForLog(false), + applyOverride(false) + { + } -bool -ResponseInspectReasons::shouldInspect() const -{ - dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" << - " OpenRedirect=" << openRedirect << - " ErrorDisclosure=" << errorDisclosure << - " RateLimiting=" << rateLimiting << - " ErrorLimiter=" << errorLimiter << - " collectResponseForLog=" << collectResponseForLog << - " applyOverride=" << applyOverride; - return openRedirect || errorDisclosure || rateLimiting || errorLimiter || collectResponseForLog || applyOverride; -} + bool + ResponseInspectReasons::shouldInspect() const + { + dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" << + " OpenRedirect=" << openRedirect << + " ErrorDisclosure=" << errorDisclosure << + " RateLimiting=" << rateLimiting << + " ErrorLimiter=" << errorLimiter << + " collectResponseForLog=" << collectResponseForLog << + " applyOverride=" << applyOverride; -void -ResponseInspectReasons::setOpenRedirect(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag; - openRedirect = flag; -} + return + openRedirect || errorDisclosure || rateLimiting || errorLimiter || + collectResponseForLog || applyOverride; + } -void -ResponseInspectReasons::setErrorDisclosure(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag; - errorDisclosure = flag; -} + void + ResponseInspectReasons::setOpenRedirect(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag; + openRedirect = flag; + } -void -ResponseInspectReasons::setRateLimiting(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag; - rateLimiting = flag; -} -void -ResponseInspectReasons::setErrorLimiter(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag; - errorLimiter = flag; -} + void + ResponseInspectReasons::setErrorDisclosure(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag; + errorDisclosure = flag; + } -void -ResponseInspectReasons::setCollectResponseForLog(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog << " to " << - flag; - collectResponseForLog = flag; -} + void + ResponseInspectReasons::setRateLimiting(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag; + rateLimiting = flag; + } -void -ResponseInspectReasons::setApplyOverride(bool flag) -{ - dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " << - flag; - applyOverride = flag; -} + void + ResponseInspectReasons::setErrorLimiter(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag; + errorLimiter = flag; + } -bool -ResponseInspectReasons::getApplyOverride(void) -{ - return applyOverride; -} + void + ResponseInspectReasons::setCollectResponseForLog(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog << + " to " << flag; + collectResponseForLog = flag; + } + + void + ResponseInspectReasons::setApplyOverride(bool flag) + { + dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " << + flag; + applyOverride = flag; + } + + bool + ResponseInspectReasons::getApplyOverride(void) + { + return applyOverride; + } } diff --git a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h index 7fd9ea0..e2e58e0 100644 --- a/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h +++ b/components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h @@ -25,6 +25,7 @@ public: void setErrorLimiter(bool flag); void setCollectResponseForLog(bool flag); void setApplyOverride(bool flag); + bool getApplyOverride(void); private: bool openRedirect; diff --git a/components/security_apps/waap/waap_clib/WaapScanner.cc b/components/security_apps/waap/waap_clib/WaapScanner.cc index 886f55c..4875b3b 100755 --- a/components/security_apps/waap/waap_clib/WaapScanner.cc +++ b/components/security_apps/waap/waap_clib/WaapScanner.cc @@ -17,8 +17,13 @@ #include #include "debug.h" #include "reputation_features_events.h" +#include USE_DEBUG_FLAG(D_WAAP_SCANNER); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); + +// id generated by xml parser for an entity attribute +const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f"; double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName) { @@ -117,7 +122,7 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN // Ignore scan results from specific fields on csp-report json in case those are not filtered by learning bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp) { - if (res.score < 8.0f && res.location == "body" && dp.getLastParser() == "jsonParser") { + if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") { if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" || (key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) { dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring."; @@ -169,11 +174,14 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp, return m_transaction->reportScanResult(res); } -int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) { + +int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) { Waf2ScanResult& res = m_lastScanResult; DeepParser &dp = m_transaction->getDeepParser(); std::string key = std::string(k, k_len); std::string value = std::string(v, v_len); + + res.clear(); dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: k='" << key << "' v='" << value << "'"; @@ -266,7 +274,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len } // Special value only matched when XML atribute is found. if (v_len == 36) { - if (value == "08a80340-06d3-11ea-9f87-0242ac11000f" && !m_transaction->shouldIgnoreOverride(res)) { + if (value == Waap::Scanner::xmlEntityAttributeId && !m_transaction->shouldIgnoreOverride(res)) { // Always return max score when 0) { dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the cookie value"; m_deepParser.m_key.push("cookie", 6); - ParserUrlEncode cookieValueParser(m_deepParserReceiver, ';'); + ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';'); cookieValueParser.push(value, value_len); cookieValueParser.finish(); m_deepParser.m_key.pop("cookie"); @@ -838,7 +840,7 @@ void Waf2Transaction::parseUnknownHeaderName(const char* name, int name_len) !m_pWaapAssetState->getSignatures()->good_header_name_re.hasMatch(std::string(name, name_len))) { dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header name"; m_deepParser.m_key.push("header", 6); - ParserRaw headerNameParser(m_deepParserReceiver, std::string(name, name_len)); + ParserRaw headerNameParser(m_deepParserReceiver, 0, std::string(name, name_len)); headerNameParser.push(name, name_len); headerNameParser.finish(); m_deepParser.m_key.pop("header name"); @@ -857,7 +859,7 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header value"; m_deepParser.m_key.push("header", 6); - ParserRaw headerValueParser(m_deepParserReceiver, headerName); + ParserRaw headerValueParser(m_deepParserReceiver, 0, headerName); headerValueParser.push(value, value_len); headerValueParser.finish(); m_deepParser.m_key.pop("header value"); @@ -1077,7 +1079,7 @@ void Waf2Transaction::start_request_body() { clearRequestParserState(); - m_requestBodyParser = new ParserRaw(m_deepParserReceiver, "body"); + m_requestBodyParser = new ParserRaw(m_deepParserReceiver, 0, "body"); m_request_body_bytes_received = 0; m_request_body.clear(); @@ -1188,6 +1190,7 @@ void Waf2Transaction::end_request() { dbgTrace(D_WAAP) << "(Waf2Engine::end_request): Security Headers State was created"; } + // Enable response headers processing if response scanning is enabled in policy auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL; m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable); @@ -1642,6 +1645,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, std::vector vOverrideIds(m_matchedOverrideIds.size()); std::copy(m_matchedOverrideIds.begin(), m_matchedOverrideIds.end(), vOverrideIds.begin()); waapLog.addToOrigin(LogField("exceptionIdList", vOverrideIds)); + if (!m_effectiveOverrideIds.empty()) { + std::vector vEffectiveOverrideIds(m_effectiveOverrideIds.size()); + std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin()); + waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds)); + } } } @@ -1697,13 +1705,18 @@ Waf2Transaction::sendLog() return; } + dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException << + " force block: " << m_overrideState.bForceBlock << + " matched overrides count: " << m_matchedOverrideIds.size() << + " effective overrides count: " << m_effectiveOverrideIds.size(); + + bool shouldBlock = false; if (m_overrideState.bForceBlock) { // If override forces "reject" decision, mention it in the "override" log field. logOverride = OVERRIDE_DROP; shouldBlock = true; - } - else if (m_overrideState.bForceException) { + } else if (m_overrideState.bForceException) { // If override forces "allow" decision, mention it in the "override" log field. logOverride = OVERRIDE_ACCEPT; } else if (m_scanner.getIgnoreOverride()) { @@ -2024,6 +2037,9 @@ Waf2Transaction::decideAutonomousSecurity( if (m_overrideState.bForceBlock) { dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() << " and override forces REJECT ..."; + if (!decision->shouldBlock()) { + m_effectiveOverrideIds.insert(m_overrideState.forceBlockIds.begin(), m_overrideState.forceBlockIds.end()); + } decision->setBlock(true); if (!m_overrideState.bIgnoreLog) { @@ -2033,6 +2049,17 @@ Waf2Transaction::decideAutonomousSecurity( else if (m_overrideState.bForceException) { dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() << " and override forces ALLOW ..."; + if (m_scanResult) { + // on accept exception the decision is not set and needs to be calculated to determine effectivness + ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score); + bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat); + if (shouldBlock) { + m_effectiveOverrideIds.insert( + m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end() + ); + } + } + decision->setBlock(false); if (!m_overrideState.bIgnoreLog) { @@ -2041,8 +2068,13 @@ Waf2Transaction::decideAutonomousSecurity( } - - if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO) { + bool log_all = false; + const std::shared_ptr triggerPolicy = sitePolicy.get_TriggerPolicy(); + if (triggerPolicy) { + const std::shared_ptr triggerLog = getTriggerLog(triggerPolicy); + if (triggerLog && triggerLog->webRequests) log_all = true; + } + if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) { decision->setLog(false); } else { decision->setLog(true); @@ -2171,8 +2203,10 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) { bool Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { auto exceptions = getConfiguration("rulebase", "exception"); - if (!exceptions.ok()) return false; - + if (!exceptions.ok()) { + dbgInfo(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr(); + return false; + } dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions"; std::unordered_map> exceptions_dict; @@ -2212,13 +2246,20 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { // calling behavior and check if there is a behavior that match to this specific param name. auto behaviors = exceptions.unpack().getBehavior(exceptions_dict, getAssetState()->m_filtersMngr->getMatchedOverrideKeywords()); - for (auto const &behavior : behaviors) { - if (behavior == action_ignore) { + for (const auto &behavior : behaviors) { + if (behavior == action_ignore) + { dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore."; std::string overrideId = behavior.getId(); if (!overrideId.empty()) { m_matchedOverrideIds.insert(overrideId); } + if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId) + { + if (!overrideId.empty()) { + m_effectiveOverrideIds.insert(overrideId); + } + } return true; } } diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index e8114ca..ea81548 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -283,6 +283,7 @@ private: // Matched override IDs std::set m_matchedOverrideIds; + std::set m_effectiveOverrideIds; //csrf state Waap::CSRF::State m_csrfState; @@ -344,7 +345,6 @@ private: // Cached pointer to const triggerLog (hence mutable) mutable std::shared_ptr m_triggerLog; - Waf2TransactionFlags m_waf2TransactionFlags; // Grace period for logging diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index d0017d5..4295772 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -42,6 +42,7 @@ USE_DEBUG_FLAG(D_WAAP); USE_DEBUG_FLAG(D_WAAP_EVASIONS); USE_DEBUG_FLAG(D_WAAP_BASE64); USE_DEBUG_FLAG(D_WAAP_JSON); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); #define MIN_HEX_LENGTH 6 #define charToDigit(c) (c - '0') @@ -1186,7 +1187,7 @@ static const SingleRegex base64_key_value_detector_re( err, "base64_key_value"); static const SingleRegex json_key_value_detector_re( - "^[^<>{};,&\\?|=\\s]+={.+(?s):.+(?s)}\\z", + "\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", err, "json_key_value"); static const SingleRegex base64_key_detector_re( @@ -1444,6 +1445,7 @@ base64_variants b64Test ( dbgTrace(D_WAAP_BASE64) << " ===b64Test===: FINAL key = '" << key << "'"; } retVal = decodeBase64Chunk(s, start, s.end(), value); + dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After testing and conversion value = " << value << "retVal = '" << retVal <<"'"; if (!retVal) { diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index fa0cedd..baf16ca 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -41,6 +41,7 @@ using namespace std; USE_DEBUG_FLAG(D_WAAP); USE_DEBUG_FLAG(D_WAAP_ULIMITS); +USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER); WaapComponent::Impl::Impl() : pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT), @@ -90,7 +91,7 @@ WaapComponent::Impl::init(const std::string &waapDataFileName) reputationAggregator.init(); waapStateTable = Singleton::Consume::by(); - + bool success = waf2_proc_start(waapDataFileName); if (!success) { dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!"; @@ -162,7 +163,6 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event) std::string httpMethodStr = event.getHttpMethod(); dbgTrace(D_WAAP) << "start Transaction: " << httpMethodStr << " " << uri << " (REQUEST)"; - // See below.. Waf2TransactionFlags &waf2TransactionFlags = waf2Transaction.getTransactionFlags(); waf2TransactionFlags.requestDataPushStarted = false; @@ -187,9 +187,11 @@ WaapComponent::Impl::respond(const NewHttpTransactionEvent &event) // Tell waf2 API that request headers started waf2Transaction.start_request_hdrs(); + return pending_response; } + // Request headers coming // Should return pending_response to hold the data (not send to upstream) EventVerdict @@ -209,6 +211,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event) dbgWarning(D_WAAP) << " * \e[31mNGEN_EVENT: http_header - " << "failed to get waf2 transaction, state not exist\e[0m"; + return drop_response; } IWaf2Transaction& waf2Transaction = waapStateTable->getState(); @@ -241,6 +244,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event) // Delete state before returning any verdict which is not pending if ((verdict.getVerdict() != pending_response.getVerdict()) && waapStateTable->hasState()) { finishTransaction(waf2Transaction); + } else { } return verdict; @@ -283,6 +287,7 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event) return EventVerdict(verdict); } + // Called when request ends and response starts. // For WAAP its time to decide and return either "accept_response" or "drop_response" EventVerdict @@ -339,7 +344,6 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event) } IWaf2Transaction& waf2Transaction = waapStateTable->getState(); - // TODO:: extract HTTP version from attachment? static const int http_version = 0x11; diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index 92c6654..4d21f21 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -55,8 +55,8 @@ public: EventVerdict respond(const EndTransactionEvent &) override; private: - void init(const std::string &waapDataFileName); + void init(const std::string &waapDataFileName); EventVerdict waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction); EventVerdict waapDecision(IWaf2Transaction& waf2Transaction); void finishTransaction(IWaf2Transaction& waf2Transaction); diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 44fe109..81f9f27 100755 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -68,6 +68,29 @@ isDirectory(const string &path) return false; } +Maybe> +getDirectoryFiles(const string &path) +{ + if (!isDirectory(path)) return genError("Path: " + path + " is not a directory"); + + struct dirent *entry = nullptr; + DIR *directory = opendir(path.c_str()); + + if (!directory) { + dbgWarning(D_INFRA_UTILS) << "Fail to open directory. Path: " << path << ", Errno: " << errno; + return genError("Failed to open directory: " + path); + } + + vector files; + while ((entry = readdir(directory))) { + if (entry->d_type == DT_REG) files.push_back(entry->d_name); + } + + closedir(directory); + + return files; +} + bool makeDir(const string &path, mode_t permission) { @@ -257,13 +280,17 @@ regexMatch(const char *file, int line, const char *sample, cmatch &match, const try { return regex_match(sample, match, regex); } catch (const runtime_error &err) { + uint sample_len = strlen(sample); dbgError(D_INFRA_UTILS) << "FAILURE during regex_match @ " << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample_len + << " sample='" + << string(sample, min(100u, sample_len)) + << "', pattern='" << regex.str() << "': " << err.what(); @@ -282,8 +309,11 @@ regexMatch(const char *file, int line, const string &sample, smatch &match, cons << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample.size() + << " sample='" + << sample.substr(0, 100) + << "', pattern='" << regex.str() << "': " << err.what(); @@ -302,8 +332,11 @@ regexMatch(const char *file, int line, const string &sample, const regex ®ex) << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample.size() + << " sample='" + << sample.substr(0, 100) + << "', pattern='" << regex.str() << "': " << err.what(); @@ -322,8 +355,11 @@ regexMatch(const char *file, int line, string &sample, const regex ®ex) << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample.size() + << " sample='" + << sample.substr(0, 100) + << "', pattern='" << regex.str() << "': " << err.what(); @@ -342,8 +378,11 @@ regexSearch(const char *file, int line, const string &sample, smatch &match, con << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample.size() + << " sample='" + << sample.substr(0, 100) + << "', pattern='" << regex.str() << "': " << err.what(); @@ -362,8 +401,11 @@ regexReplace(const char *file, int line, const string &sample, const regex ®e << file << ":" << line - << "; sample='" - << sample << "', pattern='" + << "; sample size: " + << sample.size() + << " sample='" + << sample.substr(0, 100) + << "', pattern='" << regex.str() << "', replace='" << replace diff --git a/core/config/config.cc b/core/config/config.cc index ce0a843..f7a69b7 100644 --- a/core/config/config.cc +++ b/core/config/config.cc @@ -52,12 +52,15 @@ public: class LoadNewConfigurationStatus : public ClientRest { public: - LoadNewConfigurationStatus(uint _id, bool _error, bool end) : id(_id), error(_error), finished(end) {} + LoadNewConfigurationStatus(uint _id, string _service_name, bool _error, bool end) + : + id(_id), service_name(_service_name), error(_error), finished(end) {} void setError(const string &error) { error_message = error; } private: C2S_PARAM(int, id); + C2S_PARAM(string, service_name); C2S_PARAM(bool, error); C2S_PARAM(bool, finished); C2S_OPTIONAL_PARAM(string, error_message); @@ -133,7 +136,7 @@ public: void registerExpectedSetting(unique_ptr> &&config) override; - bool loadConfiguration(istream &json_contents) override; + bool loadConfiguration(istream &json_contents, const string &path) override; bool loadConfiguration(const vector &configuration_flags) override; AsyncLoadConfigStatus reloadConfiguration(const string &version, bool is_async, uint id) override; bool saveConfiguration(ostream &) const override { return false; } @@ -565,13 +568,13 @@ ConfigComponent::Impl::registerExpectedSetting(unique_ptr> } bool -ConfigComponent::Impl::loadConfiguration(istream &stream) +ConfigComponent::Impl::loadConfiguration(istream &stream, const string &path) { vector> archive; try { archive.emplace_back(make_shared(stream)); } catch (const cereal::Exception &e) { - dbgError(D_CONFIG) << "Failed to load stream: " << e.what(); + dbgError(D_CONFIG) << "Failed to serialize stream. Path: " << path << ", Error: " << e.what(); return false; } return loadConfiguration(archive, false); @@ -872,7 +875,12 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as for (const auto &file : files) { if (file.second->is_open()) { dbgTrace(D_CONFIG) << "Succesfully opened configuration file. File: " << file.first; - archives.push_back(make_shared(*file.second)); + try { + archives.push_back(make_shared(*file.second)); + } catch (const cereal::Exception &e) { + dbgError(D_CONFIG) << "Failed in file serialization. Path: " << file.first << ", Error: " << e.what(); + return false; + } } else { dbgTrace(D_CONFIG) << "Could not open configuration file. Path: " << file.first; } @@ -904,8 +912,9 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version { dbgFlow(D_CONFIG) << "Running reloadConfigurationContinuesWrapper. Version: " << version << ", Id: " << id; auto mainloop = Singleton::Consume::by(); - - LoadNewConfigurationStatus in_progress(id, false, false); + auto maybe_service_name = Singleton::Consume::by()->get("Service Name"); + string service_name = maybe_service_name.ok() ? maybe_service_name.unpack() : "serviceNameNotRegistered"; + LoadNewConfigurationStatus in_progress(id, service_name, false, false); auto routine_id = mainloop->addRecurringRoutine( I_MainLoop::RoutineType::Timer, std::chrono::seconds(30), @@ -916,7 +925,7 @@ ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version bool res = reloadConfigurationImpl(version, true); mainloop->stop(routine_id); - LoadNewConfigurationStatus finished(id, !res, true); + LoadNewConfigurationStatus finished(id, service_name, !res, true); if (!res) finished.setError("Failed to reload configuration"); sendOrchestatorReloadStatusMsg(finished); diff --git a/core/include/attachments/nginx_attachment_common.h b/core/include/attachments/nginx_attachment_common.h index 89697b4..7f6353f 100755 --- a/core/include/attachments/nginx_attachment_common.h +++ b/core/include/attachments/nginx_attachment_common.h @@ -128,6 +128,8 @@ typedef enum ngx_http_plugin_metric_type AVERAGE_RSS_MEMORY_USAGE, MAX_VM_MEMORY_USAGE, MAX_RSS_MEMORY_USAGE, + REQUEST_OVERALL_SIZE_COUNT, + RESPONSE_OVERALL_SIZE_COUNT, METRIC_TYPES_COUNT } ngx_http_plugin_metric_type_e; diff --git a/core/include/services_sdk/resources/config/i_config.h b/core/include/services_sdk/resources/config/i_config.h index 68e1586..70e5ab3 100644 --- a/core/include/services_sdk/resources/config/i_config.h +++ b/core/include/services_sdk/resources/config/i_config.h @@ -88,7 +88,7 @@ public: // TODO: merge both loadConfiguration functions to one with vector of streams input when moving to c++17 // (c++ < 17 does not support copy of streams and thus it cannot be part of any container) - virtual bool loadConfiguration(istream &json_contents) = 0; + virtual bool loadConfiguration(istream &json_contents, const string &path = "") = 0; virtual bool loadConfiguration(const vector &configuration_flags) = 0; virtual AsyncLoadConfigStatus diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 3db35b1..bf408c5 100755 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -60,6 +60,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_STREAMING_DATA, D_STREAMING) DEFINE_FLAG(D_CHECKSUM, D_STREAMING) DEFINE_FLAG(D_WAAP, D_COMPONENT) + DEFINE_FLAG(D_OA_SCHEMA_UPDATER, D_WAAP) DEFINE_FLAG(D_WAAP_API, D_WAAP) DEFINE_FLAG(D_WAAP_AUTOMATION, D_WAAP) DEFINE_FLAG(D_WAAP_REGEX, D_WAAP) @@ -76,6 +77,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_BASE64, D_WAAP) DEFINE_FLAG(D_WAAP_JSON, D_WAAP) DEFINE_FLAG(D_WAAP_BOT_PROTECTION, D_WAAP) + DEFINE_FLAG(D_WAAP_STREAMING_PARSING, D_WAAP) DEFINE_FLAG(D_WAAP_PARSER, D_WAAP) DEFINE_FLAG(D_WAAP_PARSER_XML, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_HTML, D_WAAP_PARSER) @@ -91,6 +93,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_PARSER_URLENCODE, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_PHPSERIALIZE, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_PERCENT, D_WAAP_PARSER) + DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_OVERRIDE, D_WAAP) DEFINE_FLAG(D_IPS, D_COMPONENT) diff --git a/core/include/services_sdk/resources/report/report_enums.h b/core/include/services_sdk/resources/report/report_enums.h index 6421991..999fad6 100755 --- a/core/include/services_sdk/resources/report/report_enums.h +++ b/core/include/services_sdk/resources/report/report_enums.h @@ -65,6 +65,7 @@ enum class Tags { HORIZON_TELEMETRY_METRICS, CROWDSEC, PLAYGROUND, + API_DISCOVERY, COUNT }; @@ -152,8 +153,10 @@ enum class IssuingEngine { IOT_NEXT, SDWAN, FILE_UPLOAD, - IDA_NEXT, - HORIZON_TELEMETRY_METRICS + IDA_NEXT_BLADE_REGISTRATION, + IDA_NEXT_CLIENT_IP_NOTIFY, + HORIZON_TELEMETRY_METRICS, + API_DISCOVERY }; } // namespace ReportIS diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index b4bb353..5cdd171 100755 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -18,6 +18,8 @@ #include #include +#include "maybe_res.h" + namespace NGEN { @@ -26,7 +28,7 @@ namespace Filesystem bool exists(const std::string &path); bool isDirectory(const std::string &path); - +Maybe> getDirectoryFiles(const std::string &path); bool makeDir(const std::string &path, mode_t permission = S_IRWXU); bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU); bool deleteDirectory(const std::string &path, bool delete_content = false); diff --git a/core/report/tag_and_enum_management.cc b/core/report/tag_and_enum_management.cc index 13b991b..971af9f 100755 --- a/core/report/tag_and_enum_management.cc +++ b/core/report/tag_and_enum_management.cc @@ -107,9 +107,10 @@ TagAndEnumManagement::convertStringToTag(const string &tag) {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL}, {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}, {"Crowdsec", ReportIS::Tags::CROWDSEC}, + {"apiDiscoveryCloudMessaging", ReportIS::Tags::API_DISCOVERY}, {"Playground", ReportIS::Tags::PLAYGROUND} }; - + auto report_is_tag = strings_to_tags.find(tag); if (report_is_tag != strings_to_tags.end()) return report_is_tag->second; return genError("illegal tag: " + tag); @@ -267,7 +268,9 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine) case IssuingEngine::IOT_NEXT: return "iotNext"; case IssuingEngine::SDWAN: return "sdwanGwSharing"; case IssuingEngine::FILE_UPLOAD: return "fileUpload"; - case IssuingEngine::IDA_NEXT: return "quantumMetaNotifyIdn"; + case IssuingEngine::IDA_NEXT_BLADE_REGISTRATION: return "quantumMetaNotifyIdn"; + case IssuingEngine::IDA_NEXT_CLIENT_IP_NOTIFY: return "quantumIPNotifyIdn"; + case IssuingEngine::API_DISCOVERY: return "apiDiscoveryCloudMessaging"; case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics"; } @@ -310,7 +313,8 @@ EnumArray TagAndEnumManagement::tags_translation_arr { "Layer 7 Access Control", "Horizon Telemetry Metrics", "Crowdsec", - "Playground" + "Playground", + "apiDiscoveryCloudMessaging" }; EnumArray TagAndEnumManagement::audience_team_translation { diff --git a/core/version/version_ut/version_ut.cc b/core/version/version_ut/version_ut.cc index f830d1e..ff6744d 100755 --- a/core/version/version_ut/version_ut.cc +++ b/core/version/version_ut/version_ut.cc @@ -17,49 +17,9 @@ TEST(Version, format) ContainsRegex("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[-+][0-9]{4}") ); - // "Build 123" or "GitID 7d67870" - EXPECT_THAT(Version::getID(), ContainsRegex("([0-9]+)|[0-9]{4}.([0-9]+)")); - - // get() return all parts of information, timestamp and id. - EXPECT_THAT(Version::get(), ContainsRegex("([0-9]+)|[0-9]{4}.([0-9]+)")); EXPECT_THAT(Version::get(), ContainsRegex("[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}[-+][0-9]{4}")); } -TEST(Version, getVerPrefix) -{ - EXPECT_EQ("1.", Version::getVerPrefix()); -} - -TEST(Version, getUser) -{ - if (Version::isPublic()) { - // public builds call this function but don't use the return value - // ut will do the same, as the user name is not accessible in public builds. - auto user = Version::getUser(); - - const char* buffer = getenv("CI_BUILD_REF_NAME"); - ASSERT_FALSE(!buffer); - EXPECT_THAT(Version::getBranch(), AnyOf(buffer, StartsWith("pipeline"))); - } else { - // Version::getUser is define by the python function: getpass.getuser(). - // The getuser() function displays the login name of the user. - // This function checks the environment variables LOGNAME, USER, LNAME and USERNAME, in order, - // and returns the value of the first non-empty string. - const char* buffer = getenv("LOGNAME"); - if (!buffer) { - buffer = getenv("USER"); - if (!buffer) { - buffer = getenv("LNAME"); - if (!buffer) { - buffer = getenv("USERNAME"); - } - } - } - ASSERT_FALSE(!buffer); - EXPECT_EQ(buffer, Version::getUser()); - EXPECT_EQ(Version::getBranch(), "private"); - } -} unique_ptr show_version; bool showVersion(const unique_ptr &p) { show_version = p->getRest(); return true; } diff --git a/nodes/orchestration/package/local-default-policy.yaml b/nodes/orchestration/package/local-default-policy.yaml index e963ebf..efc0223 100644 --- a/nodes/orchestration/package/local-default-policy.yaml +++ b/nodes/orchestration/package/local-default-policy.yaml @@ -52,7 +52,7 @@ log-triggers: url-path: false url-query: false log-destination: - cloud: true + cloud: false stdout: format: json diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 0eb9eb1..84ccb69 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -964,13 +964,21 @@ run_status() # Initials - rs if echo "$rs_orch_status" | grep -q "update status"; then rs_line_count=$(echo "$rs_orch_status" | grep -c '^') rs_policy_load_time="$(echo "${rs_orch_status}" | grep "Last policy update"| sed "s|\"||g" | sed "s|,||g")" - rs_ai_model_ver="$(echo "${rs_orch_status}" | grep "AI model version"| sed "s|\"||g" | sed "s|,||g")" rs_temp_old_status=$(echo "$rs_orch_status" | sed -r "${rs_line_count},${rs_line_count}d; "' 1,1d; s/^\s*//g; s/^\n//g; s/\"//g; s/\\n/\n/g; s/\,//g') else rs_temp_old_status=$(sed 's/{//g' <${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestration_status.json | sed 's/}//g' | sed 's/"//g' | sed 's/,//g' | sed -r '/^\s*$/d' | sed -r 's/^ //g') rs_policy_load_time="$(cat ${FILESYSTEM_PATH}/conf/orchestration_status.json | grep "Last policy update" | sed "s|\"||g" | sed "s|,||g")" - rs_ai_model_ver="$(cat ${FILESYSTEM_PATH}/conf/orchestration_status.json | grep "AI model version" | sed "s|\"||g" | sed "s|,||g")" + fi + + if [ -f ${FILESYSTEM_PATH}/conf/waap/waap.data ]; then + rs_ai_model_ver="$(cat ${FILESYSTEM_PATH}/conf/waap/waap.data | ${FILESYSTEM_PATH}/${CP_PICOJSON_PATH} | grep 'model_version')" + + if [ -z "$rs_ai_model_ver" ]; then + echo "AI model version: None" + else + echo "$rs_ai_model_ver" | sed "s/\"//g; s/,//g; s/model_version/AI model version/g; s/^[ \t]*//" + fi fi if [ -n "$(cat ${FILESYSTEM_PATH}/conf/agent_details.json | grep "hybrid_mode")" ]; then @@ -1010,7 +1018,6 @@ run_status() # Initials - rs fi echo "Policy load status: ${rs_policy_load_status}" echo ${rs_policy_load_time} - echo ${rs_ai_model_ver} echo "" for service in $all_services; do @@ -1258,7 +1265,7 @@ run_ai() # Initials - ra ra_https_prefix="https://" ra_agent_details=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/agent_details.json) if echo "$ra_agent_details" | grep -q "Fog domain"; then - ra_fog_address=$(printf "%s" "$ra_agent_details" | grep "Fog domain" | cut -d '"' -f4) + [ -f ${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestrations_status.json ] && ra_orch_status=$(cat ${FILESYSTEM_PATH}/$cp_nano_conf_location/orchestration_status.json) ra_tenant_id=$(printf "%s" "$ra_agent_details" | grep "Tenant ID" | cut -d '"' -f4) ra_agent_id=$(printf "%s" "$ra_agent_details" | grep "Agent ID" | cut -d '"' -f4) else diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 8ab2fcb..92fd5a6 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -709,6 +709,7 @@ copy_orchestration_executable() cp_copy open-appsec-cloud-mgmt ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-cloud-mgmt cp_copy open-appsec-cloud-mgmt-k8s ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-cloud-mgmt-k8s cp_copy open-appsec-ctl.sh ${FILESYSTEM_PATH}/${SCRIPTS_PATH}/open-appsec-ctl.sh + if [ $var_hybrid_mode = true ]; then if [ -f /ext/appsec/local_policy.yaml ]; then cp_exec "ln -s /ext/appsec/local_policy.yaml ${FILESYSTEM_PATH}/${CONF_PATH}/local_policy.yaml"