github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Nov_12_2023-Dev

Browse Source
This commit is contained in:
Ned Wright 2023-11-12 18:50:17 +00:00
parent 0869b8f24d
commit 3061342b45
114 changed files with 3627 additions and 1305 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
attachments/kernel_modules/core/include/common_is/kdebug_flags.h
View File

@ -32,5 +32,6 @@ DEFINE_KDEBUG_FLAG(statelessValidation)
DEFINE_KDEBUG_FLAG(kernelMetric)
DEFINE_KDEBUG_FLAG(tproxy)
DEFINE_KDEBUG_FLAG(tenantStats)
DEFINE_KDEBUG_FLAG(uuidTranslation)
#endif // DEFINE_KDEBUG_FLAG

26
components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc
View File

@ -49,6 +49,8 @@ nginxIntakerEvent::resetAllCounters()
req_proccessing_timeout = 0;
res_proccessing_timeout = 0;
req_failed_to_reach_upstream = 0;
req_overall_size = 0;
res_overall_size = 0;
cpu_event.setCPU(0);
}
@ -249,10 +251,22 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie
cpu_event.setCPU(amount);
break;
}
case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: {
req_overall_size += amount;
static const uint64_t max_expected_res_size = 100ULL * 1024 * 1024 * 1024;
if (amount > max_expected_res_size) {
dbgWarning(D_METRICS_NGINX_ATTACHMENT) << "Requests sizes higher than expected: " << amount;
}
break;
}
case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: {
res_overall_size += amount;
break;
}
default:
dbgWarning(D_METRICS_NGINX_ATTACHMENT)
<< "Unsupported metric type. Type: " << static_cast<int>(metric_type);
return;
break;
}
}
}
@ -353,6 +367,10 @@ nginxIntakerEvent::getPluginMetricCounter(ngx_http_plugin_metric_type_e metric_t
return req_failed_to_reach_upstream;
case ngx_http_plugin_metric_type_e::CPU_USAGE:
return static_cast<uint64_t>(cpu_event.getCPU());
case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT:
return req_overall_size;
case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT:
return res_overall_size;
default:
dbgWarning(D_METRICS_NGINX_ATTACHMENT)
<< "Unsupported metric type. Type: " << static_cast<int>(metric_type);
@ -498,5 +516,11 @@ nginxIntakerMetric::upon(const nginxIntakerEvent &event)
req_failed_to_reach_upstream.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM)
);
req_overall_size.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT)
);
res_overall_size.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT)
);
event.notifyCPU();
}

1
components/include/generic_rulebase/parameters_config.h
View File

@ -224,5 +224,6 @@ private:
};
static const ParameterBehavior action_ignore(BehaviorKey::ACTION, BehaviorValue::IGNORE);
static const ParameterBehavior action_accept(BehaviorKey::ACTION, BehaviorValue::ACCEPT);
#endif //__PARAMETERS_CONFIG_H__

1
components/include/i_orchestration_status.h
View File

@ -34,7 +34,6 @@ public:
virtual const std::string & getUpdateTime() const = 0;
virtual const std::string & getLastManifestUpdate() const = 0;
virtual const std::string & getPolicyVersion() const = 0;
virtual const std::string & getWaapModelVersion() const = 0;
virtual const std::string & getLastPolicyUpdate() const = 0;
virtual const std::string & getLastSettingsUpdate() const = 0;
virtual const std::string & getUpgradeMode() const = 0;

2
components/include/i_service_controller.h
View File

@ -31,7 +31,7 @@ public:
virtual const std::string & getPolicyVersions() const = 0;
virtual const std::string & getPolicyVersion() const = 0;
virtual const std::string & getUpdatePolicyVersion() const = 0;
virtual void updateReconfStatus(int id, ReconfStatus status) = 0;
virtual void updateReconfStatus(int id, const std::string &service_name, ReconfStatus status) = 0;
virtual void startReconfStatus(
int id,
ReconfStatus status,

8
components/include/local_policy_mgmt_gen.h
View File

@ -19,13 +19,19 @@
#include "i_mainloop.h"
#include "i_local_policy_mgmt_gen.h"
#include "i_env_details.h"
#include "i_shell_cmd.h"
#include "i_orchestration_tools.h"
class LocalPolicyMgmtGenerator
:
public Component,
Singleton::Provide<I_LocalPolicyMgmtGen>,
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_EnvDetails>
Singleton::Consume<I_EnvDetails>,
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_OrchestrationTools>,
Singleton::Consume<I_Messaging>
{
public:
LocalPolicyMgmtGenerator();

4
components/include/nginx_intaker_metric.h
View File

@ -81,6 +81,8 @@ private:
uint64_t req_proccessing_timeout = 0;
uint64_t res_proccessing_timeout = 0;
uint64_t req_failed_to_reach_upstream = 0;
uint64_t req_overall_size = 0;
uint64_t res_overall_size = 0;
CPUEvent cpu_event;
};
@ -140,6 +142,8 @@ private:
Counter thread_failure{this, "attachmentThreadFailureSum"};
Counter req_proccessing_timeout{this, "httpRequestProcessingReachedTimeoutSum"};
Counter res_proccessing_timeout{this, "httpResponseProcessingReachedTimeoutSum"};
Counter req_overall_size{this, "httpRequestsSizeSum"};
Counter res_overall_size{this, "httpResponsesSizeSum"};
Counter req_failed_to_reach_upstream{this, "httpRequestFailedToReachWebServerUpstreamSum"};
};

2
components/include/orchestration_status.h
View File

@ -24,7 +24,6 @@
#include "i_time_get.h"
#include "i_mainloop.h"
#include "i_agent_details.h"
#include "i_details_resolver.h"
#include "customized_cereal_map.h"
class OrchestrationStatus
@ -33,7 +32,6 @@ class OrchestrationStatus
Singleton::Provide<I_OrchestrationStatus>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_DetailsResolver>,
Singleton::Consume<I_OrchestrationTools>,
Singleton::Consume<I_MainLoop>
{

4
components/include/rate_limit.h
View File

@ -7,13 +7,15 @@
#include "singleton.h"
#include "i_mainloop.h"
#include "i_environment.h"
#include "i_generic_rulebase.h"
class RateLimit
:
public Component,
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_TimeGet>,
Singleton::Consume<I_Environment>
Singleton::Consume<I_Environment>,
Singleton::Consume<I_GenericRulebase>
{
public:
RateLimit();

200
components/security_apps/layer_7_access_control/layer_7_access_control.cc
View File

@ -1,16 +1,14 @@
#include "layer_7_access_control.h"
#include <string>
#include <unordered_set>
#include <boost/algorithm/string/case_conv.hpp>
#include <unordered_set>
#include "config.h"
#include "cache.h"
#include "http_inspection_events.h"
#include "http_transaction_common.h"
#include "nginx_attachment_common.h"
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "log_generator.h"
@ -103,7 +101,7 @@ private:
unsigned int crowdsec_event_id;
};
class Layer7AccessControl::Impl : public Listener<HttpRequestHeaderEvent>
class Layer7AccessControl::Impl : public Listener<HttpRequestHeaderEvent>, Listener<WaitTransactionEvent>
{
public:
void init();
@ -126,27 +124,25 @@ public:
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
}
auto source_identifier = i_env->get<string>(HttpTransactionData::source_identifier);
if (source_identifier.ok() && IPAddr::createIPAddr(source_identifier.unpack()).ok()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Found a valid source identifier value: " << source_identifier.unpack();
return checkReputation(source_identifier.unpack());
}
return handleEvent();
}
auto orig_source_ip = i_env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
if (!orig_source_ip.ok()) {
dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
EventVerdict
respond(const WaitTransactionEvent &) override
{
dbgFlow(D_L7_ACCESS_CONTROL) << "Handling wait verdict";
stringstream ss_client_ip;
ss_client_ip << orig_source_ip.unpack();
return checkReputation(ss_client_ip.str());
return handleEvent();
}
private:
void queryIntelligence();
void scheduleIntelligenceQuery(const string &ip);
void processIntelligenceResponse(const string &ip, const vector<AssetReply<IntelligenceIpReputation>> &response);
Maybe<IntelligenceIpReputation> getIpReputation(const string &ip);
ngx_http_cp_verdict_e checkReputation(const string &source_ip);
void generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const;
EventVerdict generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const;
EventVerdict queryIpReputation(const string &source_ip);
EventVerdict handleEvent();
bool isAppEnabled() const;
bool isPrevent() const;
@ -154,9 +150,12 @@ private:
Maybe<LogField, Context::Error> genLogField(const string &log_key, const string &env_key) const;
Maybe<LogField, Context::Error> genLogIPField(const string &log_key, const string &env_key) const;
bool is_intelligence_routine_running = false;
I_Environment *i_env = nullptr;
I_Intelligence_IS_V2 *i_intelligence = nullptr;
I_MainLoop *i_mainloop = nullptr;
TemporaryCache<string, IntelligenceIpReputation> ip_reputation_cache;
unordered_set<string> pending_ips;
};
bool
@ -177,79 +176,139 @@ Layer7AccessControl::Impl::isPrevent() const
return mode == "prevent";
}
void
Layer7AccessControl::Impl::scheduleIntelligenceQuery(const string &ip)
{
dbgFlow(D_L7_ACCESS_CONTROL) << "Scheduling intelligence query about reputation of IP: " << ip;
pending_ips.emplace(ip);
if (!is_intelligence_routine_running) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Starting intelligence routine";
is_intelligence_routine_running = true;
i_mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
[&] () { queryIntelligence(); },
"Check IP reputation"
);
}
}
Maybe<IntelligenceIpReputation>
Layer7AccessControl::Impl::getIpReputation(const string &ip)
{
dbgFlow(D_L7_ACCESS_CONTROL) << "Getting reputation of IP " << ip;
if (ip_reputation_cache.doesKeyExists(ip)) return ip_reputation_cache.getEntry(ip);
dbgTrace(D_L7_ACCESS_CONTROL) << "Not found in cache - about to query intelligence";
dbgTrace(D_L7_ACCESS_CONTROL) << ip << " reputation was not found in cache";
QueryRequest request = QueryRequest(
Condition::EQUALS,
"ipv4Addresses",
ip,
true,
AttributeKeyType::REGULAR
);
auto response = i_intelligence->queryIntelligence<IntelligenceIpReputation>(request);
if (!response.ok()) {
dbgWarning(D_L7_ACCESS_CONTROL) << "Failed to query intelligence about reputation of IP: " << ip;
return genError("Failed to query intelligence");
}
auto &unpacked_response = response.unpack();
if (unpacked_response.empty()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Intelligence reputation response collection is empty. IP is clean.";
return IntelligenceIpReputation();
}
for (const auto &intelligence_reply : unpacked_response) {
if (intelligence_reply.getAssetType() == crowdsec_asset_type && !intelligence_reply.getData().empty()){
dbgTrace(D_L7_ACCESS_CONTROL) << intelligence_reply.getData().front();
return intelligence_reply.getData().front();
}
}
return IntelligenceIpReputation();
return genError("Intelligence needed");
}
ngx_http_cp_verdict_e
Layer7AccessControl::Impl::checkReputation(const string &source_ip)
EventVerdict
Layer7AccessControl::Impl::queryIpReputation(const string &source_ip)
{
auto ip_reputation = getIpReputation(source_ip);
if (!ip_reputation.ok()) {
dbgWarning(D_L7_ACCESS_CONTROL) << "Could not query intelligence. Retruning default verdict";
bool is_drop_by_default = getProfileAgentSettingWithDefault<bool>(false, "layer7AccessControl.dropByDefault");
if (!(is_drop_by_default && isPrevent())) return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
generateLog(source_ip, IntelligenceIpReputation());
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
dbgTrace(D_L7_ACCESS_CONTROL) << "Scheduling Intelligence query - returning Wait verdict";
scheduleIntelligenceQuery(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
}
if (!ip_reputation.unpack().isMalicious()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip;
ip_reputation_cache.deleteEntry(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
ip_reputation_cache.emplaceEntry(source_ip, ip_reputation.unpack());
return generateLog(source_ip, ip_reputation.unpack());
}
if (isPrevent()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip;
generateLog(source_ip, ip_reputation.unpack());
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
EventVerdict
Layer7AccessControl::Impl::handleEvent()
{
auto source_identifier = i_env->get<string>(HttpTransactionData::source_identifier);
if (source_identifier.ok() && IPAddr::createIPAddr(source_identifier.unpack()).ok()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Found a valid source identifier value: " << source_identifier.unpack();
return queryIpReputation(source_identifier.unpack());
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip;
generateLog(source_ip, ip_reputation.unpack());
auto orig_source_ip = i_env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
if (orig_source_ip.ok()) {
stringstream ss_client_ip;
ss_client_ip << orig_source_ip.unpack();
return queryIpReputation(ss_client_ip.str());
}
dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
void
Layer7AccessControl::Impl::processIntelligenceResponse(
const string &ip,
const vector<AssetReply<IntelligenceIpReputation>> &response)
{
if (response.empty()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Intelligence reputation response collection is empty. IP is clean.";
ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation());
return;
}
for (const auto &intelligence_reply : response) {
if (intelligence_reply.getAssetType() == crowdsec_asset_type && !intelligence_reply.getData().empty()) {
dbgTrace(D_L7_ACCESS_CONTROL) << intelligence_reply.getData().front();
ip_reputation_cache.emplaceEntry(ip, intelligence_reply.getData().front());
return;
}
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Could not find a matching intelligence asset type for IP: " << ip;
ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation());
}
void
Layer7AccessControl::Impl::queryIntelligence()
{
dbgFlow(D_L7_ACCESS_CONTROL) << "Started IP reputation intelligence routine";
while (!pending_ips.empty()) {
i_mainloop->yield();
auto ip = *(pending_ips.begin());
pending_ips.erase(pending_ips.begin());
if (ip_reputation_cache.doesKeyExists(ip)) continue;
dbgTrace(D_L7_ACCESS_CONTROL) << "Querying intelligence about reputation of IP: " << ip;
QueryRequest request = QueryRequest(
Condition::EQUALS,
"ipv4Addresses",
ip,
true,
AttributeKeyType::REGULAR
);
auto response = i_intelligence->queryIntelligence<IntelligenceIpReputation>(request);
if (!response.ok()) {
dbgWarning(D_L7_ACCESS_CONTROL)
<< "Failed to query intelligence about reputation of IP: "
<< ip
<< ", error: "
<< response.getErr();
ip_reputation_cache.emplaceEntry(ip, IntelligenceIpReputation());
continue;
}
processIntelligenceResponse(ip, response.unpack());
}
is_intelligence_routine_running = false;
}
EventVerdict
Layer7AccessControl::Impl::generateLog(const string &source_ip, const IntelligenceIpReputation &ip_reputation) const
{
dbgFlow(D_L7_ACCESS_CONTROL) << "About to generate Layer-7 Access Control log";
@ -287,6 +346,14 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen
<< ip_reputation.getOrigin()
<< ip_reputation.getIpv4Address()
<< ip_reputation.getScenario();
if (isPrevent()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
Maybe<LogField, Context::Error>
@ -315,6 +382,7 @@ Layer7AccessControl::Impl::init()
registerListener();
i_env = Singleton::Consume<I_Environment>::by<Layer7AccessControl>();
i_intelligence = Singleton::Consume<I_Intelligence_IS_V2>::by<Layer7AccessControl>();
i_mainloop = Singleton::Consume<I_MainLoop>::by<Layer7AccessControl>();
chrono::minutes expiration(
getProfileAgentSettingWithDefault<uint>(60u, "layer7AccessControl.crowdsec.cacheExpiration")
@ -322,7 +390,7 @@ Layer7AccessControl::Impl::init()
ip_reputation_cache.startExpiration(
expiration,
Singleton::Consume<I_MainLoop>::by<Layer7AccessControl>(),
i_mainloop,
Singleton::Consume<I_TimeGet>::by<Layer7AccessControl>()
);
}

96
components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc
View File

@ -52,6 +52,7 @@ public:
const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
const EventVerdict wait_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
Layer7AccessControl l7_access_control;
::Environment env;
ConfigComponent config;
@ -62,6 +63,7 @@ public:
NiceMock<MockRestApi> mock_rest;
AgentDetails agent_details;
IntelligenceComponentV2 intelligence_comp;
I_MainLoop::Routine query_intelligence_routine;
Context ctx;
};
@ -273,6 +275,13 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1 };
const HttpHeader header3{ Buffer("x-forwarded-for"), Buffer("1.2.3.4"), 2, true};
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Check IP reputation", _))
.WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0))
);
EXPECT_CALL(mock_ml, yield(A<bool>())).Times(1);
EXPECT_THAT(
HttpRequestHeaderEvent(header1).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", inspect_verdict))
@ -283,6 +292,13 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
);
EXPECT_THAT(
HttpRequestHeaderEvent(header3).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", wait_verdict))
);
query_intelligence_routine();
EXPECT_THAT(
WaitTransactionEvent().performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", accept_verdict))
);
}
@ -299,6 +315,13 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE)
).WillOnce(Return(malicious_intelligence_response));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Check IP reputation", _))
.WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0))
);
EXPECT_CALL(mock_ml, yield(A<bool>())).Times(1);
registerTransactionData();
ctx.registerValue<string>(HttpTransactionData::source_identifier, "1.2.3.4");
const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 };
@ -310,7 +333,18 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict));
EXPECT_THAT(
HttpRequestHeaderEvent(header3).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", wait_verdict))
);
query_intelligence_routine();
EXPECT_THAT(
WaitTransactionEvent().performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", drop_verdict))
);
verifyReport(report, "1.2.3.4", "Prevent");
}
@ -327,6 +361,13 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE)
).WillOnce(Return(malicious_intelligence_response));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Check IP reputation", _))
.WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0))
);
EXPECT_CALL(mock_ml, yield(A<bool>())).Times(1);
registerTransactionData();
ctx.registerValue<string>(HttpTransactionData::source_identifier, "1.2.3.4");
const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 };
@ -338,7 +379,18 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(drop_verdict));
EXPECT_THAT(
HttpRequestHeaderEvent(header3).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", wait_verdict))
);
query_intelligence_routine();
EXPECT_THAT(
WaitTransactionEvent().performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", drop_verdict))
);
verifyReport(report, "1.2.3.4", "Prevent");
@ -361,6 +413,13 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect)
sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE)
).WillOnce(Return(malicious_intelligence_response));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Check IP reputation", _))
.WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0))
);
EXPECT_CALL(mock_ml, yield(A<bool>())).Times(1);
registerTransactionData();
ctx.registerValue<string>(HttpTransactionData::source_identifier, "1.2.3.4");
const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 };
@ -372,7 +431,18 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect)
EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header3).query(), ElementsAre(accept_verdict));
EXPECT_THAT(
HttpRequestHeaderEvent(header3).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", wait_verdict))
);
query_intelligence_routine();
EXPECT_THAT(
WaitTransactionEvent().performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", accept_verdict))
);
verifyReport(report, "1.2.3.4", "Detect");
}
@ -389,6 +459,13 @@ TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop)
sendMessage(true, _, _, _, _, _, _, MessageTypeTag::INTELLIGENCE)
).WillOnce(Return(malicious_intelligence_response));
EXPECT_CALL(
mock_ml,
addOneTimeRoutine(_, _, "Check IP reputation", _))
.WillOnce(DoAll(SaveArg<1>(&query_intelligence_routine), Return(0))
);
EXPECT_CALL(mock_ml, yield(A<bool>())).Times(1);
registerTransactionData();
const HttpHeader header1{ Buffer("Content-Type"), Buffer("application/json"), 0 };
const HttpHeader header2{ Buffer("date"), Buffer("Sun, 26 Mar 2023 18:45:22 GMT"), 1, true };
@ -397,7 +474,18 @@ TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop)
EXPECT_CALL(mock_logging, sendLog(_)).WillOnce(SaveArg<0>(&report));
EXPECT_THAT(HttpRequestHeaderEvent(header1).query(), ElementsAre(inspect_verdict));
EXPECT_THAT(HttpRequestHeaderEvent(header2).query(), ElementsAre(drop_verdict));
EXPECT_THAT(
HttpRequestHeaderEvent(header2).performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", wait_verdict))
);
query_intelligence_routine();
EXPECT_THAT(
WaitTransactionEvent().performNamedQuery(),
ElementsAre(Pair("Layer-7 Access Control app", drop_verdict))
);
verifyReport(report, "", "Prevent");
}

1
components/security_apps/local_policy_mgmt_gen/CMakeLists.txt
View File

@ -20,4 +20,5 @@ add_library(local_policy_mgmt_gen
new_exceptions.cc
access_control_practice.cc
configmaps.cc
reverse_proxy_section.cc
)

38
components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc
View File

@ -316,7 +316,7 @@ TriggersInWaapSection::save(cereal::JSONOutputArchive &out_ar) const
}
ParsedMatch::ParsedMatch(const string &_operator, const string &_tag, const string &_value)
:
:
operator_type(_operator),
tag(_tag),
value(_value)
@ -368,7 +368,7 @@ AppSecOverride::AppSecOverride(const SourcesIdentifiers &parsed_trusted_sources)
// LCOV_EXCL_START Reason: no test exist
AppSecOverride::AppSecOverride(const InnerException &parsed_exceptions)
:
:
id(parsed_exceptions.getBehaviorId()),
parsed_match(parsed_exceptions.getMatch())
{
@ -413,7 +413,7 @@ WebAppSection::WebAppSection(
const string &default_mode,
const AppSecTrustedSources &parsed_trusted_sources,
const vector<InnerException> &parsed_exceptions)
:
:
application_urls(_application_urls),
asset_id(_asset_id),
asset_name(_asset_name),
@ -460,7 +460,7 @@ WebAppSection::WebAppSection(
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources)
:
:
application_urls(_application_urls),
asset_id(_asset_id),
asset_name(_asset_name),
@ -477,6 +477,7 @@ WebAppSection::WebAppSection(
{
web_attack_mitigation = true;
web_attack_mitigation_action =
web_attack_mitigation_mode != "Prevent" ? "Transparent" :
web_attack_mitigation_severity == "critical" ? "low" :
web_attack_mitigation_severity == "high" ? "balanced" :
web_attack_mitigation_severity == "medium" ? "high" :
@ -584,6 +585,9 @@ ParsedRule::load(cereal::JSONInputArchive &archive_in)
parseAppsecJSONKey<string>("custom-response", custom_response, archive_in);
parseAppsecJSONKey<string>("source-identifiers", source_identifiers, archive_in);
parseAppsecJSONKey<string>("trusted-sources", trusted_sources, archive_in);
parseAppsecJSONKey<string>("upstream", rpm_upstream, archive_in);
parseAppsecJSONKey<string>("rp-settings", rpm_settings, archive_in);
parseAppsecJSONKey<bool>("ssl", rpm_is_ssl, archive_in);
try {
archive_in(cereal::make_nvp("host", host));
} catch (const cereal::Exception &e)
@ -620,6 +624,24 @@ ParsedRule::getMode() const
return mode;
}
const string &
ParsedRule::rpmGetUpstream() const
{
return rpm_upstream;
}
const std::string &
ParsedRule::rpmGetRPSettings() const
{
return rpm_settings;
}
bool
ParsedRule::rpmIsHttps() const
{
return rpm_is_ssl;
}
void
ParsedRule::setHost(const string &_host)
{
@ -691,6 +713,7 @@ AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading Appsec Linux Policy";
parseAppsecJSONKey<AppsecPolicySpec>("policies", policies, archive_in);
parseAppsecJSONKey<vector<RPMSettings>>("rp-settings", rpm_settings, archive_in);
parseAppsecJSONKey<vector<AppSecPracticeSpec>>("practices", practices, archive_in);
parseAppsecJSONKey<vector<AppsecTriggerSpec>>("log-triggers", log_triggers, archive_in);
parseAppsecJSONKey<vector<AppSecCustomResponseSpec>>("custom-responses", custom_responses, archive_in);
@ -745,6 +768,13 @@ AppsecLinuxPolicy::getAppsecSourceIdentifierSpecs() const
return sources_identifiers;
}
const vector<RPMSettings> &
AppsecLinuxPolicy::rpmGetRPSettings() const
{
return rpm_settings;
}
void
AppsecLinuxPolicy::addSpecificRule(const ParsedRule &_rule)
{

4
components/security_apps/local_policy_mgmt_gen/exceptions_section.cc
View File

@ -304,11 +304,13 @@ ExceptionMatch::getMatch() const
ExceptionBehavior::ExceptionBehavior(const string &_value)
{
key = _value == "suppressLog" ? "log" : "action";
value = key_to_action.at(_value);
try {
value = key_to_action.at(_value);
id = to_string(boost::uuids::random_generator()());
} catch (const boost::uuids::entropy_error &e) {
dbgWarning(D_LOCAL_POLICY) << "Failed to generate exception behavior UUID. Error: " << e.what();
} catch (std::exception &e) {
dbgWarning(D_LOCAL_POLICY) << "Failed to find exception name: " << _value << ". Error: " << e.what();
}
}

57
components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h
View File

@ -28,6 +28,7 @@
#include "triggers_section.h"
#include "exceptions_section.h"
#include "trusted_sources_section.h"
#include "reverse_proxy_section.h"
#include "new_practice.h"
class AppSecWebBotsURI
@ -148,7 +149,7 @@ public:
PracticeAdvancedConfig() {}
PracticeAdvancedConfig(const AppSecPracticeSpec &parsed_appsec_spec)
:
:
http_header_max_size(parsed_appsec_spec.getWebAttacks().getMaxHeaderSizeBytes()),
http_illegal_methods_allowed(0),
http_request_body_max_size(parsed_appsec_spec.getWebAttacks().getMaxBodySizeKb()),
@ -162,7 +163,7 @@ public:
int _http_request_body_max_size,
int _json_max_object_depth,
int _url_max_size)
:
:
http_header_max_size(_http_header_max_size),
http_illegal_methods_allowed(0),
http_request_body_max_size(_http_request_body_max_size),
@ -186,7 +187,7 @@ class TriggersInWaapSection
{
public:
TriggersInWaapSection(const LogTriggerSection &log_section)
:
:
trigger_type("log"),
id(log_section.getTriggerId()),
name(log_section.getTriggerName()),
@ -241,13 +242,13 @@ public:
AppsecPracticeAntiBotSection(const NewAppSecPracticeAntiBot &anti_bot) :
injected_uris(anti_bot.getIjectedUris()),
validated_uris(anti_bot.getValidatedUris())
{};
{};
// LCOV_EXCL_STOP
AppsecPracticeAntiBotSection(const AppSecPracticeAntiBot &anti_bot) :
injected_uris(anti_bot.getIjectedUris()),
validated_uris(anti_bot.getValidatedUris())
{};
{};
void save(cereal::JSONOutputArchive &out_ar) const;
@ -278,20 +279,20 @@ public:
);
WebAppSection(
const std::string &_application_urls,
const std::string &_asset_id,
const std::string &_asset_name,
const std::string &_rule_id,
const std::string &_rule_name,
const std::string &_practice_id,
const std::string &_practice_name,
const std::string &_context,
const std::string &_web_attack_mitigation_severity,
const std::string &_web_attack_mitigation_mode,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources);
const std::string &_application_urls,
const std::string &_asset_id,
const std::string &_asset_name,
const std::string &_rule_id,
const std::string &_rule_name,
const std::string &_practice_id,
const std::string &_practice_name,
const std::string &_context,
const std::string &_web_attack_mitigation_severity,
const std::string &_web_attack_mitigation_mode,
const PracticeAdvancedConfig &_practice_advanced_config,
const AppsecPracticeAntiBotSection &_anti_bots,
const LogTriggerSection &parsed_log_trigger,
const AppSecTrustedSources &parsed_trusted_sources);
void save(cereal::JSONOutputArchive &out_ar) const;
@ -331,7 +332,7 @@ public:
const std::string &_web_attack_mitigation_mode,
bool _web_attack_mitigation,
const PracticeAdvancedConfig &_practice_advanced_config)
:
:
application_urls(_application_urls),
asset_id(_asset_id),
asset_name(_asset_name),
@ -345,7 +346,7 @@ public:
web_attack_mitigation_mode(_web_attack_mitigation_mode),
web_attack_mitigation(_web_attack_mitigation),
practice_advanced_config(_practice_advanced_config)
{}
{}
void save(cereal::JSONOutputArchive &out_ar) const;
@ -371,7 +372,7 @@ public:
AppSecRulebase(
std::vector<WebAppSection> _webApplicationPractices,
std::vector<WebAPISection> _webAPIPractices)
:
:
webApplicationPractices(_webApplicationPractices),
webAPIPractices(_webAPIPractices) {}
@ -387,7 +388,7 @@ class AppSecWrapper
{
public:
AppSecWrapper(const AppSecRulebase &_app_sec)
:
:
app_sec_rulebase(_app_sec)
{}
@ -409,6 +410,9 @@ public:
const std::vector<std::string> & getPractices() const;
const std::string & getHost() const;
const std::string & getMode() const;
const std::string &rpmGetUpstream() const;
const std::string &rpmGetRPSettings() const;
bool rpmIsHttps() const;
void setHost(const std::string &_host);
void setMode(const std::string &_mode);
const std::string & getCustomResponse() const;
@ -424,6 +428,9 @@ private:
std::string custom_response;
std::string source_identifiers;
std::string trusted_sources;
std::string rpm_upstream;
std::string rpm_settings;
bool rpm_is_ssl = false;
};
class AppsecPolicySpec : Singleton::Consume<I_Environment>
@ -453,7 +460,7 @@ public:
const std::vector<AppsecException> &_exceptions,
const std::vector<TrustedSourcesSpec> &_trusted_sources,
const std::vector<SourceIdentifierSpecWrapper> &_sources_identifiers)
:
:
policies(_policies),
practices(_practices),
log_triggers(_log_triggers),
@ -471,6 +478,7 @@ public:
const std::vector<AppsecException> & getAppsecExceptions() const;
const std::vector<TrustedSourcesSpec> & getAppsecTrustedSourceSpecs() const;
const std::vector<SourceIdentifierSpecWrapper> & getAppsecSourceIdentifierSpecs() const;
const std::vector<RPMSettings> &rpmGetRPSettings() const;
void addSpecificRule(const ParsedRule &_rule);
private:
@ -481,6 +489,7 @@ private:
std::vector<AppsecException> exceptions;
std::vector<TrustedSourcesSpec> trusted_sources;
std::vector<SourceIdentifierSpecWrapper> sources_identifiers;
std::vector<RPMSettings> rpm_settings;
};
#endif // __APPSEC_PRACTICE_SECTION_H__

6
components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h
View File

@ -50,7 +50,7 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
static const std::unordered_map<std::string, std::string> key_to_practices_val = {
{ "prevent-learn", "Prevent"},
{ "detect-learn", "Detect"},
{ "detect-learn", "Learn"},
{ "prevent", "Prevent"},
{ "detect", "Detect"},
{ "inactive", "Inactive"}
@ -70,9 +70,9 @@ parseAppsecJSONKey(
archive_in.setNextName(nullptr);
value = default_value;
dbgDebug(D_LOCAL_POLICY)
<< "Could not parse the required key. Key: "
<< "Could not parse the required key. Key: \""
<< key_name
<< ", Error: "
<< "\", Error: "
<< e.what();
}
}

1
components/security_apps/local_policy_mgmt_gen/include/new_appsec_linux_policy.h
View File

@ -59,6 +59,7 @@ public:
trusted_sources(_trusted_sources),
sources_identifiers(_sources_identifiers) {}
// LCOV_EXCL_STOP
void serialize(cereal::JSONInputArchive &archive_in);
const NewAppsecPolicySpec & getAppsecPolicySpec() const;
const std::vector<NewAppSecPracticeSpec> & getAppSecPracticeSpecs() const;

16
components/security_apps/local_policy_mgmt_gen/include/new_practice.h
View File

@ -147,8 +147,8 @@ public:
// LCOV_EXCL_STOP
FileSecurityProtectionsSection(
int _file_size_limit,
int _archive_file_size_limit,
uint64_t _file_size_limit,
uint64_t _archive_file_size_limit,
bool _allow_files_without_name,
bool _required_file_size_limit,
bool _required_archive_extraction,
@ -171,8 +171,8 @@ public:
void save(cereal::JSONOutputArchive &out_ar) const;
private:
int file_size_limit;
int archive_file_size_limit;
uint64_t file_size_limit;
uint64_t archive_file_size_limit;
bool allow_files_without_name;
bool required_file_size_limit;
bool required_archive_extraction;
@ -233,13 +233,13 @@ class NewFileSecurityArchiveInspection
public:
void load(cereal::JSONInputArchive &archive_in);
int getArchiveFileSizeLimit() const;
uint64_t getArchiveFileSizeLimit() const;
bool getrequiredArchiveExtraction() const;
const std::string & getMultiLevelArchiveAction() const;
const std::string & getUnopenedArchiveAction() const;
private:
int scan_max_file_size;
uint64_t scan_max_file_size;
bool extract_archive_files;
std::string scan_max_file_size_unit;
std::string archived_files_within_archived_files;
@ -251,11 +251,11 @@ class NewFileSecurityLargeFileInspection
public:
void load(cereal::JSONInputArchive &archive_in);
int getFileSizeLimit() const;
uint64_t getFileSizeLimit() const;
const std::string & getFileSizeLimitAction() const;
private:
int file_size_limit;
uint64_t file_size_limit;
std::string file_size_limit_unit;
std::string files_exceeding_size_limit_action;
};

12
components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h
View File

@ -40,6 +40,7 @@
#include "trusted_sources_section.h"
#include "new_appsec_linux_policy.h"
#include "access_control_practice.h"
#include "reverse_proxy_section.h"
enum class AnnotationTypes {
PRACTICE,
@ -109,11 +110,6 @@ private:
};
class PolicyMakerUtils
:
Singleton::Consume<I_Environment>,
Singleton::Consume<I_OrchestrationTools>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_ShellCmd>
{
public:
std::string proccesSingleAppsecPolicy(
@ -206,6 +202,7 @@ private:
createThreatPreventionPracticeSections(
const std::string &asset_name,
const std::string &url,
const std::string &port,
const std::string &uri,
const std::string &default_mode,
const V1beta2AppsecLinuxPolicy &policy,
@ -231,6 +228,11 @@ private:
template<class T, class R>
void createAgentPolicyFromAppsecPolicy(const std::string &policy_name, const T &appsec_policy);
void rpmBuildNginxServers(const AppsecLinuxPolicy &policy);
void rpmReportInfo(const std::string &msg);
void rpmReportError(const std::string &msg);
std::string policy_version_name;
std::map<std::string, LogTriggerSection> log_triggers;
std::map<std::string, WebUserResponseTriggerSection> web_user_res_triggers;
std::map<std::string, std::vector<InnerException>> inner_exceptions;

68
components/security_apps/local_policy_mgmt_gen/include/reverse_proxy_section.h Normal file
View File

@ -0,0 +1,68 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __REVERSE_PROXY_SECTION_H__
#define __REVERSE_PROXY_SECTION_H__
#include <cereal/archives/json.hpp>
#include <unordered_map>
#include "agent_core_utilities.h"
#include "i_shell_cmd.h"
class ParsedRule;
class RPMSettings
{
public:
void load(cereal::JSONInputArchive &archive_in);
const std::string & getName() const;
std::string applySettings(const std::string &server_content) const;
private:
std::string name;
std::string host_hdr = "$host";
std::string dns_resolver = "127.0.0.11";
};
class ReverseProxyBuilder
{
public:
static void init();
static Maybe<void> addNginxServerLocation(
std::string location,
const std::string &host,
const ParsedRule &rule,
const RPMSettings &rp_settings);
static Maybe<void> createNewNginxServer(
const std::string &host,
const ParsedRule &rule,
const RPMSettings &rp_settings);
static std::string replaceTemplate(
const std::string &content,
const boost::regex &nginx_directive_template,
const std::string &value);
static Maybe<void> reloadNginx();
private:
static Maybe<void> createSSLNginxServer(const std::string &host, const RPMSettings &rp_settings);
static Maybe<void> createHTTPNginxServer(const std::string &host, const RPMSettings &rp_settings);
static Maybe<std::string> getTemplateContent(const std::string &nginx_template_name);
};
#endif // __REVERSE_PROXY_SECTION_H__

1
components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h
View File

@ -90,6 +90,7 @@ public:
RulesConfigRulebase(
const std::string &_name,
const std::string &_url,
const std::string &_port,
const std::string &_uri,
std::vector<PracticeSection> _practices,
std::vector<ParametersSection> _parameters,

5
components/security_apps/local_policy_mgmt_gen/local_policy_mgmt_gen.cc
View File

@ -55,9 +55,7 @@ const static string default_local_mgmt_policy_path = "/conf/local_policy.yaml";
class LocalPolicyMgmtGenerator::Impl
:
public Singleton::Provide<I_LocalPolicyMgmtGen>::From<LocalPolicyMgmtGenerator>,
public Singleton::Consume<I_MainLoop>,
public Singleton::Consume<I_EnvDetails>
public Singleton::Provide<I_LocalPolicyMgmtGen>::From<LocalPolicyMgmtGenerator>
{
public:
@ -111,7 +109,6 @@ public:
private:
PolicyMakerUtils policy_maker_utils;
};
LocalPolicyMgmtGenerator::LocalPolicyMgmtGenerator()

28
components/security_apps/local_policy_mgmt_gen/new_appsec_linux_policy.cc
View File

@ -70,3 +70,31 @@ V1beta2AppsecLinuxPolicy::addSpecificRule(const NewParsedRule &_rule)
policies.addSpecificRule(_rule);
}
// LCOV_EXCL_STOP
void
V1beta2AppsecLinuxPolicy::serialize(cereal::JSONInputArchive &archive_in)
{
dbgInfo(D_LOCAL_POLICY) << "Loading Appsec V1Beta2 Linux Policy";
// Check for the presence of "apiVersion" key, present only from V1Beta2
string api_version;
archive_in(cereal::make_nvp("apiVersion", api_version));
if (api_version != "v1beta2") throw cereal::Exception("Failed to parse JSON as v1Beta2 version");
parseAppsecJSONKey<NewAppsecPolicySpec>("policies", policies, archive_in);
parseAppsecJSONKey<vector<NewAppSecPracticeSpec>>(
"threatPreventionPractices",
threat_prevection_practices,
archive_in
);
parseAppsecJSONKey<vector<AccessControlPracticeSpec>>(
"accessControlPractices",
access_control_practices,
archive_in
);
parseAppsecJSONKey<vector<NewAppsecLogTrigger>>("logTriggers", log_triggers, archive_in);
parseAppsecJSONKey<vector<NewAppSecCustomResponse>>("customResponse", custom_responses, archive_in);
parseAppsecJSONKey<vector<NewAppsecException>>("exceptions", exceptions, archive_in);
parseAppsecJSONKey<vector<NewTrustedSourcesSpec>>("trustedSources", trusted_sources, archive_in);
parseAppsecJSONKey<vector<NewSourcesIdentifiers>>("sourcesIdentifiers", sources_identifiers, archive_in);
}

2
components/security_apps/local_policy_mgmt_gen/new_exceptions.cc
View File

@ -44,7 +44,7 @@ void
NewAppsecException::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading New AppSec exception";
parseAppsecJSONKey<string>("name", name, archive_in);
parseAppsecJSONKey<string>("name", name, archive_in, "exception");
parseAppsecJSONKey<string>("action", action, archive_in);
parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
if (valid_actions.count(action) == 0) {

14
components/security_apps/local_policy_mgmt_gen/new_practice.cc
View File

@ -42,7 +42,7 @@ static const std::unordered_map<std::string, std::string> key_to_mode_val = {
{ "detect", "Detect"},
{ "inactive", "Inactive"}
};
static const std::unordered_map<std::string, int> unit_to_int = {
static const std::unordered_map<std::string, uint64_t> unit_to_int = {
{ "bytes", 1},
{ "KB", 1024},
{ "MB", 1048576},
@ -631,8 +631,8 @@ NewIntrusionPrevention::getMode() const
}
FileSecurityProtectionsSection::FileSecurityProtectionsSection(
int _file_size_limit,
int _archive_file_size_limit,
uint64_t _file_size_limit,
uint64_t _archive_file_size_limit,
bool _allow_files_without_name,
bool _required_file_size_limit,
bool _required_archive_extraction,
@ -720,7 +720,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security Archive Inspection practice";
parseAppsecJSONKey<bool>("extractArchiveFiles", extract_archive_files, archive_in);
parseAppsecJSONKey<int>("scanMaxFileSize", scan_max_file_size, archive_in, 0);
parseAppsecJSONKey<uint64_t>("scanMaxFileSize", scan_max_file_size, archive_in, 0);
parseAppsecJSONKey<string>("scanMaxFileSizeUnit", scan_max_file_size_unit, archive_in, "bytes");
if (size_unit.count(scan_max_file_size_unit) == 0) {
dbgWarning(D_LOCAL_POLICY)
@ -749,7 +749,7 @@ NewFileSecurityArchiveInspection::load(cereal::JSONInputArchive &archive_in)
}
}
int
uint64_t
NewFileSecurityArchiveInspection::getArchiveFileSizeLimit() const
{
if (unit_to_int.find(scan_max_file_size_unit) == unit_to_int.end()) {
@ -784,7 +784,7 @@ void
NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading AppSec File Security large File Inspection practice";
parseAppsecJSONKey<int>("fileSizeLimit", file_size_limit, archive_in);
parseAppsecJSONKey<uint64_t>("fileSizeLimit", file_size_limit, archive_in);
parseAppsecJSONKey<string>("fileSizeLimitUnit", file_size_limit_unit, archive_in, "bytes");
if (size_unit.count(file_size_limit_unit) == 0) {
dbgWarning(D_LOCAL_POLICY)
@ -803,7 +803,7 @@ NewFileSecurityLargeFileInspection::load(cereal::JSONInputArchive &archive_in)
}
}
int
uint64_t
NewFileSecurityLargeFileInspection::getFileSizeLimit() const
{
if (unit_to_int.find(file_size_limit_unit) == unit_to_int.end()) {

2
components/security_apps/local_policy_mgmt_gen/new_trusted_sources.cc
View File

@ -64,7 +64,7 @@ void
Identifier::load(cereal::JSONInputArchive &archive_in)
{
dbgTrace(D_LOCAL_POLICY) << "Loading source identifiers spec";
parseAppsecJSONKey<string>("sourceIdentifier", identifier, archive_in);
parseAppsecJSONKey<string>("identifier", identifier, archive_in);
if (valid_identifiers.count(identifier) == 0) {
dbgWarning(D_LOCAL_POLICY) << "AppSec identifier invalid: " << identifier;
}

235
components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc
View File

@ -13,6 +13,11 @@
#include "policy_maker_utils.h"
#include <regex>
#include "local_policy_mgmt_gen.h"
#include "log_generator.h"
using namespace std;
USE_DEBUG_FLAG(D_NGINX_POLICY);
@ -58,7 +63,7 @@ template<class T>
Maybe<T>
PolicyMakerUtils::openFileAsJson(const string &path)
{
auto maybe_file_as_json = Singleton::Consume<I_ShellCmd>::by<PolicyMakerUtils>()->getExecOutput(
auto maybe_file_as_json = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(
getFilesystemPathConfig() + "/bin/yq " + path + " -o json"
);
@ -67,7 +72,7 @@ PolicyMakerUtils::openFileAsJson(const string &path)
return genError("Could not convert policy from yaml to json. Error: " + maybe_file_as_json.getErr());
}
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<PolicyMakerUtils>();
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
auto maybe_file = i_orchestration_tools->jsonStringToObject<T>(
maybe_file_as_json.unpack()
);
@ -136,10 +141,11 @@ PolicyMakerUtils::splitHostName(const string &host_name)
url = url.substr(0, url.find(":"));
}
if (host_name == "*") {
if (host_name == "*" || host_name == "*:*") {
url = "Any";
uri = "Any";
}
return make_tuple(url, port, uri);
}
@ -323,6 +329,7 @@ extractAnnotationsNames<NewParsedRule>(
if (!trusted_sources_annotation_name.empty()) {
rule_annotation[AnnotationTypes::TRUSTED_SOURCES] = policy_name + "/" + trusted_sources_annotation_name;
}
return rule_annotation;
}
// LCOV_EXCL_STOP
@ -451,6 +458,23 @@ getAppsecCustomResponseSpec(const string &custom_response_annotation_name, const
return *custom_response_it;
}
template<class T, class R>
R
rpmGetAppsecRPSettingSpec(const string &rp_settings_name, const T &policy)
{
auto rp_settings_vec = policy.rpmGetRPSettings();
auto rp_settings_it = extractElement(
rp_settings_vec.begin(),
rp_settings_vec.end(),
rp_settings_name);
if (rp_settings_it == rp_settings_vec.end()) {
dbgTrace(D_NGINX_POLICY) << "Failed to retrieve AppSec RP Settings";
return R();
}
return *rp_settings_it;
}
template<class T, class R>
R
getAppsecSourceIdentifierSpecs(const string &source_identifiers_annotation_name, const T &policy)
@ -843,6 +867,7 @@ createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
RulesConfigRulebase
createMultiRulesSections(
const string &url,
const string &port,
const string &uri,
const string &practice_id,
const string &practice_name,
@ -878,6 +903,7 @@ createMultiRulesSections(
RulesConfigRulebase rules_config = RulesConfigRulebase(
asset_name,
url,
port,
uri,
{practice},
exceptions_result,
@ -890,6 +916,7 @@ createMultiRulesSections(
RulesConfigRulebase
createMultiRulesSections(
const string &url,
const string &port,
const string &uri,
const string &practice_id,
const string &practice_name,
@ -907,7 +934,8 @@ createMultiRulesSections(
const string &exception_name,
const vector<InnerException> &exceptions)
{
ParametersSection exception_param = ParametersSection(exceptions[0].getBehaviorId(), exception_name);
string behaviorId = exceptions.empty() ? "" : exceptions[0].getBehaviorId();
ParametersSection exception_param = ParametersSection(behaviorId, exception_name);
vector<PracticeSection> practices;
if (!practice_id.empty()) {
@ -934,6 +962,7 @@ createMultiRulesSections(
RulesConfigRulebase rules_config = RulesConfigRulebase(
asset_name,
url,
port,
uri,
practices,
{exception_param},
@ -983,7 +1012,7 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, const st
auto snort_scriipt_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
auto cmd = "python " + snort_scriipt_path + " " + path + ".rule " + path + ".out " + path + ".err";
auto res = Singleton::Consume<I_ShellCmd>::by<PolicyMakerUtils>()->getExecOutput(cmd);
auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
if (!res.ok()) {
dbgWarning(D_LOCAL_POLICY) << res.getErr();
@ -996,7 +1025,7 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, const st
return;
}
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<PolicyMakerUtils>();
auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
i_orchestration_tools->removeFile(path + ".rule");
i_orchestration_tools->removeFile(path + ".out");
i_orchestration_tools->removeFile(path + ".err");
@ -1153,12 +1182,15 @@ void
PolicyMakerUtils::createThreatPreventionPracticeSections(
const string &asset_name,
const string &url,
const string &port,
const string &uri,
const string &default_mode,
const V1beta2AppsecLinuxPolicy &policy,
map<AnnotationTypes, string> &rule_annotations)
{
if (rule_annotations[AnnotationTypes::PRACTICE].empty()) {
if (rule_annotations[AnnotationTypes::PRACTICE].empty() ||
web_apps.count(asset_name)
) {
return;
}
string practice_id = "";
@ -1170,6 +1202,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
RulesConfigRulebase rule_config = createMultiRulesSections(
url,
port,
uri,
practice_id,
rule_annotations[AnnotationTypes::PRACTICE],
@ -1353,7 +1386,14 @@ PolicyMakerUtils::createPolicyElementsByRule(
);
}
if (!rule_annotations[AnnotationTypes::PRACTICE].empty()) {
string full_url = rule.getHost() == "*" || rule.getHost() == "*:*"
? "Any"
: rule.getHost();
if (!rule_annotations[AnnotationTypes::PRACTICE].empty() &&
!web_apps.count(full_url)
) {
string practice_id = "";
try {
practice_id = to_string(boost::uuids::random_generator()());
@ -1362,12 +1402,10 @@ PolicyMakerUtils::createPolicyElementsByRule(
}
tuple<string, string, string> splited_host_name = splitHostName(rule.getHost());
string full_url = rule.getHost() == "*"
? "Any"
: rule.getHost();
RulesConfigRulebase rule_config = createMultiRulesSections(
std::get<0>(splited_host_name),
std::get<1>(splited_host_name),
std::get<2>(splited_host_name),
practice_id,
rule_annotations[AnnotationTypes::PRACTICE],
@ -1426,7 +1464,9 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
dbgTrace(D_LOCAL_POLICY) << "Creating policy elements from version V1beta2";
map<AnnotationTypes, string> rule_annotations =
extractAnnotationsNames<NewParsedRule>(rule, default_rule, policy_name);
if (
rule_annotations.count(AnnotationTypes::TRIGGER) > 0 &&
!rule_annotations[AnnotationTypes::TRIGGER].empty() &&
!log_triggers.count(rule_annotations[AnnotationTypes::TRIGGER])
) {
@ -1438,6 +1478,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
}
if (
rule_annotations.count(AnnotationTypes::WEB_USER_RES) > 0 &&
!rule_annotations[AnnotationTypes::WEB_USER_RES].empty() &&
!web_user_res_triggers.count(rule_annotations[AnnotationTypes::WEB_USER_RES])
) {
@ -1449,6 +1490,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
}
if (
rule_annotations.count(AnnotationTypes::EXCEPTION) > 0 &&
!rule_annotations[AnnotationTypes::EXCEPTION].empty() &&
!inner_exceptions.count(rule_annotations[AnnotationTypes::EXCEPTION])
) {
@ -1460,6 +1502,8 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
}
if (
rule_annotations.count(AnnotationTypes::TRUSTED_SOURCES) > 0 &&
rule_annotations.count(AnnotationTypes::SOURCE_IDENTIFIERS) > 0 &&
!rule_annotations[AnnotationTypes::TRUSTED_SOURCES].empty() &&
!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty() &&
!trusted_sources.count(rule_annotations[AnnotationTypes::TRUSTED_SOURCES])
@ -1473,6 +1517,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
}
if (
rule_annotations.count(AnnotationTypes::PRACTICE) > 0 &&
!rule_annotations[AnnotationTypes::PRACTICE].empty() &&
!web_apps.count(rule_annotations[AnnotationTypes::PRACTICE])
) {
@ -1484,7 +1529,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
);
}
string full_url = rule.getHost() == "*"
string full_url = rule.getHost() == "*" || rule.getHost() == "*:*"
? "Any"
: rule.getHost();
tuple<string, string, string> splited_host_name = splitHostName(rule.getHost());
@ -1501,6 +1546,7 @@ PolicyMakerUtils::createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsed
createThreatPreventionPracticeSections(
full_url,
std::get<0>(splited_host_name),
std::get<1>(splited_host_name),
std::get<2>(splited_host_name),
rule.getMode(),
policy,
@ -1531,11 +1577,11 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy(const string &policy_name, c
R default_rule = appsec_policy.getAppsecPolicySpec().getDefaultRule();
// add default rule to policy
createPolicyElementsByRule<T, R>(default_rule, default_rule, appsec_policy, policy_name);
vector<R> specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules();
createPolicyElements<T, R>(specific_rules, default_rule, appsec_policy, policy_name);
// add default rule to policy
createPolicyElementsByRule<T, R>(default_rule, default_rule, appsec_policy, policy_name);
}
// LCOV_EXCL_START Reason: no test exist
@ -1545,17 +1591,10 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy<V1beta2AppsecLinuxPolicy, Ne
const string &policy_name,
const V1beta2AppsecLinuxPolicy &appsec_policy)
{
dbgTrace(D_LOCAL_POLICY) << "Proccesing policy, name: " << policy_name;
dbgTrace(D_LOCAL_POLICY) << "Proccesing v1beta2 policy, name: " << policy_name;
NewParsedRule default_rule = appsec_policy.getAppsecPolicySpec().getDefaultRule();
// add default rule to policy
createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsedRule>(
default_rule,
default_rule,
appsec_policy,
policy_name);
vector<NewParsedRule> specific_rules = appsec_policy.getAppsecPolicySpec().getSpecificRules();
createPolicyElements<V1beta2AppsecLinuxPolicy, NewParsedRule>(
specific_rules,
@ -1563,6 +1602,13 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy<V1beta2AppsecLinuxPolicy, Ne
appsec_policy,
policy_name
);
// add default rule to policy
createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsedRule>(
default_rule,
default_rule,
appsec_policy,
policy_name);
}
// LCOV_EXCL_STOP
@ -1572,15 +1618,31 @@ PolicyMakerUtils::proccesSingleAppsecPolicy(
const string &policy_version,
const string &local_appsec_policy_path)
{
Maybe<AppsecLinuxPolicy> maybe_policy = openFileAsJson<AppsecLinuxPolicy>(policy_path);
if (!maybe_policy.ok()){
dbgWarning(D_LOCAL_POLICY) << maybe_policy.getErr();
return "";
Maybe<V1beta2AppsecLinuxPolicy> maybe_policy_v1beta2 = openFileAsJson<V1beta2AppsecLinuxPolicy>(policy_path);
if (maybe_policy_v1beta2.ok()) {
policy_version_name = "v1beta2";
createAgentPolicyFromAppsecPolicy<V1beta2AppsecLinuxPolicy, NewParsedRule>(
getPolicyName(policy_path),
maybe_policy_v1beta2.unpack()
);
} else {
policy_version_name = "v1beta1";
dbgInfo(D_LOCAL_POLICY)
<< "Failed to retrieve AppSec local policy with version: v1beta2, Trying version: v1beta1";
Maybe<AppsecLinuxPolicy> maybe_policy_v1beta1 = openFileAsJson<AppsecLinuxPolicy>(policy_path);
if (!maybe_policy_v1beta1.ok()){
dbgWarning(D_LOCAL_POLICY) << maybe_policy_v1beta1.getErr();
return "";
}
createAgentPolicyFromAppsecPolicy<AppsecLinuxPolicy, ParsedRule>(
getPolicyName(policy_path),
maybe_policy_v1beta1.unpack()
);
if (getenv("OPENAPPSEC_STANDALONE")) rpmBuildNginxServers(maybe_policy_v1beta1.unpack());
}
createAgentPolicyFromAppsecPolicy<AppsecLinuxPolicy, ParsedRule>(
getPolicyName(policy_path),
maybe_policy.unpack()
);
PolicyWrapper policy_wrapper = combineElementsToPolicy(policy_version);
return dumpPolicyToFile(
@ -1588,3 +1650,114 @@ PolicyMakerUtils::proccesSingleAppsecPolicy(
local_appsec_policy_path
);
}
void
PolicyMakerUtils::rpmReportInfo(const std::string &msg)
{
dbgTrace(D_LOCAL_POLICY) << msg;
LogGen(
msg,
ReportIS::Audience::SECURITY,
ReportIS::Severity::INFO,
ReportIS::Priority::LOW,
ReportIS::Tags::ORCHESTRATOR
);
}
void
PolicyMakerUtils::rpmReportError(const std::string &msg)
{
dbgWarning(D_LOCAL_POLICY) << msg;
LogGen(
msg,
ReportIS::Audience::SECURITY,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::URGENT,
ReportIS::Tags::ORCHESTRATOR
);
}
void
PolicyMakerUtils::rpmBuildNginxServers(const AppsecLinuxPolicy &policy)
{
rpmReportInfo("Started building NGINX servers");
ReverseProxyBuilder::init();
bool full_success = true;
bool partial_success = false;
set<pair<string, bool>> processed_rules;
for (ParsedRule const &rule : policy.getAppsecPolicySpec().getSpecificRules()) {
tuple<string, string, string> splited_host_name = splitHostName(rule.getHost());
string host = std::get<0>(splited_host_name);
if (host.empty() || rule.rpmGetUpstream().empty()) continue;
string location = std::get<2>(splited_host_name);
if (location.empty()) location = "/";
dbgTrace(D_LOCAL_POLICY)
<< "Building NGINX server: "
<< host
<< ", location: "
<< location
<< " RP-Settings: "
<< rule.rpmGetRPSettings();
RPMSettings rp_settings =
rpmGetAppsecRPSettingSpec<AppsecLinuxPolicy, RPMSettings>(rule.rpmGetRPSettings(), policy);
pair<string, bool> server = {host, rule.rpmIsHttps()};
auto it = processed_rules.find(server);
if (it != processed_rules.end()) {
auto maybe_res = ReverseProxyBuilder::addNginxServerLocation(location, host, rule, rp_settings);
if (!maybe_res.ok()) {
rpmReportError(
"Could not add an NGINX server location: " + location + " to server: " + host +
", error: " + maybe_res.getErr()
);
full_success = false;
continue;
}
rpmReportInfo("NGINX server location: " + location + " was successfully added to server: " + host);
partial_success = true;
} else {
auto maybe_res = ReverseProxyBuilder::createNewNginxServer(host, rule, rp_settings);
if (!maybe_res.ok()) {
rpmReportError("Could not create a new NGINX server: " + host + ", error: " + maybe_res.getErr());
full_success = false;
continue;
}
rpmReportInfo(
(rule.rpmIsHttps() ? string("SSL") : string("HTTP")) + " NGINX server: " + host +
" was successfully built"
);
processed_rules.insert(server);
maybe_res = ReverseProxyBuilder::addNginxServerLocation(location, host, rule, rp_settings);
if (!maybe_res.ok()) {
rpmReportError(
"Could not add an NGINX server location: " + location + " to server: " + host +
", error: " + maybe_res.getErr()
);
full_success = false;
continue;
}
rpmReportInfo("NGINX server location: " + location + " was successfully added to server: " + host);
partial_success = true;
}
}
auto maybe_reload_nginx = ReverseProxyBuilder::reloadNginx();
if (!maybe_reload_nginx.ok()) {
rpmReportError("Could not reload NGINX, error: " + maybe_reload_nginx.getErr());
return;
}
if (full_success) {
rpmReportInfo("NGINX configuration was loaded successfully!");
} else if (partial_success) {
rpmReportInfo("NGINX configuration was partially loaded");
} else {
rpmReportError("Could not load any NGINX configuration");
}
}

456
components/security_apps/local_policy_mgmt_gen/reverse_proxy_section.cc Executable file
View File

@ -0,0 +1,456 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "reverse_proxy_section.h"
#include <algorithm>
#include <openssl/pem.h>
#include <openssl/rsa.h>
#include <openssl/x509v3.h>
#include <fstream>
#include "local_policy_mgmt_gen.h"
#include "local_policy_common.h"
#include "appsec_practice_section.h"
#include "debug.h"
using namespace std;
USE_DEBUG_FLAG(D_LOCAL_POLICY);
static string conf_base_path = "/etc/cp/conf/";
static string certs_path = "/etc/certs/";
static string nginx_templates_path = "/etc/nginx/nginx-templates/";
static const string nginx_configuration_path = "openappsec-nginx-servers/";
static const string nginx_http_server_template = "nginx-http-server";
static const string nginx_ssl_server_template = "nginx-ssl-server";
static const string nginx_location_template = "nginx-location-block";
static const boost::regex host_template("<host>");
static const boost::regex private_key_template("<private-key>");
static const boost::regex certificate_template("<certificate>");
static const boost::regex location_template("<location>");
static const boost::regex upstream_template("<upstream>");
static const boost::regex host_header_template("<host-header>");
static const boost::regex dns_resolver_template("<dns-resolver>");
class ReverseProxyCertUtils
{
public:
static std::pair<std::string, std::string> findMatchingCertificate(const std::string &host);
static void init();
private:
static std::vector<std::string> getFilesByExtension(const std::string &extension);
static void untarCertificatesPackages();
static Maybe<std::string> extractModulus(const std::string &path, const std::string &type);
static std::unordered_map<std::string, std::string>
calculatePublicModulus(const std::vector<std::string> &certs);
static std::unordered_map<std::string, std::string>
calculatePrivateModulus(const std::vector<std::string> &keys);
static std::unordered_map<std::string, std::string> cert_key_map;
};
unordered_map<string, string> ReverseProxyCertUtils::cert_key_map;
void
RPMSettings::load(cereal::JSONInputArchive &archive_in)
{
dbgFlow(D_LOCAL_POLICY) << "Loading RP Settings";
parseAppsecJSONKey<string>("name", name, archive_in);
parseAppsecJSONKey<string>("host-header", host_hdr, archive_in, "$host");
parseAppsecJSONKey<string>("dns-resolver", dns_resolver, archive_in, "127.0.0.11");
}
const string &
RPMSettings::getName() const
{
return name;
}
string
RPMSettings::applySettings(const std::string &server_content) const
{
string new_server_content = ReverseProxyBuilder::replaceTemplate(server_content, host_header_template, host_hdr);
return ReverseProxyBuilder::replaceTemplate(new_server_content, dns_resolver_template, dns_resolver);
}
void
ReverseProxyCertUtils::init()
{
certs_path = getProfileAgentSettingWithDefault<string>("/etc/certs/", "openappsec.reverseProxy.certs");
untarCertificatesPackages();
cert_key_map.clear();
auto public_modulus_map = calculatePublicModulus(getFilesByExtension(".pem"));
auto private_modulus_map = calculatePrivateModulus(getFilesByExtension(".key"));
for (const auto &public_modulus_entry : public_modulus_map) {
auto public_modulus = public_modulus_entry.second;
if (private_modulus_map.find(public_modulus) != private_modulus_map.end()) {
dbgTrace(D_LOCAL_POLICY)
<< "Successfully parsed certificate: "
<< public_modulus_entry.first
<< " with private key: "
<< private_modulus_map[public_modulus];
cert_key_map[public_modulus_entry.first] = private_modulus_map[public_modulus];
}
}
}
vector<string>
ReverseProxyCertUtils::getFilesByExtension(const string &extension)
{
auto maybe_files = NGEN::Filesystem::getDirectoryFiles(certs_path);
if (!maybe_files.ok()) return {};
auto files = maybe_files.unpack();
files.erase(
remove_if(
files.begin(),
files.end(),
[&](const string& file) { return file.length() < 4 || file.substr(file.length() - 4) != extension; }
),
files.end()
);
for (const auto &file : files) {
dbgTrace(D_LOCAL_POLICY) << "Found file: " << file;
}
return files;
}
pair<string, string>
ReverseProxyCertUtils::findMatchingCertificate(const string &host)
{
dbgFlow(D_LOCAL_POLICY) << "Looking for a matching certificate to host: " << host;
for (const auto &entry : cert_key_map) {
string cert_path = entry.first;
dbgTrace(D_LOCAL_POLICY) << "Checking match of certificate: " << cert_path;
// Create a BIO object to read the certificate
BIO* cert_bio = BIO_new_file(cert_path.c_str(), "rb");
if (!cert_bio) {
dbgWarning(D_LOCAL_POLICY) << "Could not open certificate file: " << cert_path;
continue;
}
// Load the PEM-encoded public key from the file
X509 *cert = PEM_read_bio_X509(cert_bio, nullptr, nullptr, nullptr);
if (!cert) {
dbgWarning(D_LOCAL_POLICY) << "Could not parse X509 certificate file: " << cert_path;
BIO_free(cert_bio);
continue;
}
// Get the subject alternative name extension
STACK_OF(GENERAL_NAME)* san_names = static_cast<STACK_OF(GENERAL_NAME)*>(
X509_get_ext_d2i(cert, NID_subject_alt_name, nullptr, nullptr)
);
if (!san_names) {
dbgWarning(D_LOCAL_POLICY) << "No Subject Alternative Name found in the certificate: " << cert_path;
X509_free(cert);
BIO_free(cert_bio);
continue;
}
// Iterate through the SAN entries
for (int i = 0; i < sk_GENERAL_NAME_num(san_names); ++i) {
GENERAL_NAME* name = sk_GENERAL_NAME_value(san_names, i);
if (name->type == GEN_DNS) {
const char* san = reinterpret_cast<const char*>(ASN1_STRING_get0_data(name->d.dNSName));
if (X509_check_host(cert, host.c_str(), host.length(), 0, nullptr) == 1) {
dbgTrace(D_LOCAL_POLICY) << "Found matching certificate: " << cert_path << ", DNS name: " << san;
sk_GENERAL_NAME_pop_free(san_names, GENERAL_NAME_free);
X509_free(cert);
BIO_free(cert_bio);
return {cert_path, cert_key_map[cert_path]};
}
}
}
dbgTrace(D_LOCAL_POLICY) << "Certificate: " << cert_path << " does not match host: " << host;
// Clean up
sk_GENERAL_NAME_pop_free(san_names, GENERAL_NAME_free);
X509_free(cert);
BIO_free(cert_bio);
}
return {};
}
Maybe<std::string>
ReverseProxyCertUtils::extractModulus(const string &path, const string &type)
{
dbgFlow(D_LOCAL_POLICY) << "Started calculating modulus of: " << path << ", type: " << type;
string modulus_cmd = "openssl " + type + " -noout -modulus -in " + path + "; echo $?";
auto modulus_maybe = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(modulus_cmd);
if (!modulus_maybe.ok()) return genError("Could not complete command, error: " + modulus_maybe.getErr());
auto modulus_cmd_output = NGEN::Strings::removeTrailingWhitespaces(modulus_maybe.unpack());
if (modulus_cmd_output.back() != '0') return genError("Could not extract modulus, error: " + modulus_cmd_output);
modulus_cmd_output.pop_back();
dbgTrace(D_LOCAL_POLICY) << "Extracted modulus for: " << path << ", " << modulus_cmd_output;
return modulus_cmd_output;
}
unordered_map<string, string>
ReverseProxyCertUtils::calculatePublicModulus(const vector<string> &certs)
{
dbgFlow(D_LOCAL_POLICY) << "Calculating certificates modulus";
unordered_map<string, string> certs_modulus;
for (const string &cert_file_name : certs) {
string cert_path = certs_path + cert_file_name;
auto modulus = extractModulus(cert_path, "x509");
if (!modulus.ok()) {
dbgWarning(D_LOCAL_POLICY) << modulus.getErr();
continue;
}
certs_modulus[cert_path] = modulus.unpack();
}
return certs_modulus;
}
unordered_map<string, string>
ReverseProxyCertUtils::calculatePrivateModulus(const vector<string> &keys)
{
unordered_map<string, string> key_modulus;
for (const string &private_key_file_name : keys) {
string private_key_path = certs_path + private_key_file_name;
auto modulus = extractModulus(private_key_path, "rsa");
if (!modulus.ok()) {
dbgWarning(D_LOCAL_POLICY) << modulus.getErr();
continue;
}
key_modulus[modulus.unpack()] = private_key_path;
}
return key_modulus;
}
void
ReverseProxyCertUtils::untarCertificatesPackages()
{
vector<string> cert_pkgs = getFilesByExtension(".pkg");
if (cert_pkgs.empty()) return;
for (const auto &cert_pkg : cert_pkgs) {
dbgTrace(D_LOCAL_POLICY) << "Untaring certificate package: " << cert_pkg;
string untar_cmd = "tar -C " + certs_path + " -xvf " + certs_path + cert_pkg;
auto maybe_tar_res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(untar_cmd);
if (!maybe_tar_res.ok()) {
dbgWarning(D_LOCAL_POLICY) << "Untar package error: " << maybe_tar_res.getErr();
}
}
}
string
ReverseProxyBuilder::replaceTemplate(
const string &content,
const boost::regex &nginx_directive_template,
const string &value)
{
return NGEN::Regex::regexReplace(__FILE__, __LINE__, content, nginx_directive_template, value);
}
Maybe<string>
ReverseProxyBuilder::getTemplateContent(const string &nginx_conf_template)
{
ifstream nginx_template_in(nginx_templates_path + nginx_conf_template);
if (!nginx_template_in.is_open()) return genError("Could not open the " + nginx_conf_template + " template");
string file_content((istreambuf_iterator<char>(nginx_template_in)), istreambuf_iterator<char>());
nginx_template_in.close();
return file_content;
}
Maybe<void>
ReverseProxyBuilder::createSSLNginxServer(const string &host, const RPMSettings &rp_settings)
{
dbgTrace(D_LOCAL_POLICY) << "Creating SSL NGINX server: " << host;
pair<string, string> cert_key = ReverseProxyCertUtils::findMatchingCertificate(host);
if (cert_key.first.empty() || cert_key.second.empty()) {
return genError("Cannot find matching certificates to host: " + host);
}
auto maybe_server_content = getTemplateContent(nginx_ssl_server_template);
if (!maybe_server_content.ok()) return maybe_server_content.passErr();
string server_content = replaceTemplate(maybe_server_content.unpack(), host_template, host);
server_content = replaceTemplate(server_content, private_key_template, cert_key.second);
server_content = replaceTemplate(server_content, certificate_template, cert_key.first);
server_content = rp_settings.applySettings(server_content);
dbgTrace(D_LOCAL_POLICY) << "NGINX SSL Server content: " << server_content;
string conf_path = conf_base_path + nginx_configuration_path + "/443_" + host + ".conf";
ofstream server_file(conf_path, ofstream::out | ofstream::trunc);
if (!server_file.is_open()) {
return genError("Could not open the output SSL NGINX configuration file: " + conf_path);
}
server_file << server_content;
server_file.close();
return {};
}
Maybe<void>
ReverseProxyBuilder::createHTTPNginxServer(const string &host, const RPMSettings &rp_settings)
{
dbgFlow(D_LOCAL_POLICY) << "Creating HTTP NGINX server: " << host;
auto maybe_server_content = getTemplateContent(nginx_http_server_template);
if (!maybe_server_content.ok()) return maybe_server_content.passErr();
string server_content = replaceTemplate(maybe_server_content.unpack(), host_template, host);
server_content = rp_settings.applySettings(server_content);
dbgTrace(D_LOCAL_POLICY) << "NGINX HTTP Server content: " << server_content;
string http_server_conf_path = conf_base_path + nginx_configuration_path + "80_" + host + ".conf";
ofstream server_file(http_server_conf_path, ofstream::out | ofstream::trunc);
if (!server_file.is_open()) {
return genError("Could not open the output HTTP NGINX configuration file: " + http_server_conf_path);
}
server_file << server_content;
server_file.close();
return {};
}
Maybe<void>
ReverseProxyBuilder::addNginxServerLocation(
string location,
const string &host,
const ParsedRule &rule,
const RPMSettings &rp_settings)
{
string port = rule.rpmIsHttps() ? string("443") : string("80");
string location_conf_path = conf_base_path + nginx_configuration_path + port + '_' + host + "_locations/";
dbgFlow(D_LOCAL_POLICY) << "Adding a new NGINX location: " << location << " to: " << location_conf_path;
NGEN::Filesystem::makeDirRecursive(location_conf_path);
if (location.empty() || location.find_first_not_of('/') == string::npos)
{
location = "/";
location_conf_path += "root_location.conf";
}
else
{
string location_conf_basename = location.substr(1, location.length() - 1) + "_location";
replace(location_conf_basename.begin(), location_conf_basename.end(), '/', '_');
location_conf_path += location_conf_basename + ".conf";
}
auto maybe_location_content = getTemplateContent(nginx_location_template);
if (!maybe_location_content.ok()) return maybe_location_content.passErr();
string location_content = replaceTemplate(maybe_location_content.unpack(), location_template, location);
location_content = replaceTemplate(location_content, upstream_template, rule.rpmGetUpstream());
location_content = rp_settings.applySettings(location_content);
dbgTrace(D_LOCAL_POLICY) << "NGINX server location content: " << location_content;
ofstream location_file(location_conf_path, ofstream::out | ofstream::trunc);
if (!location_file.is_open()) {
return genError("Could not open the output NGINX location block: " + location_conf_path);
}
location_file << location_content;
location_file.close();
return {};
}
Maybe<void>
ReverseProxyBuilder::createNewNginxServer(const string &host, const ParsedRule &rule, const RPMSettings &rp_settings)
{
dbgFlow(D_LOCAL_POLICY) << "Creating a new NGINX server: " << host << ", SSL: " << rule.rpmIsHttps();
if (rule.rpmIsHttps()) {
auto maybe_res = ReverseProxyBuilder::createSSLNginxServer(host, rp_settings);
if (!maybe_res.ok()) {
return genError("Could not create an SSL NGINX server configuration: " + maybe_res.getErr());
}
} else {
auto maybe_res = ReverseProxyBuilder::createHTTPNginxServer(host, rp_settings);
if (!maybe_res.ok()) {
return genError("Could not create an HTTP NGINX server: " + maybe_res.getErr());
}
}
return {};
}
Maybe<void>
ReverseProxyBuilder::reloadNginx()
{
dbgFlow(D_LOCAL_POLICY) << "Reloading NGINX...";
auto maybe_nginx_t = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(
"nginx -t 2>&1; echo $?"
);
if (!maybe_nginx_t.ok()){
return genError("Could not check NGINX configuration: " + maybe_nginx_t.getErr());
}
string nginx_t_output = NGEN::Strings::removeTrailingWhitespaces(maybe_nginx_t.unpack());
if (nginx_t_output.back() != '0') return genError("Invalid NGINX configuration: " + nginx_t_output);
auto maybe_nginx_reload = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(
"nginx -s reload 2>&1;"
);
if (!maybe_nginx_reload.ok()){
return genError("Could not reload NGINX: " + maybe_nginx_reload.getErr());
}
return {};
}
void
ReverseProxyBuilder::init()
{
conf_base_path = getConfigurationWithDefault<string>("/etc/cp/conf/", "Config Component", "configuration path");
nginx_templates_path = getProfileAgentSettingWithDefault<string>(
"/etc/nginx/nginx-templates/", "openappsec.reverseProxy.nginxTemplates"
);
NGEN::Filesystem::deleteDirectory(conf_base_path + nginx_configuration_path, true);
NGEN::Filesystem::makeDir(conf_base_path + nginx_configuration_path);
ReverseProxyCertUtils::init();
}

46
components/security_apps/local_policy_mgmt_gen/rules_config_section.cc
View File

@ -156,6 +156,7 @@ RulesTriggerSection::save(cereal::JSONOutputArchive &out_ar) const
RulesConfigRulebase::RulesConfigRulebase(
const string &_name,
const string &_url,
const string &_port,
const string &_uri,
vector<PracticeSection> _practices,
vector<ParametersSection> _parameters,
@ -169,39 +170,19 @@ RulesConfigRulebase::RulesConfigRulebase(
try {
bool any = _name == "Any" && _url == "Any" && _uri == "Any";
id = any ? "Any" : _url+_uri;
if (_uri != "/") {
context = any ? "All()" : "Any("
"All("
"Any("
"EqualHost(" + _url + ")"
"),"
"EqualListeningPort(80)" +
string(_uri.empty() ? "" : ",BeginWithUri(" + _uri + ")") +
"),"
"All("
"Any("
"EqualHost(" + _url + ")"
"),"
"EqualListeningPort(443)" +
string(_uri.empty() ? "" : ",BeginWithUri(" + _uri + ")") +
")"
")";
} else {
context = any ? "All()" : "Any("
"All("
"Any("
"EqualHost(" + _url + ")"
"),"
"EqualListeningPort(80)"
"),"
"All("
"Any("
"EqualHost(" + _url + ")"
"),"
"EqualListeningPort(443)"
")"
")";
if (any) {
context ="All()";
return;
}
string host_check = "Any(EqualHost(" + _url + ")),";
string uri_check = (_uri.empty() || _uri == "/" ) ? "" : ",BeginWithUri(" + _uri + ")";
auto ports = _port.empty() ? vector<string>({"80", "443"}) : vector<string>({_port});
context = "Any(";
for (auto &port : ports) {
string check_last = (ports.size() == 1 || port == "443") ? ")" : "),";
context += "All(" + host_check + "EqualListeningPort(" + port + ")" + uri_check + check_last;
}
context += ")";
} catch (const boost::uuids::entropy_error &e) {
dbgWarning(D_LOCAL_POLICY) << "Failed to generate rule UUID. Error: " << e.what();
}
@ -284,6 +265,7 @@ UsersIdentifiersRulebase::UsersIdentifiersRulebase(
const string &
UsersIdentifiersRulebase::getIdentifier() const
{
if (source_identifiers.empty()) return source_identifier;
return source_identifiers[0].getIdentifier();
}
// LCOV_EXCL_STOP

28
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -45,11 +45,7 @@ checkSamlPortal(const string &command_output)
Maybe<string>
getIDAGaia(const string &command_output)
{
if (command_output.find("Portal is running") != string::npos) {
return string("ida_gaia");
}
return genError("Current host does not have SAML Portal configured");
return string("ida_gaia");
}
Maybe<string>
@ -72,6 +68,22 @@ checkIDP(shared_ptr<istream> file_stream)
#if defined(gaia) || defined(smb)
Maybe<string>
checkIsCpviewRunning(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
return genError("cpview is not running");
}
Maybe<string>
checkIsCPotelcolGRET64(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
return genError("CPotelcol is not installed or its take is below T64");
}
Maybe<string>
checkHasSDWan(const string &command_output)
{
@ -193,6 +205,12 @@ checkIfSdwanRunning(const string &command_output)
return genError("Could not determine if sd-wan is running or not");
}
Maybe<string>
getClusterObjectIP(const string &command_output)
{
return getAttr(command_output, "Cluster object IP was not found");
}
Maybe<string>
getSmbObjectName(const string &command_output)
{

18
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -30,6 +30,14 @@
#ifdef SHELL_CMD_HANDLER
#if defined(gaia) || defined(smb)
SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
SHELL_CMD_HANDLER("isCpviewRunning",
"pidof cpview_api_service > /dev/null 2>&1 && [ -f $CPDIR/conf/cpview_api_service.version ] "
"&& echo 'true' || echo 'false'",
checkIsCpviewRunning)
SHELL_CMD_HANDLER("isCPotelcolGRET64",
"grep -A 10 '(BUNDLE_CPOTELCOL_AUTOUPDATE' ${CPDIR}/registry/HKLM_registry.data | "
"awk '/SU_Build_Take/{val = substr($2, 2, length($2)-2); if (val >=64) print \"true\"; else print \"false\" }'",
checkIsCPotelcolGRET64)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
@ -50,12 +58,20 @@ SHELL_CMD_HANDLER(
"cat /etc/cp-release | grep -oE 'R[0-9]+(\\.[0-9]+)?'",
getGWVersion
)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectIP",
"obj=\"$(cpsdwan get_data | jq -r .cluster_name)\";"
" awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")"
" { gsub(/[()]/, \"\", $2); print $2; exit; } }'"
" $FWDIR/state/local/FW1/local.gateway_cluster",
getClusterObjectIP
)
#endif //gaia || smb
#if defined(gaia)
SHELL_CMD_HANDLER("hasSupportedBlade", "enabled_blades", checkHasSupportedBlade)
SHELL_CMD_HANDLER("hasSamlPortal", "mpclient status saml-vpn", checkSamlPortal)
SHELL_CMD_HANDLER("requiredNanoServices", "mpclient status saml-vpn", getIDAGaia)
SHELL_CMD_HANDLER("requiredNanoServices", "ida_gaia", getIDAGaia)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectName",
"cat $FWDIR/database/myself_objects.C "

32
components/security_apps/orchestration/downloader/curl_client.cc
View File

@ -278,36 +278,6 @@ HttpsCurl::HttpsCurl(const HttpsCurl &other) :
HttpCurl(other),
ca_path(other.ca_path) {}
bool
HttpsCurl::downloadOpenAppsecPackages()
{
char errorstr[CURL_ERROR_SIZE];
CURL* curl_handle = curl_easy_init();
if (!curl_handle) return false;
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYPEER, 1);
curl_easy_setopt(curl_handle, CURLOPT_SSL_VERIFYHOST, 2);
curl_easy_setopt(curl_handle, CURLOPT_URL, ("https://" + curl_url).c_str());
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeResponseCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &out_file);
curl_easy_setopt(curl_handle, CURLOPT_VERBOSE, 1L);
curl_easy_setopt(curl_handle, CURLOPT_ERRORBUFFER, errorstr);
CURLcode res = curl_easy_perform(curl_handle);
if (res == CURLE_OK) {
dbgTrace(D_HTTP_REQUEST) << "CURL HTTP request successfully completed.";
} else {
dbgWarning(D_HTTP_REQUEST) << "CURL result " + string(curl_easy_strerror(res));
curl_easy_cleanup(curl_handle);
return false;
}
curl_easy_cleanup(curl_handle);
return true;
}
void
HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version)
{
@ -347,7 +317,7 @@ HttpsCurl::setCurlOpts(long timeout, HTTP_VERSION http_version)
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, writeResponseCallback);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, &out_file);
curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, timeout);
curl_easy_setopt(curl_handle, CURLOPT_CAINFO, ca_path.c_str());
if (ca_path != "") curl_easy_setopt(curl_handle, CURLOPT_CAINFO, ca_path.c_str());
headers = curl_slist_append(headers, "Accept: */*");
string auth = string("Authorization: Bearer ") + bearer;
headers = curl_slist_append(headers, auth.c_str());

1
components/security_apps/orchestration/downloader/curl_client.h
View File

@ -105,7 +105,6 @@ public:
static CURLcode ssl_ctx_verify_certificate(CURL *curl, void *ssl_ctx, void *opq);
static int verify_certificate(int preverify_ok, X509_STORE_CTX *ctx);
void setCurlOpts(long timeout = 60L, HTTP_VERSION http_version = HTTP_VERSION::HTTP_VERSION_1_1) override;
bool downloadOpenAppsecPackages();
private:
std::string ca_path;

9
components/security_apps/orchestration/downloader/https_client.cc
View File

@ -592,13 +592,8 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
proxy_config->getProxyCredentials(ProxyProtocol::HTTPS),
cert_file_path);
bool connection_ok;
if (url.getBaseURL().unpack() == "downloads.openappsec.io") {
connection_ok = ssl_curl_client.downloadOpenAppsecPackages();
} else {
ssl_curl_client.setCurlOpts();
connection_ok = ssl_curl_client.connect();
}
ssl_curl_client.setCurlOpts();
bool connection_ok = ssl_curl_client.connect();
if (!connection_ok)
{
stringstream url_s;

21
components/security_apps/orchestration/health_check/health_check.cc
View File

@ -256,10 +256,23 @@ private:
if (!getenv("DOCKER_RPM_ENABLED")) return HealthCheckStatus::IGNORED;
static const string standalone_cmd = "/usr/sbin/cpnano -s --docker-rpm; echo $?";
static int timeout_tolerance = 1;
static HealthCheckStatus health_status = HealthCheckStatus::HEALTHY;
dbgTrace(D_HEALTH_CHECK) << "Checking the standalone docker health status with command: " << standalone_cmd;
auto maybe_result = Singleton::Consume<I_ShellCmd>::by<HealthChecker>()->getExecOutput(standalone_cmd, 1000);
auto maybe_result = Singleton::Consume<I_ShellCmd>::by<HealthChecker>()->getExecOutput(standalone_cmd, 5000);
if (!maybe_result.ok()) {
if (maybe_result.getErr().find("Reached timeout") != string::npos) {
dbgWarning(D_HEALTH_CHECK)
<< "Reached timeout while querying standalone health status, attempt number: "
<< timeout_tolerance;
return health_status == HealthCheckStatus::UNHEALTHY || timeout_tolerance++ > 3 ?
HealthCheckStatus::UNHEALTHY :
health_status;
}
dbgWarning(D_HEALTH_CHECK) << "Unable to get the standalone docker status. Returning unhealthy status.";
return HealthCheckStatus::UNHEALTHY;
}
@ -267,10 +280,10 @@ private:
auto response = NGEN::Strings::removeTrailingWhitespaces(maybe_result.unpack());
if (response.back() == '0') return HealthCheckStatus::HEALTHY;
if (response.back() == '1') return HealthCheckStatus::UNHEALTHY;
if (response.back() == '1') return health_status = HealthCheckStatus::UNHEALTHY;
return HealthCheckStatus::DEGRADED;
timeout_tolerance = 1;
return health_status = (response.back() == '0') ? HealthCheckStatus::HEALTHY : HealthCheckStatus::DEGRADED;
}
bool

2
components/security_apps/orchestration/include/declarative_policy_utils.h
View File

@ -87,10 +87,12 @@ public:
private:
std::string getCleanChecksum(const std::string &unclean_checksum);
void periodicPolicyLoad();
std::string local_policy_path;
std::string curr_version;
std::string curr_policy;
std::string curr_checksum;
bool should_apply_policy;
};

2
components/security_apps/orchestration/include/get_status_rest.h
View File

@ -54,7 +54,6 @@ public:
last_update = i_orch_status->getUpdateTime();
last_update_status = i_orch_status->getUpdateStatus();
policy_version = i_orch_status->getPolicyVersion();
waap_model_version = i_orch_status->getWaapModelVersion();
last_policy_update = i_orch_status->getLastPolicyUpdate();
last_manifest_update = i_orch_status->getLastManifestUpdate();
last_settings_update = i_orch_status->getLastSettingsUpdate();
@ -73,7 +72,6 @@ private:
S2C_LABEL_PARAM(std::string, last_update, "Last update");
S2C_LABEL_PARAM(std::string, last_update_status, "Last update status");
S2C_LABEL_PARAM(std::string, policy_version, "Policy version");
S2C_LABEL_PARAM(std::string, waap_model_version, "AI model version");
S2C_LABEL_PARAM(std::string, last_policy_update, "Last policy update");
S2C_LABEL_PARAM(std::string, last_manifest_update, "Last manifest update");
S2C_LABEL_PARAM(std::string, last_settings_update, "Last settings update");

1
components/security_apps/orchestration/include/mock/mock_orchestration_status.h
View File

@ -45,7 +45,6 @@ public:
MOCK_CONST_METHOD0(getUpdateTime, const std::string&());
MOCK_CONST_METHOD0(getLastManifestUpdate, const std::string&());
MOCK_CONST_METHOD0(getPolicyVersion, const std::string&());
MOCK_CONST_METHOD0(getWaapModelVersion, const std::string&());
MOCK_CONST_METHOD0(getLastPolicyUpdate, const std::string&());
MOCK_CONST_METHOD0(getLastSettingsUpdate, const std::string&());
MOCK_CONST_METHOD0(getUpgradeMode, const std::string&());

2
components/security_apps/orchestration/include/mock/mock_service_controller.h
View File

@ -66,7 +66,7 @@ public:
typedef std::map<std::string, PortNumber> ServicePortMap;
MOCK_METHOD0(getServiceToPortMap, ServicePortMap());
MOCK_METHOD2(updateReconfStatus, void(int id, ReconfStatus status));
MOCK_METHOD3(updateReconfStatus, void(int id, const std::string &service_name, ReconfStatus status));
MOCK_METHOD4(
startReconfStatus,
void(int id, ReconfStatus status, const std::string &serivce_name, const std::string &service_id)

20
components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc
View File

@ -11,7 +11,6 @@
#include "mock/mock_time_get.h"
#include "mock/mock_orchestration_tools.h"
#include "mock/mock_agent_details.h"
#include "mock/mock_details_resolver.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_rest_api.h"
@ -39,17 +38,9 @@ public:
.WillOnce(DoAll(SaveArg<2>(&routine), Return(1))
);
EXPECT_CALL(mock_tools, readFile(file_path)).WillOnce(Return(start_file_content));
prepareResolvedDetails();
orchestration_status.init();
}
void
prepareResolvedDetails()
{
map<string, string> resolved_details({{"AppSecModelVersion", waap_model}});
EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_details));
}
string
orchestrationStatusFileToString()
{
@ -91,8 +82,7 @@ public:
const string &registration_details_architecture = "",
const string &agent_id = "None",
const string &profile_id = "None",
const string &tenant_id = "None",
const string &waap_model_version = "Advanced model"
const string &tenant_id = "None"
)
{
return "{\n"
@ -101,7 +91,6 @@ public:
" \"Last update\": \"" + last_update + "\",\n"
" \"Last manifest update\": \"" + last_manifest_update + "\",\n"
" \"Policy version\": \"" + policy_version + "\",\n"
" \"AI model version\": \"" + waap_model_version + "\",\n"
" \"Last policy update\": \"" + last_policy_update + "\",\n"
" \"Last settings update\": \"" + last_settings_update + "\",\n"
" \"Upgrade mode\": \"" + upgrade_mode + "\",\n"
@ -129,14 +118,12 @@ public:
ostringstream capture_debug;
StrictMock<MockOrchestrationTools> mock_tools;
StrictMock<MockAgentDetails> mock_agent_details;
StrictMock<MockDetailsResolver> mock_details_resolver;
OrchestrationStatus orchestration_status;
I_OrchestrationStatus * i_orchestration_status =
Singleton::Consume<I_OrchestrationStatus>::from(orchestration_status);
string file_path;
Maybe<string> start_file_content = genError("No file");
I_MainLoop::Routine routine;
string waap_model = "Advanced model";
};
TEST_F(OrchestrationStatusTest, doNothing)
@ -160,7 +147,6 @@ TEST_F(OrchestrationStatusTest, recoverFields)
TEST_F(OrchestrationStatusTest, loadFromFile)
{
prepareResolvedDetails();
Maybe<string> status = genError("No file");;
CPTestTempfile status_file;
file_path = status_file.fname;
@ -228,14 +214,12 @@ TEST_F(OrchestrationStatusTest, recoveryFields)
const string agent_id = "AgentId";
const string profile_id = "ProfileId";
const string tenant_id = "TenantId";
auto fog_addr = Maybe<string>(string("FogDomain"));
EXPECT_CALL(mock_agent_details, getAgentId()).WillOnce(Return(agent_id));
EXPECT_CALL(mock_agent_details, getProfileId()).WillOnce(Return(profile_id));
EXPECT_CALL(mock_agent_details, getTenantId()).WillOnce(Return(tenant_id));
EXPECT_CALL(mock_agent_details, getFogDomain()).WillOnce(Return(fog_addr));
i_orchestration_status->writeStatusToFile();
EXPECT_THAT(capture_debug.str(), HasSubstr("Repairing status fields"));
@ -243,7 +227,6 @@ TEST_F(OrchestrationStatusTest, recoveryFields)
EXPECT_EQ(i_orchestration_status->getProfileId(), profile_id);
EXPECT_EQ(i_orchestration_status->getTenantId(), tenant_id);
EXPECT_EQ(i_orchestration_status->getFogAddress(), fog_addr.unpack());
EXPECT_EQ(i_orchestration_status->getWaapModelVersion(), waap_model);
}
TEST_F(OrchestrationStatusTest, updateAllLastUpdatesTypes)
@ -436,7 +419,6 @@ TEST_F(OrchestrationStatusTest, setAllFields)
" \"Last update\": \"current time\",\n"
" \"Last manifest update\": \"current time\",\n"
" \"Policy version\": \"12\",\n"
" \"AI model version\": \"Advanced model\",\n"
" \"Last policy update\": \"current time\",\n"
" \"Last settings update\": \"current time\",\n"
" \"Upgrade mode\": \"Test Mode\",\n"

19
components/security_apps/orchestration/modules/orchestration_status.cc
View File

@ -108,7 +108,6 @@ public:
last_update_attempt = from.last_update_attempt;
last_manifest_update = from.last_manifest_update;
policy_version = from.policy_version;
waap_model_version = from.waap_model_version;
last_policy_update = from.last_policy_update;
last_settings_update = from.last_settings_update;
upgrade_mode = from.upgrade_mode;
@ -129,7 +128,6 @@ public:
const string & getUpdateTime() const { return last_update_time; }
const string & getLastManifestUpdate() const { return last_manifest_update; }
const string & getPolicyVersion() const { return policy_version; }
const string & getWaapModelVersion() const { return waap_model_version; }
const string & getLastPolicyUpdate() const { return last_policy_update; }
const string & getLastSettingsUpdate() const { return last_settings_update; }
const string & getUpgradeMode() const { return upgrade_mode; }
@ -144,16 +142,6 @@ public:
const map<string, string> & getServicePolicies() const { return service_policies; }
const map<string, string> & getServiceSettings() const { return service_settings; }
void updateWaapModelVersion() {
map<string, string> details_resolver =
Singleton::Consume<I_DetailsResolver>::by<OrchestrationStatus>()->getResolvedDetails();
if (details_resolver.find("AppSecModelVersion") != details_resolver.end()) {
waap_model_version = details_resolver["AppSecModelVersion"];
} else {
waap_model_version = "None";
}
}
void
insertServicePolicy(const string &key, const string &value)
{
@ -279,7 +267,6 @@ public:
last_manifest_update = "None";
last_policy_update = "None";
last_settings_update = "None";
waap_model_version = "None";
fog_address = "None";
agent_id = "None";
profile_id = "None";
@ -305,7 +292,6 @@ public:
} else {
fog_address = "None";
}
updateWaapModelVersion();
}
}
@ -318,7 +304,6 @@ public:
archive(cereal::make_nvp("Last update", last_update_time));
archive(cereal::make_nvp("Last manifest update", last_manifest_update));
archive(cereal::make_nvp("Policy version", policy_version));
archive(cereal::make_nvp("AI model version", waap_model_version));
archive(cereal::make_nvp("Last policy update", last_policy_update));
archive(cereal::make_nvp("Last settings update", last_settings_update));
archive(cereal::make_nvp("Upgrade mode", upgrade_mode));
@ -346,7 +331,6 @@ public:
archive.setNextName(nullptr);
}
archive(cereal::make_nvp("AI model version", waap_model_version));
archive(cereal::make_nvp("Last policy update", last_policy_update));
archive(cereal::make_nvp("Last settings update", last_settings_update));
@ -384,7 +368,6 @@ private:
string last_update_attempt;
string last_manifest_update;
string policy_version;
string waap_model_version;
string last_policy_update;
string last_settings_update;
string upgrade_mode;
@ -411,7 +394,6 @@ public:
"orchestration",
"Orchestration status path"
);
status.updateWaapModelVersion();
auto write_result =
orchestration_tools->objectToJsonFile<Status>(status, orchestration_status_path);
if (!write_result) {
@ -515,7 +497,6 @@ private:
const string & getUpdateTime() const override { return status.getUpdateTime(); }
const string & getLastManifestUpdate() const override { return status.getLastManifestUpdate(); }
const string & getPolicyVersion() const override { return status.getPolicyVersion(); }
const string & getWaapModelVersion() const override { return status.getWaapModelVersion(); }
const string & getLastPolicyUpdate() const override { return status.getLastPolicyUpdate(); }
const string & getLastSettingsUpdate() const override { return status.getLastSettingsUpdate(); }
const string & getUpgradeMode() const override { return status.getUpgradeMode(); }

2
components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc
View File

@ -1797,7 +1797,6 @@ TEST_F(OrchestrationTest, GetRestOrchStatus)
" \"Last update\": \"" + test_str + "\",\n"
" \"Last update status\": \"" + test_str + "\",\n"
" \"Policy version\": \"" + test_str + "\",\n"
" \"AI model version\": \"" + test_str + "\",\n"
" \"Last policy update\": \"" + test_str + "\",\n"
" \"Last manifest update\": \"" + test_str + "\",\n"
" \"Last settings update\": \"" + test_str + "\",\n"
@ -1824,7 +1823,6 @@ TEST_F(OrchestrationTest, GetRestOrchStatus)
EXPECT_CALL(mock_status, getUpdateTime()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getLastManifestUpdate()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getPolicyVersion()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getWaapModelVersion()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getLastPolicyUpdate()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getLastSettingsUpdate()).WillOnce(ReturnRef(test_str));
EXPECT_CALL(mock_status, getUpgradeMode()).WillOnce(ReturnRef(test_str));

36
components/security_apps/orchestration/service_controller/service_controller.cc
View File

@ -53,32 +53,37 @@ public:
{
auto service_controller = Singleton::Consume<I_ServiceController>::by<ServiceReconfStatusMonitor>();
if (!finished.get()) {
service_controller->updateReconfStatus(id.get(), ReconfStatus::IN_PROGRESS);
service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::IN_PROGRESS);
dbgTrace(D_ORCHESTRATOR)
<< "Request for service reconfiguration, with id "
<< "Request for service reconfiguration is still in progress. ID: "
<< id.get()
<< ", is still in progress.";
<< ", Service Name: "
<< service_name.get();
return;
}
if (error.get()) {
service_controller->updateReconfStatus(id.get(), ReconfStatus::FAILED);
service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::FAILED);
dbgError(D_ORCHESTRATOR)
<< "Request for service reconfiguration, with id "
<< "Request for service reconfiguration failed to complete. ID: "
<< id.get()
<< ", failed to complete."
<< ", Service Name: "
<< service_name.get()
<< "."
<< (error_message.isActive() ? " Error: " + error_message.get() : "");
return;
}
service_controller->updateReconfStatus(id.get(), ReconfStatus::SUCCEEDED);
service_controller->updateReconfStatus(id.get(), service_name.get(), ReconfStatus::SUCCEEDED);
dbgInfo(D_ORCHESTRATOR)
<< "Request for service reconfiguration, with id "
<< "Request for service reconfiguration successfully accomplished. Reconf ID: "
<< id.get()
<< ", successfully accomplished.";
<< ", Service Name: "
<< service_name.get();
return;
}
private:
C2S_PARAM(int, id);
C2S_PARAM(string, service_name);
C2S_PARAM(bool, error);
C2S_PARAM(bool, finished);
C2S_OPTIONAL_PARAM(string, error_message);
@ -292,7 +297,7 @@ public:
const string & getPolicyVersion() const override;
const string & getUpdatePolicyVersion() const override;
const string & getPolicyVersions() const override;
void updateReconfStatus(int id, ReconfStatus status) override;
void updateReconfStatus(int id, const string &service_name, ReconfStatus status) override;
void startReconfStatus(
int id,
ReconfStatus status,
@ -780,6 +785,7 @@ ServiceController::Impl::updateServiceConfiguration(
string version_value;
string send_signal_for_services_err;
changed_policy_files.clear();
for (auto &single_policy : all_security_policies.unpack()) {
if (single_policy.first == version_param) {
version_value = single_policy.second;
@ -1076,19 +1082,25 @@ ServiceController::Impl::getUpdatePolicyVersion() const
}
void
ServiceController::Impl::updateReconfStatus(int id, ReconfStatus status)
ServiceController::Impl::updateReconfStatus(int id, const string &service_name, ReconfStatus status)
{
if (status == ReconfStatus::FAILED) {
failed_services.emplace(id, status);
}
if (services_reconf_status.find(id) == services_reconf_status.end()) {
dbgError(D_ORCHESTRATOR) << "Service reconfiguration monitor received illegal id :" << id;
dbgError(D_ORCHESTRATOR)
<< "Unable to find a mapping for reconfiguration ID:"
<< id
<< ". Service name: "
<< service_name;
return;
}
dbgTrace(D_ORCHESTRATOR)
<< "Updating reconf status for reconfiguration ID "
<< id
<< ", Service name: "
<< service_name
<< ". Status: "
<< static_cast<int>(status);
services_reconf_status[id] = status;

1
components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc
View File

@ -1928,6 +1928,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
reconf_status
<< "{"
<< " \"id\": 1,"
<< " \"service_name\": \"max\","
<< " \"finished\": true,"
<< " \"error\": false,"
<< " \"error_message\": \"\""

26
components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
View File

@ -22,6 +22,16 @@ DeclarativePolicyUtils::init()
RestAction::SET, "apply-policy"
);
registerListener();
char *automatic_load = getenv("autoPolicyLoad");
if (automatic_load != nullptr && automatic_load == string("true")) {
auto mainloop = Singleton::Consume<I_MainLoop>::by<DeclarativePolicyUtils>();
mainloop->addRecurringRoutine(
I_MainLoop::RoutineType::Offline,
chrono::minutes(1),
[&] () { periodicPolicyLoad(); },
"Automatic Policy Loading"
);
}
}
// LCOV_EXCL_START Reason: no test exist
@ -170,3 +180,19 @@ DeclarativePolicyUtils::getUpdate(CheckUpdateRequest &request)
curr_version = maybe_new_version.unpack();
return policy_response;
}
void
DeclarativePolicyUtils::periodicPolicyLoad()
{
auto new_checksum = getLocalPolicyChecksum();
if (!new_checksum.ok()) {
dbgWarning(D_ORCHESTRATOR) << "Failed to calculate checksum";
return;
}
if (*new_checksum == curr_checksum) return;
should_apply_policy = true;
curr_checksum = *new_checksum;
}

107
components/security_apps/rate_limit/rate_limit.cc
View File

@ -13,6 +13,7 @@
#include "http_inspection_events.h"
#include "Waf2Util.h"
#include "generic_rulebase/evaluators/asset_eval.h"
#include "generic_rulebase/parameters_config.h"
#include "WaapConfigApi.h"
#include "WaapConfigApplication.h"
#include "PatternMatcher.h"
@ -49,13 +50,13 @@ public:
Maybe<string>
extractUri(const string &address)
{
size_t protocolPos = address.find("://");
if (protocolPos == std::string::npos) return genError("Invalid URI format: " + address);
size_t protocol_pos = address.find("://");
if (protocol_pos == string::npos) return genError("Invalid URI format: " + address);
size_t domainPos = address.find('/', protocolPos + 3);
if (domainPos == std::string::npos) return string("");
size_t domain_pos = address.find('/', protocol_pos + 3);
if (domain_pos == string::npos) return string("");
return address.substr(domainPos);
return address.substr(domain_pos);
}
bool
@ -179,15 +180,35 @@ public:
{
if (!event.isLastHeader()) return INSPECT;
auto uri_ctx = Singleton::Consume<I_Environment>::by<RateLimit>()->get<string>(HttpTransactionData::uri_ctx);
auto env = Singleton::Consume<I_Environment>::by<RateLimit>();
auto uri_ctx = env->get<string>(HttpTransactionData::uri_ctx);
if (!uri_ctx.ok()) {
dbgWarning(D_RATE_LIMIT) << "Unable to get URL from context, Not enforcing rate limit";
return ACCEPT;
}
string asset_id;
auto uri = uri_ctx.unpack();
transform(uri.begin(), uri.end(), uri.begin(), [](unsigned char c) { return tolower(c); });
auto maybe_source_identifier = env->get<string>(HttpTransactionData::source_identifier);
if (!maybe_source_identifier.ok()) {
dbgWarning(D_RATE_LIMIT) << "Unable to get source identifier from context, not enforcing rate limit";
return ACCEPT;
}
auto &source_identifier = maybe_source_identifier.unpack();
dbgDebug(D_RATE_LIMIT) << "source identifier value: " << source_identifier;
auto maybe_source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
string source_ip = "";
if (maybe_source_ip.ok()) source_ip = ipAddrToStr(maybe_source_ip.unpack());
if (shouldApplyException(uri, source_identifier, source_ip)) {
dbgDebug(D_RATE_LIMIT) << "found accept exception, not enforcing rate limit on this uri: " << uri;
return ACCEPT;
}
string asset_id;
auto maybe_rule = findRateLimitRule(uri, asset_id);
if (!maybe_rule.ok()) {
dbgDebug(D_RATE_LIMIT) << "Not Enforcing Rate Limit: " << maybe_rule.getErr();
@ -205,16 +226,6 @@ public:
<< (rule.getRateLimitScope() == "Minute" ? 60 : 1)
<< " seconds";
auto maybe_source_identifier =
Singleton::Consume<I_Environment>::by<RateLimit>()->get<string>(HttpTransactionData::source_identifier);
if (!maybe_source_identifier.ok()) {
dbgWarning(D_RATE_LIMIT) << "Unable to get source identifier from context, not enforcing rate limit";
return ACCEPT;
}
auto &source_identifier = maybe_source_identifier.unpack();
dbgDebug(D_RATE_LIMIT) << "source identifier value: " << source_identifier;
string unique_key = asset_id + ":" + source_identifier + ":" + uri;
if (unique_key.back() == '/') unique_key.pop_back();
@ -224,7 +235,7 @@ public:
return ACCEPT;
}
if (verdict == RateLimitVedict::DROP_AND_LOG) sendLog(uri, source_identifier, rule);
if (verdict == RateLimitVedict::DROP_AND_LOG) sendLog(uri, source_identifier, source_ip, rule);
if (mode == "Active") {
dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit";
@ -242,7 +253,7 @@ public:
}
RateLimitVedict
decide(const std::string &key) {
decide(const string &key) {
if (redis == nullptr) {
dbgDebug(D_RATE_LIMIT)
<< "there is no connection to the redis at the moment, unable to enforce rate limit";
@ -288,7 +299,7 @@ public:
}
void
sendLog(const string &uri, const string &source_identifier, const RateLimitRule& rule)
sendLog(const string &uri, const string &source_identifier, const string &source_ip, const RateLimitRule &rule)
{
set<string> rate_limit_triggers_set;
for (const auto &trigger : rule.getRateLimitTriggers()) {
@ -296,7 +307,7 @@ public:
}
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, rate_limit_triggers_set);
ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, rate_limit_triggers_set);
auto log_trigger = getConfigurationWithDefault(LogTriggerConf(), "rulebase", "log");
if (!log_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) {
@ -336,25 +347,44 @@ public:
<< LogField("securityAction", (mode == "Active" ? "Prevent" : "Detect"))
<< LogField("waapIncidentType", "Rate Limit");
auto http_method =
Singleton::Consume<I_Environment>::by<RateLimit>()->get<string>(HttpTransactionData::method_ctx);
auto env = Singleton::Consume<I_Environment>::by<RateLimit>();
auto http_method = env->get<string>(HttpTransactionData::method_ctx);
if (http_method.ok()) log << LogField("httpMethod", http_method.unpack());
auto http_host =
Singleton::Consume<I_Environment>::by<RateLimit>()->get<string>(HttpTransactionData::host_name_ctx);
auto http_host = env->get<string>(HttpTransactionData::host_name_ctx);
if (http_host.ok()) log << LogField("httpHostName", http_host.unpack());
auto source_ip =
Singleton::Consume<I_Environment>::by<RateLimit>()->get<IPAddr>(HttpTransactionData::client_ip_ctx);
if (source_ip.ok()) log << LogField("sourceIP", ipAddrToStr(source_ip.unpack()));
if (!source_ip.empty()) log << LogField("sourceIP", source_ip);
auto proxy_ip =
Singleton::Consume<I_Environment>::by<RateLimit>()->get<std::string>(HttpTransactionData::proxy_ip_ctx);
if (proxy_ip.ok() && source_ip.ok() && ipAddrToStr(source_ip.unpack()) != proxy_ip.unpack()) {
log << LogField("proxyIP", static_cast<std::string>(proxy_ip.unpack()));
auto proxy_ip = env->get<string>(HttpTransactionData::proxy_ip_ctx);
if (proxy_ip.ok() && !source_ip.empty() && source_ip != proxy_ip.unpack()) {
log << LogField("proxyIP", static_cast<string>(proxy_ip.unpack()));
}
}
bool
shouldApplyException(const string &uri, const string &source_identifier, const string &source_ip)
{
dbgTrace(D_RATE_LIMIT) << "matching exceptions";
unordered_map<string, set<string>> exceptions_dict;
// collect sourceip, sourceIdentifier, url
if (!source_ip.empty()) exceptions_dict["sourceIP"].insert(source_ip);
exceptions_dict["sourceIdentifier"].insert(source_identifier);
exceptions_dict["url"].insert(uri);
auto behaviors = Singleton::Consume<I_GenericRulebase>::by<RateLimit>()->getBehavior(exceptions_dict);
for (auto const &behavior : behaviors) {
if (behavior == action_accept) {
dbgTrace(D_RATE_LIMIT) << "matched exceptions for " << uri << " should accept";
return true;
}
}
dbgTrace(D_RATE_LIMIT) << "No accept exceptions found for this uri and source ip";
return false;
}
string
ipAddrToStr(const IPAddr& ip_address) const
{
@ -368,14 +398,21 @@ public:
{
disconnectRedis();
const string &redis_ip = getConfigurationWithDefault<string>("127.0.0.1", "connection", "Redis IP");
int redis_port = getConfigurationWithDefault<int>(6379, "connection", "Redis Port");
redisOptions options;
memset(&options, 0, sizeof(redisOptions));
REDIS_OPTIONS_SET_TCP(
&options,
"127.0.0.1",
getConfigurationWithDefault<int>(6379, "connection", "Redis Port")
);
timeval timeout;
timeout.tv_sec = 0;
timeout.tv_usec = getConfigurationWithDefault<int>(30000, "connection", "Redis Timeout");
options.connect_timeout = &timeout;
options.command_timeout = &timeout;
redisContext* context = redisConnectWithTimeout(redis_ip.c_str(), redis_port, timeout);
redisContext* context = redisConnectWithOptions(&options);
if (context != nullptr && context->err) {
dbgDebug(D_RATE_LIMIT)
<< "Error connecting to Redis: "

1
components/security_apps/waap/include/i_transaction.h
View File

@ -126,7 +126,6 @@ public:
virtual void add_request_body_chunk(const char* data, int data_len) = 0;
virtual void end_request_body() = 0;
virtual void end_request() = 0;
// Response
virtual void start_response(int response_status, int http_version) = 0;
virtual void start_response_hdrs() = 0;

1
components/security_apps/waap/include/i_waapConfig.h
View File

@ -23,7 +23,6 @@
#include "../waap_clib/UserLimitsPolicy.h"
#include "../waap_clib/RateLimiting.h"
#include "../waap_clib/SecurityHeadersPolicy.h"
#include <memory>
enum class BlockingLevel {

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@ -84,6 +84,7 @@ add_library(waap_clib
WaapSampleValue.cc
ParserGql.cc
ParserPercentEncode.cc
ParserPairs.cc
)
add_definitions("-Wno-unused-function")

12
components/security_apps/waap/waap_clib/ContentTypeParser.cc
View File

@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_CONTENT_TYPE);
const std::string ContentTypeParser::m_parserName = "contentTypeParser";
int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth)
{
dbgTrace(D_WAAP_PARSER_CONTENT_TYPE) << "ContentTypeParser::onKv(): " << std::string(v, v_len);
assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH);
@ -42,10 +42,12 @@ int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v
return 0; // ok
}
ContentTypeParser::ContentTypeParser()
:ctParserState(CTP_STATE_CONTENT_TYPE), m_rcvr(*this), m_hvp(m_rcvr), m_error(false)
{
}
ContentTypeParser::ContentTypeParser() :
ctParserState(CTP_STATE_CONTENT_TYPE),
m_rcvr(*this),
m_hvp(m_rcvr),
m_error(false)
{}
size_t ContentTypeParser::push(const char *data, size_t data_len)
{

2
components/security_apps/waap/waap_clib/ContentTypeParser.h
View File

@ -25,7 +25,7 @@ class ContentTypeParser : public ParserBase, private IParserReceiver {
CTP_STATE_CONTENT_TYPE_PARAMS
} ctParserState;
private:
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
public:
ContentTypeParser();

2
components/security_apps/waap/waap_clib/DeepAnalyzer.cc
View File

@ -107,7 +107,7 @@ AnalysisResult DeepAnalyzer::Impl::analyzeData(const D2InputData& data, const IW
(shouldExcept ? "true" : "false");
analysis.threatLevel = threat;
analysis.shouldBlock = shouldBlock && !shouldExcept;
analysis.shouldBlock = shouldBlock;
return analysis;
}

1454
components/security_apps/waap/waap_clib/DeepParser.cc
View File

File diff suppressed because it is too large Load Diff

59
components/security_apps/waap/waap_clib/DeepParser.h
View File

@ -31,7 +31,7 @@ public:
void setWaapAssetState(std::shared_ptr<WaapAssetState> pWaapAssetState);
// This callback receives input key/value pairs, dissects, decodes and deep-scans these, recursively
// finally, it calls onDetected() on each detected parameter.
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
void clear();
void showStats(std::string& buff, const ValueStatsAnalyzer& valueStats);
@ -44,7 +44,7 @@ public:
void setMultipartBoundary(const std::string &boundary);
const std::string &getMultipartBoundary() const;
bool isBinaryData() const;
const std::string getLastParser() const;
const std::string getActualParser(size_t parser_depth) const;
bool isWBXmlData() const;
Maybe<std::string> getSplitType() const;
std::vector<std::pair<std::string, std::string> > kv_pairs;
@ -94,6 +94,7 @@ public:
std::vector<KeywordInfo> m_keywordInfo;
KeyStack m_key;
int getShiftInUrlEncodedBuffer(const ValueStatsAnalyzer &valueStats, std::string &cur_val);
private:
class Ref
@ -115,18 +116,60 @@ private:
// Split a value by given regexp. Return true if split, false otherwise.
// note: This function calls onKv(), and the call can be recursive!
// TODO:: maybe convert this splitter to Parser-derived class?!
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix);
void createInternalParser(const char *k, size_t k_len, std::string& cur_val,
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix, size_t parser_depth);
int createInternalParser(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags);
int pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound);
int parseBuffer(ValueStatsAnalyzer& valueStats, const std::string &cur_val, bool base64ParamFound,
bool shouldUpdateKeyStack);
int flags,
size_t parser_depth
);
int createUrlParserForJson(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags,
size_t parser_depth
);
void printParserDeque();
int parseAfterMisleadingMultipartBoundaryCleaned(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags,
size_t parser_depth,
bool base64ParamFound
);
int pushValueToTopParser(std::string &cur_val, int flags, bool base64ParamFound, int offset, size_t parser_depth);
int parseBuffer(
ValueStatsAnalyzer &valueStats,
const std::string &cur_val,
bool base64ParamFound,
bool shouldUpdateKeyStack,
size_t parser_depth
);
bool shouldEnforceDepthLimit(const std::shared_ptr<ParserBase>& parser) const;
void setLocalMaxObjectDepth(size_t depth) { m_localMaxObjectDepth = depth; }
void setGlobalMaxObjectDepthReached() { m_globalMaxObjectDepthReached = true; }

12
components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc
View File

@ -19,9 +19,15 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PHPSERIALIZE);
const std::string PHPSerializedDataParser::m_parserName = "PHPSerializedDataParser";
PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver)
: m_state(), m_outReceiver(outReceiver), m_keyStack("php_serialized")
PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth) :
m_state(),
m_outReceiver(outReceiver),
m_keyStack("php_serialized"),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_PHPSERIALIZE)
<< "parser_depth="
<< parser_depth;
}
size_t
@ -344,7 +350,7 @@ size_t PHPSerializedDataParser::handleValue (const char &c)
// else will parse it normaly.
dbgTrace(D_WAAP_PARSER_PHPSERIALIZE) << "PHPSerializedDataParser::push(): End of Class object" <<
" sending class object data to PHPSerializedDataParser";
PHPSerializedDataParser psdp(m_outReceiver);
PHPSerializedDataParser psdp(m_outReceiver, m_parser_depth);
psdp.push(m_value.c_str(), m_value.length());
if(psdp.error())
{

4
components/security_apps/waap/waap_clib/PHPSerializedDataParser.h
View File

@ -20,7 +20,7 @@
class PHPSerializedDataParser : public ParserBase {
public:
PHPSerializedDataParser(IParserStreamReceiver &outReceiver);
PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth);
size_t push(const char* buf, size_t len);
void finish();
virtual const std::string &name() const;
@ -84,6 +84,6 @@ private:
IParserStreamReceiver &m_outReceiver;
KeyStack m_keyStack;
std::stack <State> m_stack;
size_t m_parser_depth;
static const std::string m_parserName;
};

60
components/security_apps/waap/waap_clib/ParserBase.cc
View File

@ -13,19 +13,38 @@
#include "ParserBase.h"
#include <string.h>
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER);
// Max size for key and value that can be stored in memory (per thread)
#define MAX_KEY_SIZE 64*1024
#define MAX_VALUE_SIZE 64*1024
BufferedReceiver::BufferedReceiver(IParserReceiver &receiver)
:m_receiver(receiver),
m_flags(BUFFERED_RECEIVER_F_FIRST)
BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_flags(BUFFERED_RECEIVER_F_FIRST),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER)
<< "parser_depth="
<< parser_depth;
}
int BufferedReceiver::onKey(const char *k, size_t k_len)
{
dbgTrace(D_WAAP_PARSER)
<< "Entering BufferedReceiver::onKey with values: \n"
<< "\tkey = "
<< k
<< "\n\tk_len = "
<< k_len
<< "\n\tm_key = "
<< m_key
<< "\n\t m_key_size = "
<< m_key.size()
<< "\n\t parser_depth = "
<< m_parser_depth;
if (m_key.size() + k_len < MAX_KEY_SIZE) {
m_key += std::string(k, k_len);
}
@ -36,7 +55,18 @@ int BufferedReceiver::onKey(const char *k, size_t k_len)
int BufferedReceiver::onValue(const char *v, size_t v_len)
{
int rc = 0;
dbgTrace(D_WAAP_PARSER)
<< "Entering BufferedReceiver::onValue with values: \n"
<< "\tvalue = "
<< v
<< "\n\tv_len = "
<< v_len
<< "\n\tm_value = "
<< m_value
<< "\n\t m_value_size = "
<< m_value.size()
<< "\n\t parser_depth = "
<< m_parser_depth;
while (v_len > 0) {
// Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size
size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size());
@ -48,7 +78,18 @@ int BufferedReceiver::onValue(const char *v, size_t v_len)
// Only push full buffers to the m_receiver
if (m_value.size() == MAX_VALUE_SIZE) {
// The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag
int tempRc= m_receiver.onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags);
dbgTrace(D_WAAP_PARSER)
<< " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag"
<< "calling onKv() while m_flags = "
<< m_flags;
int tempRc= m_receiver.onKv(
m_key.data(),
m_key.size(),
m_value.data(),
m_value.size(),
m_flags,
m_parser_depth
);
if (tempRc != 0) {
rc = tempRc;
}
@ -68,16 +109,19 @@ int BufferedReceiver::onKvDone()
// Call onKv on the remainder of the buffer not yet pushed to the receiver
// This must be called even if m_value is empty in order to signal the BUFFERED_RECEIVER_F_LAST flag to the
// receiver!
int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags);
dbgTrace(D_WAAP_PARSER)
<< " Call onKv on the remainder of the buffer not yet pushed to the receiver "
<< "calling onKv()";
int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags, m_parser_depth);
// Reset the object's state to allow reuse for other parsers
clear();
return rc;
}
int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth)
{
return m_receiver.onKv(k, k_len, v, v_len, flags);
return m_receiver.onKv(k, k_len, v, v_len, flags, parser_depth);
}
void BufferedReceiver::clear()

15
components/security_apps/waap/waap_clib/ParserBase.h
View File

@ -33,7 +33,7 @@
// Interface for receiver classes that accept full key/value pairs
struct IParserReceiver {
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) = 0;
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) = 0;
};
struct IParserReceiver2 {
@ -80,11 +80,11 @@ struct IParserStreamReceiver : public IParserReceiver {
// This will in many cases cause sub-parsers to also work in zero-copy style too!
class BufferedReceiver : public IParserStreamReceiver {
public:
BufferedReceiver(IParserReceiver &receiver);
BufferedReceiver(IParserReceiver &receiver, size_t parser_depth=0);
virtual int onKey(const char *k, size_t k_len);
virtual int onValue(const char *v, size_t v_len);
virtual int onKvDone();
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
virtual void clear();
// Helper methods to access accumulated key and value (read-only)
@ -97,6 +97,8 @@ private:
// Accumulated key/value pair
std::string m_key;
std::string m_value;
size_t m_parser_depth;
};
// Base class for various streaming parsers that accept data stream in multiple pieces through
@ -123,10 +125,11 @@ class BufferedParser : public ParserBase
{
public:
template<typename ..._Args>
explicit BufferedParser(IParserReceiver &receiver, _Args... _args)
explicit BufferedParser(IParserReceiver &receiver, size_t parser_depth, _Args... _args)
:
m_bufferedReceiver(receiver),
m_parser(m_bufferedReceiver, _args...) // pass any extra arguments to specific parser's constructor
m_bufferedReceiver(receiver, parser_depth),
// pass any extra arguments to specific parser's constructor
m_parser(m_bufferedReceiver, parser_depth, _args...)
{}
virtual ~BufferedParser() {}
virtual size_t push(const char *data, size_t data_len) { return m_parser.push(data, data_len); }

8
components/security_apps/waap/waap_clib/ParserBinary.cc
View File

@ -18,13 +18,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_BINARY);
#define MIN_TEXT_SIZE 10
ParserBinary::ParserBinary(IParserStreamReceiver& receiver) :
ParserBinary::ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth) :
m_parserName("binary"),
m_receiver(receiver),
m_state(s_start),
m_textFromLastBuffer(),
m_textCharCount(0)
m_textCharCount(0),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_BINARY)
<< "parser_depth="
<< parser_depth;
}
ParserBinary::~ParserBinary()

4
components/security_apps/waap/waap_clib/ParserBinary.h
View File

@ -19,7 +19,7 @@
class ParserBinary : public ParserBase
{
public:
ParserBinary(IParserStreamReceiver& receiver);
ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth);
virtual ~ParserBinary();
virtual size_t push(const char* data, size_t data_len);
virtual void finish();
@ -39,7 +39,7 @@ private:
state m_state;
std::string m_textFromLastBuffer;
size_t m_textCharCount;
size_t m_parser_depth;
void flush();
};

9
components/security_apps/waap/waap_clib/ParserConfluence.cc
View File

@ -16,12 +16,17 @@
USE_DEBUG_FLAG(D_WAAP_PARSER_CONFLUENCE);
ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver) :
ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth) :
m_parserName("confluence"),
m_state(s_start),
m_receiver(receiver),
m_name()
m_name(),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_CONFLUENCE)
<< "parser_depth="
<< parser_depth;
}
ParserConfluence::~ParserConfluence()

3
components/security_apps/waap/waap_clib/ParserConfluence.h
View File

@ -19,7 +19,7 @@
class ParserConfluence : public ParserBase
{
public:
ParserConfluence(IParserStreamReceiver& receiver);
ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth);
virtual ~ParserConfluence();
virtual size_t push(const char* data, size_t data_len);
@ -43,6 +43,7 @@ private:
state m_state;
IParserStreamReceiver& m_receiver;
std::string m_name;
size_t m_parser_depth;
};
#endif

15
components/security_apps/waap/waap_clib/ParserDelimiter.cc
View File

@ -16,15 +16,22 @@
USE_DEBUG_FLAG(D_WAAP_PARSER_DELIMITER);
ParserDelimiter::ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName)
: ParserBase(),
ParserDelimiter::ParserDelimiter(
IParserStreamReceiver& receiver,
size_t parser_depth,
char delim,
const std::string& delimName
) : ParserBase(),
m_state(s_start),
m_receiver(receiver),
m_delim(delim),
m_delim_name(delimName),
m_found_delim(false)
m_found_delim(false),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_DELIMITER)
<< "parsing delimiter: parser depth="
<< parser_depth;
}
ParserDelimiter::~ParserDelimiter()

3
components/security_apps/waap/waap_clib/ParserDelimiter.h
View File

@ -19,7 +19,7 @@
class ParserDelimiter : public ParserBase
{
public:
ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName);
ParserDelimiter(IParserStreamReceiver& receiver, size_t parser_depth, char delim, const std::string& delimName);
virtual ~ParserDelimiter();
virtual size_t push(const char* data, size_t data_len);
@ -45,6 +45,7 @@ private:
char m_delim;
std::string m_delim_name;
bool m_found_delim;
size_t m_parser_depth;
};

18
components/security_apps/waap/waap_clib/ParserGql.cc
View File

@ -21,12 +21,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_GQL);
const std::string ParserGql::m_parserName = "gqlParser";
ParserGql::ParserGql(IParserReceiver& receiver) :
ParserGql::ParserGql(IParserReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_error(false),
m_curNameValues(0)
m_curNameValues(0),
m_parser_depth(parser_depth)
{
dbgFlow(D_WAAP_PARSER_GQL);
dbgTrace(D_WAAP_PARSER_GQL) << "parser_depth=" << parser_depth;
}
ParserGql::~ParserGql() {
@ -56,7 +58,9 @@ size_t ParserGql::push(const char* buf, size_t len) {
// Handle corner case of last name visited without value: don't forget to output that name too
if (m_curNameValues == 0 && !m_curNodeName.empty()) {
dbgTrace(D_WAAP_PARSER_GQL) << "handle last name: '" << m_curNodeName << "'";
if (m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH) != 0) {
if (m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
) != 0) {
m_error = true;
}
}
@ -81,7 +85,9 @@ bool ParserGql::visitValue(const char *value)
{
dbgTrace(D_WAAP_PARSER_GQL) << "'" << value << "'";
m_curNameValues++;
return m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH);
return m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH, m_parser_depth
);
}
bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
@ -89,7 +95,9 @@ bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
dbgTrace(D_WAAP_PARSER_GQL) << node.getValue() << "'";
bool ret = true;
if (m_curNameValues == 0 && !m_curNodeName.empty()) {
ret = m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH);
ret = m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
);
}
// wait for next name
m_curNodeName = std::string(node.getValue());

3
components/security_apps/waap/waap_clib/ParserGql.h
View File

@ -25,7 +25,7 @@
class ParserGql : public ParserBase, public facebook::graphql::ast::visitor::AstVisitor {
public:
ParserGql(IParserReceiver &receiver);
ParserGql(IParserReceiver &receiver, size_t parser_depth);
virtual ~ParserGql();
size_t push(const char *data, size_t data_len);
void finish();
@ -51,6 +51,7 @@ private:
bool visitEnumValue(const facebook::graphql::ast::EnumValue &node) override;
public:
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_JQL_H

146
components/security_apps/waap/waap_clib/ParserHTML.cc
View File

@ -39,15 +39,23 @@ void ParserHTML::onStartElement(
attr_value = (const xmlChar*)"";
}
dbgTrace(D_WAAP_PARSER_HTML) << "\tHTML ATTR: elem='" << (char*)localname << "', " << attr_localname <<
"='" << std::string((char*)attr_value) << "'";
p->m_key.push((const char*)attr_localname, xmlStrlen(attr_localname));
dbgTrace(D_WAAP_PARSER_HTML)
<< "\tHTML ATTR: elem='"
<< (char *)localname
<< "', "
<< attr_localname
<< "='"
<< std::string((char *)attr_value)
<< "'";
p->m_key.push((const char *)attr_localname, xmlStrlen(attr_localname));
if (p->m_receiver.onKv(
p->m_key.first().c_str(),
p->m_key.first().size(),
(const char*)attr_value, strlen((const char*)attr_value),
BUFFERED_RECEIVER_F_BOTH
) != 0) {
p->m_key.first().c_str(),
p->m_key.first().size(),
(const char *)attr_value,
strlen((const char *)attr_value),
BUFFERED_RECEIVER_F_BOTH,
p->m_parser_depth
) != 0) {
p->m_state = s_error;
}
p->m_key.pop("HTML end attribute");
@ -73,8 +81,8 @@ ParserHTML::onEndElement(
dbgTrace(D_WAAP_PARSER_HTML) << "HTML CLOSE: '" << localname << "'";
if (p->m_elemTrackStack.empty()) {
dbgWarning(D_WAAP_PARSER_HTML) <<
"HTML closing tag and elem track stack is empty. This is probably sign of a bug!";
dbgWarning(D_WAAP_PARSER_HTML)
<< "HTML closing tag and elem track stack is empty. This is probably sign of a bug!";
return;
}
@ -111,8 +119,10 @@ ParserHTML::onEndElement(
p->m_key.pop("HTML end element");
}
void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) {
ParserHTML* p = (ParserHTML*)ctx;
void
ParserHTML::onCharacters(void *ctx, const xmlChar *ch, int len)
{
ParserHTML *p = (ParserHTML *)ctx;
if (p->m_elemTrackStack.empty()) {
dbgWarning(D_WAAP_PARSER_HTML) << "HTML text and elem track stack is empty. This is probably sign of a bug!";
@ -141,7 +151,9 @@ void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) {
elemTrackInfo.value += val;
}
static void onError(void* ctx, const char* msg, ...) {
static void
onError(void *ctx, const char *msg, ...)
{
static const size_t TMP_BUF_SIZE = 4096;
char string[TMP_BUF_SIZE];
va_list arg_ptr;
@ -152,9 +164,19 @@ static void onError(void* ctx, const char* msg, ...) {
dbgTrace(D_WAAP_PARSER_HTML) << "LIBXML (html) onError: " << std::string(string);
}
ParserHTML::ParserHTML(IParserStreamReceiver& receiver)
:m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("html_parser"), m_pushParserCtxPtr(NULL) {
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::ParserHTML()";
ParserHTML::ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_bufLen(0),
m_key("html_parser"),
m_pushParserCtxPtr(NULL),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::ParserHTML()"
<< "parser_depth="
<< parser_depth;
// TODO:: is zeroing this really needed?
memset(m_buf, 0, sizeof(m_buf));
@ -173,7 +195,8 @@ ParserHTML::ParserHTML(IParserStreamReceiver& receiver)
m_key.push("html", 4);
}
ParserHTML::~ParserHTML() {
ParserHTML::~ParserHTML()
{
// Cleanup HTML
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::~ParserHTML()";
@ -182,9 +205,15 @@ ParserHTML::~ParserHTML() {
}
}
bool ParserHTML::filterErrors(xmlErrorPtr xmlError) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): xmlError " << xmlError->code << ": '" <<
xmlError->message << "'";
bool
ParserHTML::filterErrors(xmlErrorPtr xmlError)
{
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::filterErrors(): xmlError "
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
// Ignore specific error: "HTML declaration allowed only at the start of the document".
// This includes the case of "multiple HTML declarations" we've seen sent by some SOAP clients.
@ -193,15 +222,21 @@ bool ParserHTML::filterErrors(xmlErrorPtr xmlError) {
// Ignoring this error prevents the WAAP code from thinking the HTML is "broken" and from scanning the HTML
// source as-is, in effect preventing false alarm on that HTML source.
if (xmlError->code == XML_ERR_RESERVED_XML_NAME || xmlError->code == XML_ERR_UNDECLARED_ENTITY) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): ignoring the '" << xmlError->code << ": " <<
xmlError->message << "' html parser error.";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::filterErrors(): ignoring the '"
<< xmlError->code
<< ": "
<< xmlError->message
<< "' html parser error.";
return false;
}
return true;
}
size_t ParserHTML::push(const char* data, size_t data_len) {
size_t
ParserHTML::push(const char *data, size_t data_len)
{
size_t i = 0;
char c;
@ -212,8 +247,12 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
if (htmlParseChunk(m_pushParserCtxPtr, m_buf, 0, 1)) {
xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr);
if (xmlError && filterErrors(xmlError)) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code << ": '" <<
xmlError->message << "'";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): xmlError: code="
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
m_state = s_error; // error
return -1;
}
@ -232,8 +271,13 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
// fall through //
CP_FALL_THROUGH;
case s_accumulate_first_bytes:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_accumulate_first_bytes. c='" << data[i] <<
"'; m_bufLen=" << m_bufLen << "; i=" << i;
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_accumulate_first_bytes. c='"
<< data[i]
<< "'; m_bufLen="
<< m_bufLen
<< "; i="
<< i;
m_buf[m_bufLen] = c;
m_bufLen++;
if (c == '?') {
@ -244,13 +288,19 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
}
break;
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_start_parsing. sending len=" << m_bufLen << ": '" <<
std::string(m_buf, m_bufLen) << "'; i=" << i;
// Create HTML SAX (push parser) context
// It is important to buffer at least first 4 bytes of input stream so libxml can determine text encoding!
m_pushParserCtxPtr = htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL,
XML_CHAR_ENCODING_UTF8);
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_start_parsing. sending len="
<< m_bufLen
<< ": '"
<< std::string(m_buf, m_bufLen)
<< "'; i="
<< i;
// Create HTML SAX (push parser) context
// It is important to buffer at least first 4 bytes of input stream so libxml can determine text
// encoding!
m_pushParserCtxPtr =
htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL, XML_CHAR_ENCODING_UTF8);
// Enable "permissive mode" for HTML SAX parser.
// In this mode, the libxml parser doesn't stop on errors, but still reports them!
@ -261,14 +311,23 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
// fall through //
CP_FALL_THROUGH;
case s_parsing:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_parsing. sending len=" << (int)(data_len - i) <<
": '" << std::string(data + i, data_len - i) << "'; i=" << i;
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_parsing. sending len="
<< (int)(data_len - i)
<< ": '"
<< std::string(data + i, data_len - i)
<< "'; i="
<< i;
if (m_pushParserCtxPtr) {
if (htmlParseChunk(m_pushParserCtxPtr, data + i, data_len - i, 0)) {
xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr);
if (xmlError && filterErrors(xmlError)) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code <<
": '" << xmlError->message << "'";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): xmlError: code="
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
m_state = s_error; // error
return 0;
}
@ -290,15 +349,20 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
return i;
}
void ParserHTML::finish() {
void
ParserHTML::finish()
{
push(NULL, 0);
}
const std::string &
ParserHTML::name() const {
ParserHTML::name() const
{
return m_parserName;
}
bool ParserHTML::error() const {
bool
ParserHTML::error() const
{
return m_state == s_error;
}

3
components/security_apps/waap/waap_clib/ParserHTML.h
View File

@ -24,7 +24,7 @@
class ParserHTML : public ParserBase {
public:
ParserHTML(IParserStreamReceiver &receiver);
ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserHTML();
size_t push(const char *data, size_t data_len);
void finish();
@ -81,4 +81,5 @@ private:
htmlParserCtxtPtr m_pushParserCtxPtr;
static const std::string m_parserName;
size_t m_parser_depth;
};

152
components/security_apps/waap/waap_clib/ParserJson.cc
View File

@ -20,17 +20,21 @@
#include <assert.h>
USE_DEBUG_FLAG(D_WAAP_PARSER_JSON);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
const std::string ParserJson::m_parserName = "jsonParser";
int ParserJson::cb_null() {
int
ParserJson::cb_null()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_null():";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), "null", 4, DataType::EMPTY);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
@ -41,20 +45,22 @@ int ParserJson::cb_null() {
return 1;
}
int ParserJson::cb_boolean(int boolean) {
int
ParserJson::cb_boolean(int boolean)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_boolean(): " << boolean;
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), NULL, boolean, DataType::BOOLEAN);
}
if (boolean) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
}
else {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH)) {
} else {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
}
@ -65,14 +71,16 @@ int ParserJson::cb_boolean(int boolean) {
return 1;
}
int ParserJson::cb_number(const char* s, yajl_size_t slen) {
int
ParserJson::cb_number(const char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_number(): '" << std::string(s, slen) << "'";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), s, slen, DataType::NUMBER);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
@ -82,14 +90,19 @@ int ParserJson::cb_number(const char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char*)s, slen) << "'";
int
ParserJson::cb_string(const unsigned char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char *)s, slen) << "'";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), (const char*)s, slen, DataType::STRING);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), (const char*)s, slen, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(
m_key.c_str(), m_key.size(), (const char *)s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
)) {
return 0;
}
@ -99,8 +112,10 @@ int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char*)s, slen) << "'";
int
ParserJson::cb_map_key(const unsigned char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char *)s, slen) << "'";
m_key.push((char*)s, slen);
@ -111,7 +126,9 @@ int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_start_map() {
int
ParserJson::cb_start_map()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_map():";
if (m_receiver2) {
@ -122,7 +139,9 @@ int ParserJson::cb_start_map() {
return 1;
}
int ParserJson::cb_end_map() {
int
ParserJson::cb_end_map()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_map():";
if (m_receiver2) {
@ -140,7 +159,9 @@ int ParserJson::cb_end_map() {
return 1;
}
int ParserJson::cb_start_array() {
int
ParserJson::cb_start_array()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_array():";
if (m_receiver2) {
@ -151,7 +172,9 @@ int ParserJson::cb_start_array() {
return 1;
}
int ParserJson::cb_end_array() {
int
ParserJson::cb_end_array()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_array():";
if (m_receiver2) {
@ -162,6 +185,7 @@ int ParserJson::cb_end_array() {
m_depthStack.pop_back();
}
if (!m_depthStack.empty() && m_depthStack.back() == js_map) {
m_key.pop("json end array");
}
@ -169,51 +193,78 @@ int ParserJson::cb_end_array() {
}
// Static functions to be called from C and forward the calls to respective class cb_* methods
int ParserJson::p_null(void* ctx)
int
ParserJson::p_null(void *ctx)
{
return ((ParserJson*)ctx)->cb_null();
}
int ParserJson::p_boolean(void* ctx, int boolean)
int
ParserJson::p_boolean(void *ctx, int boolean)
{
return ((ParserJson*)ctx)->cb_boolean(boolean);
}
int ParserJson::p_number(void* ctx, const char* s, yajl_size_t slen)
int
ParserJson::p_number(void *ctx, const char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_number(s, slen);
}
int ParserJson::p_string(void* ctx, const unsigned char* s, yajl_size_t slen)
int
ParserJson::p_string(void *ctx, const unsigned char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_string(s, slen);
}
int ParserJson::p_map_key(void* ctx, const unsigned char* s, yajl_size_t slen)
int
ParserJson::p_map_key(void *ctx, const unsigned char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_map_key(s, slen);
}
int ParserJson::p_start_map(void* ctx)
int
ParserJson::p_start_map(void *ctx)
{
return ((ParserJson*)ctx)->cb_start_map();
}
int ParserJson::p_end_map(void* ctx)
int
ParserJson::p_end_map(void *ctx)
{
return ((ParserJson*)ctx)->cb_end_map();
}
int ParserJson::p_start_array(void* ctx)
int
ParserJson::p_start_array(void *ctx)
{
return ((ParserJson*)ctx)->cb_start_array();
}
int ParserJson::p_end_array(void* ctx)
int
ParserJson::p_end_array(void *ctx)
{
return ((ParserJson*)ctx)->cb_end_array();
}
ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) :
ParserJson::ParserJson(
IParserReceiver &receiver,
bool should_collect_oas,
size_t parser_depth,
IParserReceiver2 *receiver2)
:
m_receiver(receiver),
m_receiver2(receiver2),
m_state(s_start),
m_bufLen(0),
m_key("json_parser"),
m_jsonHandler(NULL)
m_jsonHandler(NULL),
is_map_empty(false),
should_collect_for_oa_schema_updater(should_collect_oas),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_JSON) << "parser_depth= " << parser_depth;
// TODO:: do we really want to clear this?
memset(m_buf, 0, sizeof(m_buf));
@ -249,7 +300,8 @@ ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) :
m_key.push("json", 4);
}
ParserJson::~ParserJson() {
ParserJson::~ParserJson()
{
// Cleanup JSON
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::~ParserJson():";
@ -258,7 +310,9 @@ ParserJson::~ParserJson() {
}
}
size_t ParserJson::push(const char* buf, size_t len) {
size_t
ParserJson::push(const char *buf, size_t len)
{
size_t i = 0;
char c;
@ -280,6 +334,7 @@ size_t ParserJson::push(const char* buf, size_t len) {
while (i < len) {
c = buf[i];
switch (m_state) {
case s_start:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start";
@ -288,8 +343,12 @@ size_t ParserJson::push(const char* buf, size_t len) {
// fallthrough //
CP_FALL_THROUGH;
case s_accumulate_first_bytes:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_accumulate_first_bytes. i=" << i <<
" c='" << buf[i] << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_accumulate_first_bytes. i="
<< i
<< " c='"
<< buf[i]
<< "'";
m_buf[m_bufLen] = c;
m_bufLen++;
if (m_bufLen == FIRST_JSON_BUFFER_SIZE) {
@ -298,15 +357,23 @@ size_t ParserJson::push(const char* buf, size_t len) {
break;
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start_parsing. sending len=" <<
(int)m_bufLen << ": '" << std::string(m_buf, m_bufLen) << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_start_parsing. sending len="
<< (int)m_bufLen
<< ": '"
<< std::string(m_buf, m_bufLen)
<< "'";
m_state = s_parsing;
// fallthrough //
CP_FALL_THROUGH;
case s_parsing:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_parsing. sending len=" << (int)(len - i) << ": '" <<
std::string(buf + i, len - i) << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_parsing. sending len="
<< (int)(len - i)
<< ": '"
<< std::string(buf + i, len - i)
<< "'";
if (m_bufLen > 0) {
// Send accumulated bytes (if any)
if (yajl_parse(m_jsonHandler, (unsigned char*)m_buf, m_bufLen) != yajl_status_ok) {
@ -333,15 +400,20 @@ size_t ParserJson::push(const char* buf, size_t len) {
return len;
}
void ParserJson::finish() {
void
ParserJson::finish()
{
push(NULL, 0);
}
const std::string &
ParserJson::name() const {
ParserJson::name() const
{
return m_parserName;
}
bool ParserJson::error() const {
bool
ParserJson::error() const
{
return m_state == s_error;
}

10
components/security_apps/waap/waap_clib/ParserJson.h
View File

@ -27,7 +27,11 @@ typedef size_t yajl_size_t;
class ParserJson : public ParserBase {
public:
ParserJson(IParserReceiver &receiver, IParserReceiver2 *receiver2=NULL);
ParserJson(
IParserReceiver &receiver,
bool should_collect_for_oa_schema_updater=false,
size_t parser_depth=0,
IParserReceiver2 *receiver2=NULL);
virtual ~ParserJson();
size_t push(const char *data, size_t data_len);
void finish();
@ -80,6 +84,10 @@ private:
KeyStack m_key;
std::vector<enum js_state> m_depthStack;
yajl_handle m_jsonHandler;
bool is_map_empty;
bool should_collect_for_oa_schema_updater;
size_t m_parser_depth;
public:
static const std::string m_parserName;
};

22
components/security_apps/waap/waap_clib/ParserMultipartForm.cc
View File

@ -26,7 +26,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_MULTIPART_FORM);
const std::string ParserMultipartForm::m_parserName = "ParserMultipartForm";
int ParserMultipartForm::HdrValueAnalyzer::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags)
int ParserMultipartForm::HdrValueAnalyzer::onKv(
const char* k,
size_t k_len,
const char* v,
size_t v_len,
int flags,
size_t parser_depth
)
{
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "HdrValueAnalyzer::onKv(): k='%.*s' v='%.*s'" << (int)k_len << v;
assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH);
@ -43,10 +50,8 @@ void ParserMultipartForm::HdrValueAnalyzer::clear() {
}
ParserMultipartForm::ParserMultipartForm(
IParserStreamReceiver& receiver,
const char* boundary,
size_t boundary_len)
:
IParserStreamReceiver &receiver, size_t parser_depth, const char *boundary, size_t boundary_len
) :
m_receiver(receiver),
m_partIdx(0),
state(s_start),
@ -55,9 +60,12 @@ ParserMultipartForm::ParserMultipartForm(
lookbehind(NULL),
multipart_boundary(NULL),
m_headerValueParser(NULL),
m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer)
m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "ParserMultipartForm::ParserMultipartForm()";
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM)
<< "ParserMultipartForm::ParserMultipartForm() parser_depth="
<< parser_depth;
boundary_len += 2; // two hyphens will be prepended to boundary string provided
multipart_boundary = (char*)malloc(boundary_len + boundary_len + 9);

19
components/security_apps/waap/waap_clib/ParserMultipartForm.h
View File

@ -18,18 +18,25 @@
#include "ParserHdrValue.h"
#include <boost/core/noncopyable.hpp>
class ParserMultipartForm : public ParserBase, boost::noncopyable {
class ParserMultipartForm : public ParserBase, boost::noncopyable
{
public:
class HdrValueAnalyzer : public IParserReceiver {
class HdrValueAnalyzer : public IParserReceiver
{
public:
int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
void clear();
const std::string &getPartName() const { return m_partName; }
private:
std::string m_partName;
};
ParserMultipartForm(IParserStreamReceiver &receiver, const char *boundary, size_t boundary_len);
ParserMultipartForm(
IParserStreamReceiver &receiver,
size_t parser_depth,
const char *boundary,
size_t boundary_len
);
virtual ~ParserMultipartForm();
size_t push(const char *buf, size_t len);
void finish();
@ -37,7 +44,8 @@ public:
virtual bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
enum state
{
s_start,
s_start_boundary,
s_key_start,
@ -88,6 +96,7 @@ private:
std::string m_partName; // Part name
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_MULTIPART_FORM_H__1c7eb4fa

479
components/security_apps/waap/waap_clib/ParserPairs.cc Normal file
View File

@ -0,0 +1,479 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserPairs.h"
#include "Waf2Util.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_PAIRS);
USE_DEBUG_FLAG(D_WAAP);
const std::string ParserPairs::m_parserName = "ParserPairs";
ParserPairs::ParserPairs(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar,
bool should_decode_per,
bool should_decode_plus_sign
) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_separatorChar(separatorChar),
m_escapedCharCandidate(0),
should_decode_percent(should_decode_per),
should_decode_plus(should_decode_plus_sign),
m_parser_depth(parser_depth),
m_bracket_counter(0)
{
dbgTrace(D_WAAP)
<< "should_decode_percent="
<< should_decode_per
<< "parser_depth="
<< parser_depth;
// TODO:: is there a need for this?
memset(m_escaped, 0, sizeof(m_escaped));
}
ParserPairs::~ParserPairs()
{}
size_t
ParserPairs::push(const char *buf, size_t len)
{
size_t i = 0;
size_t mark = 0;
char c;
int is_last = 0;
dbgTrace(D_WAAP) << "ParserPairs::push(): starting (len=" << len << ")";
if (len == 0) {
dbgTrace(D_WAAP) << "ParserPairs::push(): end of data signal! m_state=" << m_state;
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_state == s_key_start) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
} else if (m_state == s_value_start) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
}
m_escapedLen = 0;
}
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
return i;
}
return 0;
}
while (i < len) {
c = buf[i];
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32) {
// Bitwise operation checking since `char` can be either signed on unsigned depending on arch
if (!isspace(c) && ((c & 0x80) == 0x00)) {
dbgDebug(D_WAAP_PARSER_PAIRS)
<< "invalid URL encoding character: "
<< c
<< " decimal value is "
<< c + 0;
m_state = s_error;
return i;
}
}
dbgTrace(D_WAAP_PARSER_PAIRS)
<< "ParserPairs::push(): state="
<< m_state
<< "; ch='"
<< c
<< "' m_bracket_counter = "
<< m_bracket_counter;
switch (m_state) {
case s_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_start";
// m_state = s_key_start;
// fallthrough //
CP_FALL_THROUGH;
}
case s_key_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_start";
mark = i;
m_state = s_key;
// fallthrough //
CP_FALL_THROUGH;
}
case s_key: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key";
// skip leading spaces in the key
if (isspace(c)) {
m_state = s_key_start; // skip the space character without including it in the output
break;
}
if (c == '%' && should_decode_percent) {
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_state = s_key_escaped1;
break;
} else if (c == '+' && should_decode_plus) {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
mark = i;
}
m_escaped[m_escapedLen] = ' ';
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_key_start;
break;
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
}
if (c == m_separatorChar) {
// this happens when there is a key without value. Example: ?p&a=b&k&%61&blah
// in this case we emit the key, but not the value, and send onKvDone to cause
// the receiver to process the pair: key will be provided with no value.
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
return i;
}
m_state = s_key_start;
break;
}
if (c == '=') {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
m_state = s_value_start;
break;
}
if (is_last) {
if (m_receiver.onKey(buf + mark, (i - mark) + 1) != 0) {
m_state = s_error;
return i;
}
}
break;
}
case s_key_escaped1: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped1";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) { // character right after the '%' is not a valid hex char.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onKey("%", 1) != 0) {
return i;
}
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onKey(&c, 1) != 0) {
return i;
}
// otherwise (the character is not '%'), switch back to the s_key state
m_state = s_key_start;
}
break;
}
m_escapedCharCandidate = c;
m_escaped[m_escapedLen] = v << 4;
m_state = s_key_escaped2;
break;
}
case s_key_escaped2: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped2";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) {
// This situation (2nd character is not valid hex) is not treated right now.
// In this case, v will be equal to 0 and output character will be invalid one.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onKey("%", 1) != 0) {
return i;
}
// add the character that was thought to be escaped value
if (m_receiver.onKey(&m_escapedCharCandidate, 1)) {
return i;
}
// re parse the character as a key (i is incremented back to current value)
i--;
m_state = s_key_start;
break;
}
m_escapedCharCandidate = 0;
m_escaped[m_escapedLen] |= v;
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_key_start;
break;
}
case s_value_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_start";
mark = i;
m_state = s_value;
// fallthrough //
CP_FALL_THROUGH;
}
case s_value: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value";
if (c == '%' && should_decode_percent) {
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_state = s_value_escaped1;
break;
} else if (c == '+' && should_decode_plus) {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_escaped[m_escapedLen] = ' ';
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_value_start;
break;
} else {
// flush unescaped data collected (if any)
if (c == '{' || c == '[') m_bracket_counter++;
if (c == '}' || c == ']') m_bracket_counter--;
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
}
if (c == m_separatorChar) {
if (m_bracket_counter) {
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
} else {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() s_value : failed on value";
m_state = s_error;
return i;
}
if (m_receiver.onKvDone() != 0) {
dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() : s_value : failed on KV";
m_state = s_error;
return i;
}
m_state = s_key_start;
break;
}
}
if (is_last) {
if (m_receiver.onValue(buf + mark, (i - mark) + 1) != 0) {
m_state = s_error;
return i;
}
}
break;
}
case s_value_escaped1: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped1";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) { // character right after the '%' is not a valid hex char.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onValue("%", 1) != 0) {
return i;
}
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences)
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onValue(&c, 1) != 0) {
return i;
}
// otherwise (the character is not '%'), switch back to the s_value state
m_state = s_value_start;
}
break;
}
m_escapedCharCandidate = c;
m_escaped[m_escapedLen] = v << 4;
m_state = s_value_escaped2;
break;
}
case s_value_escaped2: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped2";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) {
// This situation (2nd character is not valid hex) is not treated right now.
// In this case, v will be equal to 0 and output character will be invalid one.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onValue("%", 1) != 0) {
return i;
}
// add the character that was thought to be escaped value
if (m_receiver.onValue(&m_escapedCharCandidate, 1)) {
return i;
}
// re parse the character as a key (i is incremented back to current value)
i--;
m_state = s_value_start;
break;
}
m_escapedCharCandidate = 0;
m_escaped[m_escapedLen] |= v;
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_value_start;
break;
}
case s_error: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_error";
return 0;
}
default: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): URL parser unrecoverable error";
m_state = s_error;
return 0;
}
} // end of switch()
++i;
}
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): finished: len=" << len;
return len;
}
void
ParserPairs::finish()
{
push(NULL, 0);
}
const std::string &
ParserPairs::name() const
{
return m_parserName;
}
bool
ParserPairs::error() const
{
return m_state == s_error;
}

65
components/security_apps/waap/waap_clib/ParserPairs.h Executable file
View File

@ -0,0 +1,65 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_PAIRS_H__
#define __PARSER_PAIRS_H__
#include "ParserBase.h"
#include <string.h>
#define MAX_PAIRS_ESCAPED_SIZE 16
class ParserPairs : public ParserBase {
public:
ParserPairs(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar = '&',
bool should_decode_per = false,
bool should_decode_plus_sign = false);
virtual ~ParserPairs();
size_t push(const char *data, size_t data_len);
void finish();
virtual const std::string &name() const;
bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
s_start,
s_key_start,
s_key,
s_key_escaped1,
s_key_escaped2,
s_value_start,
s_value,
s_value_escaped1,
s_value_escaped2,
s_end,
s_error
};
IParserStreamReceiver &m_receiver;
enum state m_state;
unsigned char m_escapedLen; // count of characters loaded in m_escaped[] buffer
char m_escaped[MAX_PAIRS_ESCAPED_SIZE];
char m_separatorChar;
char m_escapedCharCandidate;
bool should_decode_percent;
bool should_decode_plus;
static const std::string m_parserName;
size_t m_parser_depth;
int m_bracket_counter;
};
#endif // __PARSER_PAIRS_H__

42
components/security_apps/waap/waap_clib/ParserPercentEncode.cc
View File

@ -19,12 +19,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PERCENT);
const std::string ParserPercentEncode::m_parserName = "ParserPercentEncode";
ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver) :
ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_escapedCharCandidate(0)
m_escapedCharCandidate(0),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "parser_depth="
<< parser_depth;
memset(m_escaped, 0, sizeof(m_escaped));
}
@ -51,7 +56,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
@ -97,7 +103,6 @@ ParserPercentEncode::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_start";
// fallthrough //
CP_FALL_THROUGH;
}
@ -107,7 +112,6 @@ ParserPercentEncode::push(const char *buf, size_t len)
<< "ParserPercentEncode::push(): s_value_start";
pointer_in_buffer = i;
m_state = s_value;
// fallthrough //
CP_FALL_THROUGH;
}
@ -120,9 +124,10 @@ ParserPercentEncode::push(const char *buf, size_t len)
if (i - pointer_in_buffer > 0)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with buffer = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
<< "<<< of size "
<< i - pointer_in_buffer;
if (m_receiver.onValue(buf + pointer_in_buffer, i - pointer_in_buffer) != 0)
{
m_state = s_error;
@ -140,7 +145,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
@ -155,7 +161,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
<< "<<< of size "
<< (i - pointer_in_buffer) + 1;
if (m_receiver.onValue(buf + pointer_in_buffer, (i - pointer_in_buffer) + 1) != 0)
{
m_state = s_error;
@ -177,7 +184,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
@ -186,7 +194,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
}
m_escapedLen = 0;
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with \"%\" = >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
@ -199,7 +207,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
{
// pass the non-hex character back to the output too.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with current char = >>>"
<< c
<< "<<<";
if (m_receiver.onValue(&c, 1) != 0)
@ -232,7 +240,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
@ -243,7 +252,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with \"%\" >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
@ -252,7 +261,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
}
// add the character that was thought to be escaped value
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with m_escapedCharCandicate = >>>"
<< m_escapedCharCandidate
<< "<<<";
if (m_receiver.onValue(&m_escapedCharCandidate, 1))
@ -273,7 +282,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;

3
components/security_apps/waap/waap_clib/ParserPercentEncode.h
View File

@ -22,7 +22,7 @@
class ParserPercentEncode : public ParserBase {
public:
ParserPercentEncode(IParserStreamReceiver &receiver);
ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserPercentEncode();
size_t push(const char *data, size_t data_len);
void finish();
@ -53,6 +53,7 @@ private:
char m_escaped[MAX_PERCENT_ENCODED_SIZE];
char m_escapedCharCandidate;
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif

2
components/security_apps/waap/waap_clib/ParserRaw.cc
View File

@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_RAW);
const std::string ParserRaw::m_parserName = "ParserRaw";
ParserRaw::ParserRaw(IParserStreamReceiver &receiver, const std::string &key)
ParserRaw::ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key)
:m_receiver(receiver), m_key(key), m_state(s_start) {
}

2
components/security_apps/waap/waap_clib/ParserRaw.h
View File

@ -19,7 +19,7 @@
class ParserRaw : public ParserBase {
public:
ParserRaw(IParserStreamReceiver &receiver, const std::string &key);
ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key);
virtual ~ParserRaw();
size_t push(const char *data, size_t data_len);
void finish();

54
components/security_apps/waap/waap_clib/ParserUrlEncode.cc
View File

@ -16,27 +16,37 @@
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_URLENCODE);
USE_DEBUG_FLAG(D_WAAP);
const std::string ParserUrlEncode::m_parserName = "ParserUrlEncode";
ParserUrlEncode::ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar, bool should_decode_per)
:
ParserUrlEncode::ParserUrlEncode(
IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar, bool should_decode_per
) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_separatorChar(separatorChar),
m_escapedCharCandidate(0),
should_decode_percent(should_decode_per)
should_decode_percent(should_decode_per),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_URLENCODE) << "should_decode_per=" << should_decode_per;
dbgTrace(D_WAAP)
<< "should_decode_percent="
<< should_decode_per
<< "parser_depth="
<< parser_depth;
// TODO:: is there a need for this?
memset(m_escaped, 0, sizeof(m_escaped));
}
ParserUrlEncode::~ParserUrlEncode() {
}
ParserUrlEncode::~ParserUrlEncode()
{}
size_t ParserUrlEncode::push(const char *buf, size_t len) {
size_t
ParserUrlEncode::push(const char *buf, size_t len)
{
size_t i = 0;
size_t mark = 0;
char c;
@ -53,8 +63,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
m_state = s_error;
return i;
}
}
else if (m_state == s_value_start) {
} else if (m_state == s_value_start) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
@ -76,8 +85,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32)
{
if (c < 32) {
dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c;
m_state = s_error;
return i;
@ -119,8 +127,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_key_escaped1;
break;
}
else if (c == '+') {
} else if (c == '+') {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
@ -140,8 +147,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_key_start;
break;
}
else {
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
@ -201,7 +207,6 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onKey(&c, 1) != 0) {
return i;
@ -279,8 +284,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_value_escaped1;
break;
}
else if (c == '+') {
} else if (c == '+') {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
@ -299,8 +303,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_value_start;
break;
}
else {
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
@ -425,15 +428,20 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
return len;
}
void ParserUrlEncode::finish() {
void
ParserUrlEncode::finish()
{
push(NULL, 0);
}
const std::string &
ParserUrlEncode::name() const {
ParserUrlEncode::name() const
{
return m_parserName;
}
bool ParserUrlEncode::error() const {
bool
ParserUrlEncode::error() const
{
return m_state == s_error;
}

7
components/security_apps/waap/waap_clib/ParserUrlEncode.h
View File

@ -21,7 +21,11 @@
class ParserUrlEncode : public ParserBase {
public:
ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar = '&', bool should_decode_per = true);
ParserUrlEncode(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar = '&',
bool should_decode_per = true);
virtual ~ParserUrlEncode();
size_t push(const char *data, size_t data_len);
void finish();
@ -52,6 +56,7 @@ private:
char m_escapedCharCandidate;
bool should_decode_percent;
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_URL_ENCODE_H__29ebe806

22
components/security_apps/waap/waap_clib/ParserXML.cc
View File

@ -60,9 +60,11 @@ void ParserXML::onStartElementNs(
if (p->m_receiver.onKv(
p->m_key.c_str(),
p->m_key.size(),
(const char*)attr_value_begin, attr_value_end - attr_value_begin,
BUFFERED_RECEIVER_F_BOTH
) != 0) {
(const char *)attr_value_begin,
attr_value_end - attr_value_begin,
BUFFERED_RECEIVER_F_BOTH,
p->m_parser_depth
) != 0) {
p->m_state = s_error;
}
p->m_key.pop("XML end attribute");
@ -195,9 +197,17 @@ static void onError(void* ctx, const char* msg, ...) {
dbgTrace(D_WAAP_PARSER_XML) << "LIBXML (xml) onError: " << std::string(string);
}
ParserXML::ParserXML(IParserStreamReceiver& receiver)
:m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("xml_parser"), m_pushParserCtxPtr(NULL) {
dbgTrace(D_WAAP_PARSER_XML) << "ParserXML::ParserXML()";
ParserXML::ParserXML(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_bufLen(0),
m_key("xml_parser"),
m_pushParserCtxPtr(NULL),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_XML)
<< "ParserXML::ParserXML() parser_depth="
<< parser_depth;
// TODO:: is zeroing this really needed?
memset(m_buf, 0, sizeof(m_buf));

3
components/security_apps/waap/waap_clib/ParserXML.h
View File

@ -24,7 +24,7 @@
class ParserXML : public ParserBase {
public:
ParserXML(IParserStreamReceiver &receiver);
ParserXML(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserXML();
size_t push(const char *data, size_t data_len);
void finish();
@ -94,6 +94,7 @@ private:
std::vector<ElemTrackInfo> m_elemTrackStack;
xmlSAXHandler m_saxHandler;
xmlParserCtxtPtr m_pushParserCtxPtr;
size_t m_parser_depth;
public:
static const std::string m_parserName;
};

49
components/security_apps/waap/waap_clib/Serializator.cc
View File

@ -96,7 +96,6 @@ Maybe<string> RestGetFile::genJson() const
return genError("Failed to compress data");
}
data = string((const char *)res.output, res.num_output_bytes);
json = data;
if (res.output) free(res.output);
@ -179,10 +178,54 @@ void SerializeToFileBase::saveData()
serialize(ss);
string data = ss.str();
auto compression_stream = initCompressionStream();
CompressionResult res = compressData(
compression_stream,
CompressionType::GZIP,
data.size(),
reinterpret_cast<const unsigned char *>(data.c_str()),
true
);
finiCompressionStream(compression_stream);
if (!res.ok) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data";
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
}
filestream << ss.str();
filestream.close();
}
string decompress(string fileContent) {
if (!isGZipped(fileContent)) {
dbgTrace(D_WAAP) << "file note zipped";
return fileContent;
}
auto compression_stream = initCompressionStream();
DecompressionResult res = decompressData(
compression_stream,
fileContent.size(),
reinterpret_cast<const unsigned char *>(fileContent.c_str())
);
finiCompressionStream(compression_stream);
if (res.ok) {
string decompressedData = string((const char *)res.output, res.num_output_bytes);
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
return decompressedData;
}
return fileContent;
}
void SerializeToFileBase::loadFromFile(string filePath)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loadFromFile() file: " << filePath;
@ -239,8 +282,8 @@ void SerializeToFileBase::loadFromFile(string filePath)
delete[] buffer;
stringstream ss(dataObfuscated);
stringstream ss;
ss << decompress(dataObfuscated);
try
{

12
components/security_apps/waap/waap_clib/Signatures.cc
View File

@ -85,13 +85,6 @@ static filtered_parameters_t to_filtermap(const picojson::value::object& JsObj)
return result;
}
std::string genDelimitedKeyValPattern(const std::string& delim)
{
std::string pattern = "^([^" + delim + "]+?=[^" + delim + "]+?" + delim + ")+"
"([^" + delim + "]+?=[^" + delim + "]+?)" + delim + "?$";
return pattern;
}
Signatures::Signatures(const std::string& filepath) :
sigsSource(loadSource(filepath)),
error(false),
@ -207,11 +200,6 @@ Signatures::Signatures(const std::string& filepath) :
html_regex("(<(?>body|head)\\b.*>(?>.|[\\r\\n]){0,400}){2}|<html", error, "htmlRegex"),
uri_parser_regex("(http|https)://([^/ :]+):?([^/ ]*)(/?[^ #?]*)", error, "uriParserRegex"),
confluence_macro_re("{[^\"]+:(?>.+\\|)+.+}"),
pipes_delimited_key_val_re(genDelimitedKeyValPattern("\\|")),
semicolon_delimited_key_val_re(genDelimitedKeyValPattern(";")),
asterisk_delimited_key_val_re(genDelimitedKeyValPattern("\\*")),
comma_delimited_key_val_re(genDelimitedKeyValPattern(",")),
ampersand_delimited_key_val_re(genDelimitedKeyValPattern("&")),
headers_re(to_regexmap(sigsSource["headers_re"].get<JsObj>(), error)),
format_magic_binary_re(sigsSource["format_magic_binary_re"].get<std::string>(), error, "format_magic_binary_re"),
params_type_re(to_regexmap(sigsSource["format_types_regex_list"].get<JsObj>(), error)),

5
components/security_apps/waap/waap_clib/Signatures.h
View File

@ -61,11 +61,6 @@ public:
const Regex html_regex;
const Regex uri_parser_regex;
const boost::regex confluence_macro_re;
const boost::regex pipes_delimited_key_val_re;
const boost::regex semicolon_delimited_key_val_re;
const boost::regex asterisk_delimited_key_val_re;
const boost::regex comma_delimited_key_val_re;
const boost::regex ampersand_delimited_key_val_re;
#if 0 // Removed by Pavel's request. Leaving here in case he'll want to add this back...
const std::set<std::string> cookie_ignored_keywords;
const std::set<std::string> cookie_ignored_patterns;

1
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@ -59,7 +59,6 @@ WaapConfigAPI::clearAssetsCount()
void WaapConfigAPI::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
WaapConfigBase::load(ar);
assets_ids_aggregation.insert(m_assetId);
}

1
components/security_apps/waap/waap_clib/WaapConfigApplication.cc
View File

@ -74,7 +74,6 @@ void WaapConfigApplication::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
loadOpenRedirectPolicy(ar);
loadErrorDisclosurePolicy(ar);
loadCsrfPolicy(ar);

5
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@ -24,6 +24,8 @@
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
using boost::algorithm::to_lower_copy;
using namespace std;
@ -253,12 +255,12 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar)
}
const std::vector<std::string> &
WaapConfigBase::get_applicationUrls() const
{
return m_applicationUrls;
}
void WaapConfigBase::loadErrorDisclosurePolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the WAAP Information Disclosure policy";
@ -432,6 +434,7 @@ const std::shared_ptr<Waap::OpenRedirect::Policy>& WaapConfigBase::get_OpenRedir
}
const std::shared_ptr<Waap::ErrorDisclosure::Policy>& WaapConfigBase::get_ErrorDisclosurePolicy() const
{
return m_errorDisclosurePolicy;

1
components/security_apps/waap/waap_clib/WaapConfigBase.h
View File

@ -99,6 +99,7 @@ private:
std::vector<std::string> m_applicationUrls;
std::shared_ptr<Waap::ErrorDisclosure::Policy> m_errorDisclosurePolicy;
std::string m_schemaValidationPoicyStatusMessage;
std::string m_schemaUpdaterPoicyStatusMessage;
std::shared_ptr<Waap::Csrf::Policy> m_csrfPolicy;
std::shared_ptr<Waap::RateLimiting::Policy> m_rateLimitingPolicy;
std::shared_ptr<Waap::RateLimiting::Policy> m_errorLimitingPolicy;

14
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@ -31,7 +31,7 @@ bool Match::operator==(const Match &other) const
}
Behavior::Behavior()
: m_action(""), m_log(""), m_sourceIdentifier("")
: m_id(""), m_action(""), m_log(""), m_sourceIdentifier("")
{
}
@ -40,6 +40,11 @@ bool Behavior::operator==(const Behavior &other) const
return (m_action == other.m_action) && (m_log == other.m_log) && (m_sourceIdentifier == other.m_sourceIdentifier);
}
const std::string & Behavior::getParentId() const
{
return m_id;
}
const std::string & Behavior::getAction() const
{
return m_action;
@ -55,6 +60,11 @@ const std::string& Behavior::getSourceIdentifier() const
return m_sourceIdentifier;
}
void Behavior::setParentId(const std::string& id)
{
m_id = id;
}
bool Rule::operator==(const Rule &other) const
{
return (m_match == other.m_match) &&
@ -70,7 +80,9 @@ bool Policy::operator==(const Policy &other) const
State::State() :
bForceBlock(false),
forceBlockIds(),
bForceException(false),
forceExceptionIds(),
bIgnoreLog(false),
bSourceIdentifierOverride(false),
sSourceIdentifierMatch("")

18
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@ -175,10 +175,13 @@ public:
}
}
const std::string &getParentId() const;
const std::string &getAction() const;
const std::string &getLog() const;
const std::string &getSourceIdentifier() const;
void setParentId(const std::string& id);
private:
std::string m_id;
std::string m_action;
std::string m_log;
std::string m_sourceIdentifier;
@ -195,20 +198,20 @@ public:
}
catch (const cereal::Exception &e)
{
dbgTrace(D_WAAP_OVERRIDE) << "An override rule has no id.";
dbgDebug(D_WAAP_OVERRIDE) << "An override rule has no id.";
m_id.clear();
}
ar(cereal::make_nvp("parsedMatch", m_match));
ar(cereal::make_nvp("parsedBehavior", m_behaviors));
m_isChangingRequestData = false;
for (std::vector<Waap::Override::Behavior>::const_iterator it = m_behaviors.begin();
for (std::vector<Waap::Override::Behavior>::iterator it = m_behaviors.begin();
it != m_behaviors.end();
++it)
{
const Behavior& behavior = *it;
Behavior& behavior = *it;
behavior.setParentId(m_id);
if (!behavior.getSourceIdentifier().empty()) // this rule changes data in request itself
{
m_isChangingRequestData = true;
@ -223,8 +226,9 @@ public:
{
if (m_match.match(testFunctor)) {
// extend matchedBehaviors list with all behaviors on this rule
dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched. Adding " << m_behaviors.size() << " new behaviors:";
std::string overrideId = getId();
dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched id: " << overrideId <<
". Adding " << m_behaviors.size() << " new behaviors:";
if (!overrideId.empty()) {
matchedOverrideIds.insert(overrideId);
}
@ -308,8 +312,10 @@ private:
struct State {
// whether to force block regardless of stage2 response (and even if bSendRequest and/or bSendResponse are false)
bool bForceBlock;
std::set<std::string> forceBlockIds;
// exception (allow) was matched, so this request won't be blocked.
bool bForceException;
std::set<std::string> forceExceptionIds;
// overrides decision in case log should be ignored
bool bIgnoreLog;
// user identfier override to be applied
@ -335,10 +341,12 @@ struct State {
if (matchedBehavior.getAction() == "accept") {
dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceException due to override behavior.";
bForceException = true;
forceExceptionIds.insert(matchedBehavior.getParentId());
}
else if (matchedBehavior.getAction() == "reject") {
dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceBlock due to override behavior.";
bForceBlock = true;
forceBlockIds.insert(matchedBehavior.getParentId());
}
if (matchedBehavior.getLog() == "ignore")

135
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc
View File

@ -15,81 +15,86 @@
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
namespace Waap {
ResponseInspectReasons::ResponseInspectReasons()
:
openRedirect(false),
errorDisclosure(false),
errorLimiter(false),
rateLimiting(false),
collectResponseForLog(false),
applyOverride(false)
{
}
ResponseInspectReasons::ResponseInspectReasons()
:
openRedirect(false),
errorDisclosure(false),
errorLimiter(false),
rateLimiting(false),
collectResponseForLog(false),
applyOverride(false)
{
}
bool
ResponseInspectReasons::shouldInspect() const
{
dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" <<
" OpenRedirect=" << openRedirect <<
" ErrorDisclosure=" << errorDisclosure <<
" RateLimiting=" << rateLimiting <<
" ErrorLimiter=" << errorLimiter <<
" collectResponseForLog=" << collectResponseForLog <<
" applyOverride=" << applyOverride;
return openRedirect || errorDisclosure || rateLimiting || errorLimiter || collectResponseForLog || applyOverride;
}
bool
ResponseInspectReasons::shouldInspect() const
{
dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" <<
" OpenRedirect=" << openRedirect <<
" ErrorDisclosure=" << errorDisclosure <<
" RateLimiting=" << rateLimiting <<
" ErrorLimiter=" << errorLimiter <<
" collectResponseForLog=" << collectResponseForLog <<
" applyOverride=" << applyOverride;
void
ResponseInspectReasons::setOpenRedirect(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag;
openRedirect = flag;
}
return
openRedirect || errorDisclosure || rateLimiting || errorLimiter ||
collectResponseForLog || applyOverride;
}
void
ResponseInspectReasons::setErrorDisclosure(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag;
errorDisclosure = flag;
}
void
ResponseInspectReasons::setOpenRedirect(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag;
openRedirect = flag;
}
void
ResponseInspectReasons::setRateLimiting(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag;
rateLimiting = flag;
}
void
ResponseInspectReasons::setErrorLimiter(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag;
errorLimiter = flag;
}
void
ResponseInspectReasons::setErrorDisclosure(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag;
errorDisclosure = flag;
}
void
ResponseInspectReasons::setCollectResponseForLog(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog << " to " <<
flag;
collectResponseForLog = flag;
}
void
ResponseInspectReasons::setRateLimiting(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag;
rateLimiting = flag;
}
void
ResponseInspectReasons::setApplyOverride(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " <<
flag;
applyOverride = flag;
}
void
ResponseInspectReasons::setErrorLimiter(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag;
errorLimiter = flag;
}
bool
ResponseInspectReasons::getApplyOverride(void)
{
return applyOverride;
}
void
ResponseInspectReasons::setCollectResponseForLog(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog <<
" to " << flag;
collectResponseForLog = flag;
}
void
ResponseInspectReasons::setApplyOverride(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " <<
flag;
applyOverride = flag;
}
bool
ResponseInspectReasons::getApplyOverride(void)
{
return applyOverride;
}
}

1
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h
View File

@ -25,6 +25,7 @@ public:
void setErrorLimiter(bool flag);
void setCollectResponseForLog(bool flag);
void setApplyOverride(bool flag);
bool getApplyOverride(void);
private:
bool openRedirect;

14
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@ -17,8 +17,13 @@
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
#include <boost/algorithm/string.hpp>
USE_DEBUG_FLAG(D_WAAP_SCANNER);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
// id generated by xml parser for an entity attribute
const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f";
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName)
{
@ -117,7 +122,7 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
// Ignore scan results from specific fields on csp-report json in case those are not filtered by learning
bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp)
{
if (res.score < 8.0f && res.location == "body" && dp.getLastParser() == "jsonParser") {
if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
@ -169,11 +174,14 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
return m_transaction->reportScanResult(res);
}
int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) {
int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) {
Waf2ScanResult& res = m_lastScanResult;
DeepParser &dp = m_transaction->getDeepParser();
std::string key = std::string(k, k_len);
std::string value = std::string(v, v_len);
res.clear();
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: k='" << key <<
"' v='" << value << "'";
@ -266,7 +274,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
}
// Special value only matched when XML <!ENTITY> atribute is found.
if (v_len == 36) {
if (value == "08a80340-06d3-11ea-9f87-0242ac11000f" && !m_transaction->shouldIgnoreOverride(res)) {
if (value == Waap::Scanner::xmlEntityAttributeId && !m_transaction->shouldIgnoreOverride(res)) {
// Always return max score when <!ENTITY tag is encoutered during XML parsing.
res.score = 10.0;
res.unescaped_line = "<!ENTITY";

6
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@ -32,13 +32,17 @@ namespace Waap {
m_bIgnoreOverride(false)
{
}
bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp,
const std::string &location, const std::string &param_name, const std::string &key);
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) override;
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) override;
const std::string &getAntibotCookie() const { return m_antibotCookie; }
bool getIgnoreOverride() { return m_bIgnoreOverride; };
const Waf2ScanResult &getLastScanResult() const { return m_lastScanResult; }
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
bool shouldIgnoreOverride(const Waf2ScanResult &res);

13
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc
View File

@ -94,7 +94,8 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
canSplitSemicolon(true),
canSplitPipe(true),
hasSpace(false),
isUrlEncoded(false)
isUrlEncoded(false),
hasCharLess(false)
{
unsigned int zerosSeq[2] = {0};
bool lastNul = false; // whether last processed character was ASCII NUL
@ -139,6 +140,12 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
case '|':
hasCharPipe = true;
break;
case '<':
hasCharLess = true;
break;
case '\"':
hasDoubleQuote = true;
break;
}
if (isspace(ch)) {
@ -259,4 +266,8 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
textual +=(hasSpace ? "true" : "false");
textual.append("\nisUrlEncoded = ");
textual +=(isUrlEncoded ? "true" : "false");
textual.append("\nhasCharLess = ");
textual +=(hasCharLess ? "true" : "false");
textual.append("\nhasDoubleQuote = ");
textual +=(hasDoubleQuote ? "true" : "false");
}

2
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h
View File

@ -34,6 +34,8 @@ struct ValueStatsAnalyzer
bool canSplitPipe;
bool hasSpace;
bool isUrlEncoded;
bool hasCharLess;
bool hasDoubleQuote;
std::string textual;
};

75
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -55,6 +55,8 @@
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
using namespace ReportIS;
#define MAX_REQUEST_BODY_SIZE (2*1024)
@ -529,6 +531,7 @@ void Waf2Transaction::set_method(const char* method) {
m_methodStr = method;
}
bool Waf2Transaction::checkIsScanningRequired()
{
bool result = false;
@ -606,7 +609,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
bool firstPush = true;
// Parse URL
ParserRaw urlParser(m_deepParserReceiver, scanStage);
ParserRaw urlParser(m_deepParserReceiver, 0, scanStage);
// Scan the uri until '?' or ';' character found, whichever comes first (or until end of the uri string),
// Do not account for last character as valid separator
@ -654,7 +657,6 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
pushed = true;
std::string url(uri);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
// We found no '?' character so set p to NULL to prevent parameters scan below.
@ -677,7 +679,6 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
pushed = true;
std::string url(p, q-p);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
}
@ -704,7 +705,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
bool ignoreScore = m_ignoreScore;
m_ignoreScore = true;
m_deepParser.m_key.push(scanStage.c_str(), scanStage.size());
ParserDelimiter uriSegmentsParser(m_deepParserReceiver, '/', scanStage);
ParserDelimiter uriSegmentsParser(m_deepParserReceiver, 0, '/', scanStage);
std::string baseUriUnescaped(baseUri);
Waap::Util::decodePercentEncoding(baseUriUnescaped);
uriSegmentsParser.push(baseUriUnescaped.c_str(), baseUriUnescaped.length());
@ -749,7 +750,8 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = uriEnd - p;
ParserUrlEncode up(m_deepParserReceiver, paramSep, checkUrlEncoded(p, buff_len));
dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@ -796,7 +798,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len)
if (value_len > 0) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the cookie value";
m_deepParser.m_key.push("cookie", 6);
ParserUrlEncode cookieValueParser(m_deepParserReceiver, ';');
ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';');
cookieValueParser.push(value, value_len);
cookieValueParser.finish();
m_deepParser.m_key.pop("cookie");
@ -838,7 +840,7 @@ void Waf2Transaction::parseUnknownHeaderName(const char* name, int name_len)
!m_pWaapAssetState->getSignatures()->good_header_name_re.hasMatch(std::string(name, name_len))) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header name";
m_deepParser.m_key.push("header", 6);
ParserRaw headerNameParser(m_deepParserReceiver, std::string(name, name_len));
ParserRaw headerNameParser(m_deepParserReceiver, 0, std::string(name, name_len));
headerNameParser.push(name, name_len);
headerNameParser.finish();
m_deepParser.m_key.pop("header name");
@ -857,7 +859,7 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header value";
m_deepParser.m_key.push("header", 6);
ParserRaw headerValueParser(m_deepParserReceiver, headerName);
ParserRaw headerValueParser(m_deepParserReceiver, 0, headerName);
headerValueParser.push(value, value_len);
headerValueParser.finish();
m_deepParser.m_key.pop("header value");
@ -1077,7 +1079,7 @@ void Waf2Transaction::start_request_body() {
clearRequestParserState();
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, "body");
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, 0, "body");
m_request_body_bytes_received = 0;
m_request_body.clear();
@ -1188,6 +1190,7 @@ void Waf2Transaction::end_request() {
dbgTrace(D_WAAP) << "(Waf2Engine::end_request): Security Headers State was created";
}
// Enable response headers processing if response scanning is enabled in policy
auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL;
m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable);
@ -1642,6 +1645,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
std::vector<std::string> vOverrideIds(m_matchedOverrideIds.size());
std::copy(m_matchedOverrideIds.begin(), m_matchedOverrideIds.end(), vOverrideIds.begin());
waapLog.addToOrigin(LogField("exceptionIdList", vOverrideIds));
if (!m_effectiveOverrideIds.empty()) {
std::vector<std::string> vEffectiveOverrideIds(m_effectiveOverrideIds.size());
std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin());
waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds));
}
}
}
@ -1697,13 +1705,18 @@ Waf2Transaction::sendLog()
return;
}
dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size();
bool shouldBlock = false;
if (m_overrideState.bForceBlock) {
// If override forces "reject" decision, mention it in the "override" log field.
logOverride = OVERRIDE_DROP;
shouldBlock = true;
}
else if (m_overrideState.bForceException) {
} else if (m_overrideState.bForceException) {
// If override forces "allow" decision, mention it in the "override" log field.
logOverride = OVERRIDE_ACCEPT;
} else if (m_scanner.getIgnoreOverride()) {
@ -2024,6 +2037,9 @@ Waf2Transaction::decideAutonomousSecurity(
if (m_overrideState.bForceBlock) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces REJECT ...";
if (!decision->shouldBlock()) {
m_effectiveOverrideIds.insert(m_overrideState.forceBlockIds.begin(), m_overrideState.forceBlockIds.end());
}
decision->setBlock(true);
if (!m_overrideState.bIgnoreLog)
{
@ -2033,6 +2049,17 @@ Waf2Transaction::decideAutonomousSecurity(
else if (m_overrideState.bForceException) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces ALLOW ...";
if (m_scanResult) {
// on accept exception the decision is not set and needs to be calculated to determine effectivness
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat);
if (shouldBlock) {
m_effectiveOverrideIds.insert(
m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end()
);
}
}
decision->setBlock(false);
if (!m_overrideState.bIgnoreLog)
{
@ -2041,8 +2068,13 @@ Waf2Transaction::decideAutonomousSecurity(
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO) {
bool log_all = false;
const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = sitePolicy.get_TriggerPolicy();
if (triggerPolicy) {
const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
if (triggerLog && triggerLog->webRequests) log_all = true;
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) {
decision->setLog(false);
} else {
decision->setLog(true);
@ -2171,8 +2203,10 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) {
bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) return false;
if (!exceptions.ok()) {
dbgInfo(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
return false;
}
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
std::unordered_map<std::string, std::set<std::string>> exceptions_dict;
@ -2212,13 +2246,20 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// calling behavior and check if there is a behavior that match to this specific param name.
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (auto const &behavior : behaviors) {
if (behavior == action_ignore) {
for (const auto &behavior : behaviors) {
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
std::string overrideId = behavior.getId();
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);
}
if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId)
{
if (!overrideId.empty()) {
m_effectiveOverrideIds.insert(overrideId);
}
}
return true;
}
}

2
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -283,6 +283,7 @@ private:
// Matched override IDs
std::set<std::string> m_matchedOverrideIds;
std::set<std::string> m_effectiveOverrideIds;
//csrf state
Waap::CSRF::State m_csrfState;
@ -344,7 +345,6 @@ private:
// Cached pointer to const triggerLog (hence mutable)
mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
Waf2TransactionFlags m_waf2TransactionFlags;
// Grace period for logging

4
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@ -42,6 +42,7 @@ USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_EVASIONS);
USE_DEBUG_FLAG(D_WAAP_BASE64);
USE_DEBUG_FLAG(D_WAAP_JSON);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
#define MIN_HEX_LENGTH 6
#define charToDigit(c) (c - '0')
@ -1186,7 +1187,7 @@ static const SingleRegex base64_key_value_detector_re(
err,
"base64_key_value");
static const SingleRegex json_key_value_detector_re(
"^[^<>{};,&\\?|=\\s]+={.+(?s):.+(?s)}\\z",
"\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
err,
"json_key_value");
static const SingleRegex base64_key_detector_re(
@ -1444,6 +1445,7 @@ base64_variants b64Test (
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: FINAL key = '" << key << "'";
}
retVal = decodeBase64Chunk(s, start, s.end(), value);
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After testing and conversion value = "
<< value << "retVal = '" << retVal <<"'";
if (!retVal) {

Some files were not shown because too many files have changed in this diff Show More