github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2026-01-17 16:00:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jan 06 2026 dev (#387)

Browse Source 
* sync code

* update code to support brotli

* update code to support brotli

* update code to support brotli

* sync code

* fix findBrotli

* sync code

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>
Co-authored-by: Daniel Eisenberg <danielei@checkpoint.com>
This commit is contained in:
Daniel-Eisenberg
2026-01-13 17:17:52 +02:00
committed by GitHub
parent c1058db57d
commit e7b6e51b31
216 changed files with 12601 additions and 2825 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
components/security_apps/http_geo_filter/http_geo_filter.cc
View File

@@ -35,7 +35,8 @@ public:
init()
{
dbgTrace(D_GEO_FILTER) << "Init Http Geo filter component";
registerListener();
handleNewPolicy();
registerConfigLoadCb([this]() { handleNewPolicy(); });
}
void
@@ -44,6 +45,19 @@ public:
unregisterListener();
}
void
handleNewPolicy()
{
if (ParameterException::isGeoLocationExceptionExists()) {
registerListener();
return;
}
if (!ParameterException::isGeoLocationExceptionExists()) {
unregisterListener();
}
}
string getListenerName() const override { return "HTTP geo filter"; }
void
@@ -56,7 +70,7 @@ public:
<< "Load http geo filter default action. Action: "
<< default_action_maybe.unpack();
} else {
default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
dbgTrace(D_GEO_FILTER) << "No http geo filter default action. Action: Irrelevant";
}
}
@@ -66,7 +80,7 @@ public:
{
dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event";
if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
if (!event.isLastHeader()) return EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_INSPECT);
std::set<std::string> ip_set;
auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
auto maybe_xff = env->get<std::string>(HttpTransactionData::xff_vals_ctx);
@@ -101,15 +115,17 @@ public:
}
ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set);
if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
ServiceVerdict exception_verdict = getExceptionVerdict(ip_set);
if (exception_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) {
return EventVerdict(exception_verdict);
}
ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(ip_set);
if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
return EventVerdict(geo_lookup_verdict);
}
// deprecated for now
// ServiceVerdict geo_lookup_verdict = getGeoLookupVerdict(ip_set);
// if (geo_lookup_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) {
// return EventVerdict(geo_lookup_verdict);
// }
return EventVerdict(default_action);
}
@@ -146,7 +162,7 @@ private:
void
removeTrustedIpsFromXff(std::set<std::string> &xff_set)
{
auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
auto identify_config = getConfigurationWithCache<UsersAllIdentifiersConfig>(
"rulebase",
"usersIdentifiers"
);
@@ -189,33 +205,34 @@ private:
return os.str();
}
ngx_http_cp_verdict_e
convertActionToVerdict(const string &action) const
{
if (action == "accept") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
if (action == "drop") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
}
ngx_http_cp_verdict_e
ServiceVerdict
convertBehaviorValueToVerdict(const BehaviorValue &behavior_value) const
{
if (behavior_value == BehaviorValue::ACCEPT || behavior_value == BehaviorValue::IGNORE) {
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (behavior_value == BehaviorValue::DROP || behavior_value == BehaviorValue::REJECT) {
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
}
ngx_http_cp_verdict_e
// LCOV_EXCL_START Reason: deprecated for now
ServiceVerdict
convertActionToVerdict(const string &action) const
{
if (action == "accept") return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
if (action == "drop") return ServiceVerdict::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
ServiceVerdict
getGeoLookupVerdict(const std::set<std::string> &sources)
{
auto maybe_geo_config = getConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
if (!maybe_geo_config.ok()) {
dbgTrace(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
}
GeoConfig geo_config = maybe_geo_config.unpack();
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
@@ -252,12 +269,12 @@ private:
<< ", country code: "
<< country_code;
generateVerdictLog(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
ServiceVerdict::TRAFFIC_VERDICT_ACCEPT,
geo_config.getId(),
true,
geo_location_data
);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (geo_config.isBlockedCountry(country_code)) {
dbgTrace(D_GEO_FILTER)
@@ -266,12 +283,12 @@ private:
<< ", country code: "
<< country_code;
generateVerdictLog(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
ServiceVerdict::TRAFFIC_VERDICT_DROP,
geo_config.getId(),
true,
geo_location_data
);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
}
dbgTrace(D_GEO_FILTER)
@@ -286,22 +303,23 @@ private:
);
return convertActionToVerdict(geo_config.getDefaultAction());
}
// LCOV_EXCL_STOP
Maybe<pair<ngx_http_cp_verdict_e, string>>
Maybe<pair<ServiceVerdict, string>>
getBehaviorsVerdict(
const unordered_map<string, set<string>> &behaviors_map_to_search,
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data)
{
bool is_matched = false;
ParameterBehavior matched_behavior;
ngx_http_cp_verdict_e matched_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
ServiceVerdict matched_verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<HttpGeoFilter>();
set<ParameterBehavior> behaviors_set = i_rulebase->getBehavior(behaviors_map_to_search);
dbgTrace(D_GEO_FILTER) << "get verdict from: " << behaviors_set.size() << " behaviors";
for (const ParameterBehavior &behavior : behaviors_set) {
matched_verdict = convertBehaviorValueToVerdict(behavior.getValue());
if (
matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP
matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP
){
dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId();
generateVerdictLog(
@@ -310,10 +328,10 @@ private:
false,
geo_location_data
);
return pair<ngx_http_cp_verdict_e, string>(matched_verdict, behavior.getId());
return pair<ServiceVerdict, string>(matched_verdict, behavior.getId());
}
else if (
matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT
matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT
){
dbgTrace(D_GEO_FILTER) << "behavior verdict: ACCEPT, exception id: " << behavior.getId();
matched_behavior = behavior;
@@ -321,19 +339,19 @@ private:
}
}
if (is_matched) {
return pair<ngx_http_cp_verdict_e, string>(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
return pair<ServiceVerdict, string>(
ServiceVerdict::TRAFFIC_VERDICT_ACCEPT,
matched_behavior.getId()
);
}
return genError("No exception matched to HTTP geo filter rule");
}
ngx_http_cp_verdict_e
ServiceVerdict
getExceptionVerdict(const std::set<std::string> &sources) {
pair<ngx_http_cp_verdict_e, string> curr_matched_behavior;
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
pair<ServiceVerdict, string> curr_matched_behavior;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
@@ -389,7 +407,7 @@ private:
if (matched_behavior_maybe.ok()) {
curr_matched_behavior = matched_behavior_maybe.unpack();
verdict = curr_matched_behavior.first;
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
return verdict;
}
}
@@ -402,13 +420,13 @@ private:
if (matched_behavior_maybe.ok()) {
curr_matched_behavior = matched_behavior_maybe.unpack();
verdict = curr_matched_behavior.first;
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
return verdict;
}
}
}
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT) {
generateVerdictLog(
verdict,
curr_matched_behavior.second,
@@ -421,7 +439,7 @@ private:
void
generateVerdictLog(
const ngx_http_cp_verdict_e &verdict,
const ServiceVerdict &verdict,
const string &matched_id,
bool is_geo_filter,
const EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data,
@@ -430,7 +448,7 @@ private:
{
dbgTrace(D_GEO_FILTER) << "Generate Log for verdict - HTTP geo filter";
auto &trigger = getConfigurationWithDefault(default_triger, "rulebase", "log");
bool is_prevent = verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
bool is_prevent = verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP;
string matched_on = is_geo_filter ? "geoFilterPracticeId" : "exceptionId";
LogGen log = trigger(
"Web Request - HTTP Geo Filter",
@@ -469,7 +487,7 @@ private:
<< LogField("sourceCountryName", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME]);
}
ngx_http_cp_verdict_e default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
ServiceVerdict default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
};
HttpGeoFilter::HttpGeoFilter() : Component("HttpGeoFilter"), pimpl(make_unique<HttpGeoFilter::Impl>()) {}
@@ -491,6 +509,6 @@ void
HttpGeoFilter::preload()
{
registerExpectedConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
registerExpectedConfiguration<UsersAllIdentifiersConfig>("rulebase", "usersIdentifiers");
registerExpectedConfigurationWithCache<UsersAllIdentifiersConfig>("assetId", "rulebase", "usersIdentifiers");
registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); });
}

14
components/security_apps/ips/include/ips_signatures.h
View File

@@ -21,13 +21,13 @@
#include <vector>
#include "config.h"
#include "i_generic_rulebase.h"
#include "i_first_tier_agg.h"
#include "ips_entry.h"
#include "ips_enums.h"
#include "log_generator.h"
#include "parsed_context.h"
#include "pm_hook.h"
#include "i_generic_rulebase.h"
#define DEFAULT_IPS_YIELD_COUNT 500
@@ -402,6 +402,7 @@ public:
LogTriggerConf getTrigger() const;
std::set<ParameterBehavior>
getBehavior(const std::unordered_map<std::string, std::set<std::string>> &exceptions_dict) const;
@@ -410,10 +411,21 @@ private:
/// \param ips_state The IPS entry.
ActionResults getAction(const IPSEntry &ips_state) const;
void sendLog(
const Buffer &context_buffer,
const IPSEntry &ips_state,
const std::tuple<
IPSSignatureSubTypes::SignatureAction,
std::string, std::vector<std::string>
> &override_action,
bool is_prevent
) const;
std::shared_ptr<CompleteSignature> signature;
SignatureAction action;
std::string trigger_id;
std::string exception_id;
mutable bool bSupressLog = false;
};
} // namespace IPSSignatureSubTypes

12
components/security_apps/ips/ips_comp.cc
View File

@@ -13,7 +13,7 @@
#include "virtual_modifiers.h"
#include "helper.h"
#include "ips_common_types.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
using namespace std;
@@ -46,9 +46,9 @@ class IPSComp::Impl
public Listener<HttpResponseBodyEvent>,
public Listener<EndTransactionEvent>
{
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
class SigsFirstTierAgg
{
@@ -383,8 +383,8 @@ IPSComp::preload()
registerExpectedResource<SnortSignaturesResource>("IPSSnortSigs", "protections");
registerExpectedConfiguration<IPSConfiguration>("IPS", "IpsConfigurations");
registerExpectedConfiguration<uint>("IPS", "Max Field Size");
registerExpectedConfiguration<IPSSignatures>("IPS", "IpsProtections");
registerExpectedConfiguration<SnortSignatures>("IPSSnortSigs", "SnortProtections");
registerExpectedConfigurationWithCache<IPSSignatures>("assetId", "IPS", "IpsProtections");
registerExpectedConfigurationWithCache<SnortSignatures>("assetId", "IPSSnortSigs", "SnortProtections");
registerExpectedConfigFile("ips", Config::ConfigFileType::Policy);
registerExpectedConfigFile("ips", Config::ConfigFileType::Data);
registerExpectedConfigFile("snort", Config::ConfigFileType::Policy);

34
components/security_apps/ips/ips_signatures.cc
View File

@@ -321,6 +321,9 @@ SignatureAndAction::getAction(const IPSEntry &ips_state) const
override_actions.insert(behavior.getValue());
const string &override_id = behavior.getId();
if (!override_id.empty()) override_ids.push_back(override_id);
} else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) {
dbgTrace(D_IPS) << "setting bSupressLog due to override behavior: " << behavior.getId();
bSupressLog = true;
}
}
@@ -358,8 +361,10 @@ set<ParameterBehavior>
SignatureAndAction::getBehavior(const unordered_map<string, set<string>> &exceptions_dict) const
{
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<IPSComp>();
if (exception_id.empty()) return i_rulebase->getBehavior(exceptions_dict);
if (exception_id.empty()) {
dbgTrace(D_RULEBASE_CONFIG) << "No exception id provided, using default behavior";
return i_rulebase->getBehavior(exceptions_dict);
}
return i_rulebase->getParameterException(exception_id).getBehavior(exceptions_dict);
}
@@ -453,10 +458,25 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
return false;
}
dbgDebug(D_IPS) << "Signature matched - sending log";
bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT;
if(bSupressLog) {
dbgTrace(D_IPS) << "Signature matched - not sending log due to exception behavior";
} else {
sendLog(context_buffer, ips_state, override_action, is_prevent);
}
return is_prevent;
}
void SignatureAndAction::sendLog(
const Buffer &context_buffer,
const IPSEntry &ips_state,
const tuple<IPSSignatureSubTypes::SignatureAction, string, vector<string>> &override_action,
bool is_prevent
) const
{
dbgFlow(D_IPS) << "Signature matched - sending log";
auto trigger = getTrigger();
bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT;
auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL;
if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::DETECT) severity = Severity::INFO;
@@ -560,7 +580,7 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
uint res_size = res_body.ok() && trigger.isWebLogFieldActive(::res_body) ? res_body.unpack().size() : 0;
if (req_size + res_size > max_size) {
if (req_size + 500 > max_size) {
res_size = std::min(500u, res_size);
res_size = min(500u, res_size);
req_size = max_size - res_size;
} else {
res_size = max_size - req_size;
@@ -572,10 +592,8 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
log << LogField("waapOverride", get<1>(override_action));
if (!get<2>(override_action).empty()) log.addToOrigin(LogField("exceptionIdList", get<2>(override_action)));
log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
return is_prevent;
log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
}
void

6
components/security_apps/ips/ips_ut/component_ut.cc
View File

@@ -109,11 +109,11 @@ operator ==(const EventVerdict &first, const EventVerdict &second)
return first.getVerdict() == second.getVerdict();
}
const EventVerdict ComponentTest::inspect(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
const EventVerdict ComponentTest::inspect(ServiceVerdict::TRAFFIC_VERDICT_INSPECT);
const EventVerdict ComponentTest::accept(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT);
const EventVerdict ComponentTest::accept(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT);
const EventVerdict ComponentTest::drop(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
const EventVerdict ComponentTest::drop(ServiceVerdict::TRAFFIC_VERDICT_DROP);
TEST_F(ComponentTest, check_init_fini_do_not_crush)
{

115
components/security_apps/ips/ips_ut/signatures_ut.cc
View File

@@ -19,6 +19,7 @@
#include "generic_rulebase/generic_rulebase_context.h"
#include "encryptor.h"
#include "mock/mock_table.h"
USE_DEBUG_FLAG(D_IPS);
using namespace testing;
using namespace std;
@@ -194,6 +195,101 @@ public:
gen_ctx->activate();
}
// Loads an exception with log: ignore behavior, as in the provided JSON
void loadExceptionsIgnoreLog()
{
BasicRuleConfig::preload();
registerExpectedConfiguration<ParameterException>("rulebase", "exception");
string test_config(
"{"
" \"rulebase\": {"
" \"rulesConfig\": ["
" {"
" \"context\": \"All()\","
" \"priority\": 1,"
" \"ruleId\": \"5eaef0726765c30010bae8bb\","
" \"ruleName\": \"Acme web API\","
" \"assetId\": \"5e243effd858007660b758ad\","
" \"assetName\": \"Acme Power API\","
" \"parameters\": ["
" {"
" \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\","
" \"parameterType\": \"exceptions\","
" \"parameterName\": \"exception\""
" }"
" ],"
" \"zoneId\": \"\","
" \"zoneName\": \"\""
" }"
" ],"
" \"exception\": ["
" {"
" \"context\": \"Any(parameterId(6c3867be-4da5-42c2-93dc-8f509a764003))\","
" \"exceptions\": ["
" {"
" \"match\": {"
" \"type\": \"operator\","
" \"op\": \"and\","
" \"items\": ["
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"sourceIdentifier\","
" \"value\": [\"1.1.1.1\"]"
" }"
" ]"
" },"
" \"behavior\": {"
" \"key\": \"log\","
" \"value\": \"ignore\""
" }"
" },"
" {"
" \"match\": {"
" \"type\": \"operator\","
" \"op\": \"or\","
" \"items\": ["
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"protectionName\","
" \"value\": [\"Test1\"]"
" },"
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"protectionName\","
" \"value\": [\"Test2\"]"
" },"
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"sourceIdentifier\","
" \"value\": [\"1.1.1.1\"]"
" }"
" ]"
" },"
" \"behavior\": {"
" \"key\": \"action\","
" \"value\": \"accept\""
" }"
" }"
" ]"
" }"
" ]"
" }"
"}"
);
istringstream ss(test_config);
auto i_config = Singleton::Consume<Config::I_Config>::from(config);
i_config->loadConfiguration(ss);
gen_ctx = make_unique<GenericRulebaseContext>();
gen_ctx->activate();
}
void
load(const IPSSignaturesResource &policy, const string &severity, const string &confidence)
{
@@ -261,6 +357,7 @@ public:
IPSSignaturesResource performance_signatures3;
IPSSignaturesResource single_broken_signature;
NiceMock<MockTable> table;
NiceMock<MockLogging> logs;
MockAgg mock_agg;
private:
@@ -273,7 +370,6 @@ private:
ConfigComponent config;
Encryptor encryptor;
AgentDetails details;
StrictMock<MockLogging> logs;
IPSEntry ips_state;
string signature1 =
@@ -524,6 +620,23 @@ TEST_F(SignatureTest, basic_load_of_signatures)
EXPECT_FALSE(sigs.isEmpty("HTTP_REQUEST_BODY"));
}
TEST_F(SignatureTest, ignore_exception_suppresses_log)
{
load(single_signature2, "Low or above", "Low");
loadExceptionsIgnoreLog();
expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\"");
EXPECT_TRUE(checkData("gggddd"));
ScopedContext ctx;
ctx.registerValue<string>("sourceIdentifiers", "1.1.1.1");
// No log should be sent when the exception matches
EXPECT_CALL(logs, sendLog(_)).Times(0);
EXPECT_FALSE(checkData("gggddd"));
}
TEST_F(SignatureTest, single_signature_matching_override)
{
load(single_signature, "Low or above", "Low");

18
components/security_apps/layer_7_access_control/layer_7_access_control.cc
View File

@@ -7,7 +7,7 @@
#include "config.h"
#include "cache.h"
#include "http_inspection_events.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "log_generator.h"
@@ -117,12 +117,12 @@ public:
if (!isAppEnabled()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (!event.isLastHeader()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Inspect verdict";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
return handleEvent();
@@ -133,7 +133,7 @@ public:
{
if (!isAppEnabled()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Handling wait verdict";
@@ -217,13 +217,13 @@ Layer7AccessControl::Impl::queryIpReputation(const string &source_ip)
if (!ip_reputation.ok()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Scheduling Intelligence query - returning Wait verdict";
scheduleIntelligenceQuery(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
return ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
}
if (!ip_reputation.unpack().isMalicious()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip;
ip_reputation_cache.deleteEntry(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
return generateLog(source_ip, ip_reputation.unpack());
@@ -246,7 +246,7 @@ Layer7AccessControl::Impl::handleEvent()
}
dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
void
@@ -354,11 +354,11 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen
if (isPrevent()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
Maybe<LogField, Context::Error>

8
components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc
View File

@@ -49,10 +49,10 @@ public:
void registerTransactionData();
void verifyReport(const Report &report, const string &source_identifier, const string &security_action);
const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
const EventVerdict wait_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
const EventVerdict drop_verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
const EventVerdict accept_verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
const EventVerdict inspect_verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
const EventVerdict wait_verdict = ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
Layer7AccessControl l7_access_control;
::Environment env;
ConfigComponent config;

170
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@@ -68,6 +68,15 @@ checkSAMLPortal(const string &command_output)
return string("false");
}
Maybe<string>
checkIdaPDP(const string &command_output)
{
if (command_output.find("is_collecting_identities (true)") != string::npos) {
return string("true");
}
return string("false");
}
Maybe<string>
checkInfinityIdentityEnabled(const string &command_output)
{
@@ -139,6 +148,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
return command_output;
}
Maybe<string>
checkIsCME(const string &command_output)
{
if (command_output == "" ) return string("false");
return command_output;
}
Maybe<string>
getOtlpAgentGaiaOsRole(const string &command_output)
{
@@ -147,6 +164,129 @@ getOtlpAgentGaiaOsRole(const string &command_output)
return command_output;
}
// Helper function for case-insensitive substring search
inline bool
containsIgnoreCase(const string& text, const string& pattern) {
string lowerText = text;
string lowerPattern = pattern;
transform(lowerText.begin(), lowerText.end(), lowerText.begin(), ::tolower);
transform(lowerPattern.begin(), lowerPattern.end(), lowerPattern.begin(), ::tolower);
return lowerText.find(lowerPattern) != string::npos;
}
inline Maybe<string>
extractValue(const string& line, const string& field) {
size_t colonPos = line.find(':');
if (colonPos == string::npos) {
return Maybe<string>(Error<string>("no match"));
}
string key = line.substr(0, colonPos);
string value = line.substr(colonPos + 1);
key.erase(0, key.find_first_not_of(" \t"));
key.erase(key.find_last_not_of(" \t") + 1);
value.erase(0, value.find_first_not_of(" \t"));
value.erase(value.find_last_not_of(" \t") + 1);
if (containsIgnoreCase(key, field)) {
return Maybe<string>(value);
}
return Maybe<string>(Error<string>("no match"));
}
inline std::pair<std::string, std::string>
parseDmidecodeOutput(const std::string& dmidecodeOutput) {
string manufacturer;
string product;
istringstream stream(dmidecodeOutput);
string line;
while (getline(stream, line)) {
if (manufacturer.empty()) {
auto extractedManufacturer = extractValue(line, "Manufacturer");
if (extractedManufacturer.ok() && !extractedManufacturer->empty()) {
manufacturer = *extractedManufacturer;
}
}
if (product.empty()) {
auto extractedProduct = extractValue(line, "Product Name");
if (extractedProduct.ok() && !extractedProduct->empty()) {
product = *extractedProduct;
}
}
if (!manufacturer.empty() && !product.empty()) {
break;
}
}
return make_pair(manufacturer, product);
}
Maybe<string>
getAiopCgnsHardwareType(const string &command_output)
{
if (command_output == "" ) return string("NA");
auto pair = parseDmidecodeOutput(command_output);
if (containsIgnoreCase(pair.first, "Amazon")) {
return string("AWS");
}
if (containsIgnoreCase(pair.first, "Microsoft")) {
return string("Azure");
}
if (containsIgnoreCase(pair.first, "Google")) {
return string("Google Cloud");
}
if (containsIgnoreCase(pair.first, "Oracle")) {
return string("OCI");
}
if (containsIgnoreCase(pair.first, "Alibaba")) {
return string("Alibaba");
}
if (containsIgnoreCase(pair.second, "VMware")) {
return string("VMware");
}
if (containsIgnoreCase(pair.first, "OpenStack")) {
return string("OpenStack");
}
// Check for KVM (manufacturer OR product)
if (containsIgnoreCase(pair.first, "QEMU") || containsIgnoreCase(pair.second, "KVM")) {
return string("KVM");
}
if (containsIgnoreCase(pair.first, "Xen")) {
return string("Xen");
}
if (containsIgnoreCase(pair.first, "Nutanix")) {
return string("Nutanix");
}
return string("NA");
}
Maybe<string>
getAiopsCgnsCloudVendor(const string &command_output)
{
if (command_output == "" ) return string("NA");
string platform = "NA";
istringstream stream(command_output);
string line;
while (getline(stream, line)) {
if (line.find("platform") != string::npos) {
size_t colonPos = line.find(' ');
if (colonPos != string::npos) {
platform = line.substr(colonPos + 1);
platform.erase(0, platform.find_first_not_of(" \t"));
platform.erase(platform.find_last_not_of(" \t") + 1);
break;
}
}
}
return platform;
}
Maybe<string>
getQUID(const string &command_output)
{
@@ -158,11 +298,24 @@ getQUID(const string &command_output)
return command_output;
}
// Handler for a comma-separated list of QUIDs
Maybe<string>
getIsAiopsRunning(const string &command_output)
getQUIDList(const string &command_output)
{
if (command_output == "" ) return string("false");
if (command_output.empty()) {
return string("false");
}
std::istringstream ss(command_output);
std::string quid;
while (std::getline(ss, quid, ',')) {
const auto res = getQUID(quid);
if (!res.ok()) {
return res; // Return the error directly with context from getQUID
}
}
return command_output;
}
@@ -349,6 +502,15 @@ getGWIPAddress(const string &command_output)
return getAttr(command_output, "IP Address was not found");
}
Maybe<string>
getGWIPv6Address(const string &command_output)
{
if (command_output.empty() || command_output == "null") {
return genError("IPv6 Address was not found");
}
return string(command_output);
}
Maybe<string>
getGWVersion(const string &command_output)
{
@@ -366,7 +528,7 @@ checkIfSdwanRunning(const string &command_output)
Maybe<string>
getClusterObjectIP(const string &command_output)
{
return getAttr(command_output, "Cluster object IP was not found");
return command_output;
}
Maybe<string>

75
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@@ -46,27 +46,33 @@ SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
"FS_PATH=<FILESYSTEM-PREFIX>; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] "
"&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
checkIsInstallHorizonTelemetrySucceeded)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"${PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
SHELL_CMD_HANDLER("isCME", "[ -d /opt/CPcme ] && echo 'true' || echo 'false'", checkIsCME)
#endif
#if defined(gaia)
SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''",
getQUID)
SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
"VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
"[ -z \"${VS_ID}\" ] && "
"(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');"
"[ -n \"${VS_ID}\" ] && "
"(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json"
" > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); "
"[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && "
"(python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
"IS_MDS=$(cpprod_util CPPROD_IsConfigured PROVIDER-1 2>/dev/null | tr -d ' ');"
"if [ \"${IS_MDS}\" = \"1\" ]; then "
"DOMAIN_NAME=$(echo \"${FS_PATH}\" | grep -o -E \"domain-[^/]+\" | sed 's|domain-||');"
"[ -z \"${DOMAIN_NAME}\" ] && echo '' && exit 0;"
"sed \"s|###DOMAIN_NAME###|${DOMAIN_NAME}|g\" /opt/CPotelcol/quid_api/get_mds_quid.json"
" > /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME};"
"[ -f /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} ] && "
"python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} 2>/dev/null | jq -r .message[0].MDS_QUID || echo '';"
"else "
"VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
"[ -z \"${VS_ID}\" ] && "
"(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');"
"[ -n \"${VS_ID}\" ] && "
"(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json"
" > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); "
"[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && "
"(python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');"
"fi",
getQUID)
SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
@@ -76,9 +82,21 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("MHO_QUID",
"[ -d /opt/CPquid ] && "
"python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_mho_quid.json 2>/dev/null | "
"jq -r '[.message[]? | select(.MHO_QUID != \"\") | .MHO_QUID] | join(\",\")' 2>/dev/null || "
"echo ''",
getQUIDList)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
"&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
getOtlpAgentGaiaOsRole)
SHELL_CMD_HANDLER("AIOPS_CGNS_HW_TYPE", ""
"command -v dmidecode &>/dev/null && dmidecode -t 1 2>/dev/null",
getAiopCgnsHardwareType)
SHELL_CMD_HANDLER("AIOPS_CGNS_CLOUD_VENDOR",
"cat /etc/cloud-version 2>/dev/null",
getAiopsCgnsCloudVendor)
SHELL_CMD_HANDLER("ETH_MGMT_IP",
"FS_PATH=<FILESYSTEM-PREFIX>;"
"VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
@@ -104,7 +122,12 @@ SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole)
#endif
#if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"hasSDWan",
"[ $(cpprod_util CPPROD_IsMgmtMachine) -eq 1 ] && echo '0' ||"
"([ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0')",
checkHasSDWan
)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
"jq -r .can_update_sdwan_data /tmp/cpsdwan_getdata_orch.json",
@@ -131,10 +154,11 @@ SHELL_CMD_HANDLER(
)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectIP",
"obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";"
"[ $(cpprod_util FwIsHighAvail) -eq 1 ] && "
"(obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";"
" awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")"
" { gsub(/[()]/, \"\", $2); print $2; exit; } }'"
" $FWDIR/state/local/FW1/local.gateway_cluster",
" $FWDIR/state/local/FW1/local.gateway_cluster) || echo \"\"",
getClusterObjectIP
)
SHELL_CMD_HANDLER(
@@ -146,7 +170,16 @@ SHELL_CMD_HANDLER("is_legacy_qos_blade_enabled",
"cpprod_util CPPROD_GetValue FG1 ProdActive 1 | grep -q '^1$' "
"&& (cpprod_util CPPROD_GetValue FG1 FgSDWAN 1 | grep -q '^1$' && echo false || echo true) || "
"echo false",
checkQosLegacyBlade)
checkQosLegacyBlade
)
SHELL_CMD_HANDLER(
"IPv6 Address",
"( [ $(cpprod_util FwIsHighAvail) -eq 1 ] && [ $(cpprod_util FwIsVSX) -eq 1 ]"
"&& (jq -r .cluster_main_ipv6 /tmp/cpsdwan_getdata_orch.json) )"
"|| ( [ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" )"
"|| (jq -r .main_ipv6 /tmp/cpsdwan_getdata_orch.json)",
getGWIPv6Address
)
#endif //gaia || smb
#if defined(gaia)
@@ -154,6 +187,10 @@ SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedB
SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade)
SHELL_CMD_HANDLER("hasVPNBlade", "enabled_blades", checkVPNBlade)
SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal)
SHELL_CMD_HANDLER("hasIdaPdpEnabled",
"cat $FWDIR/database/myself_objects.C | grep is_collecting_identities",
checkIdaPDP
)
SHELL_CMD_HANDLER("hasInfinityIdentityEnabled",
"cat $FWDIR/database/myself_objects.C | grep get_identities_from_infinity_identity",
checkInfinityIdentityEnabled

41
components/security_apps/orchestration/health_check/health_check.cc
View File

@@ -154,7 +154,8 @@ private:
static const map<string, pair<string, int>> ip_port_defaults_map = {
{"Azure", make_pair(getenv("DOCKER_RPM_ENABLED") ? "" : "168.63.129.16", 8117)},
{"Aws", make_pair("", 8117)},
{"Local", make_pair("", 8117)}
{"Local", make_pair("", 8117)},
{"VMware", make_pair("", 8117)}
};
auto cloud_vendor_maybe = getSetting<string>("reverseProxy", "cloudVendorName");
@@ -271,6 +272,12 @@ private:
return HealthCheckStatus::UNHEALTHY;
}
if (checkReadinessFilesExist()) {
dbgTrace(D_HEALTH_CHECK)
<< "Readiness file exists, instance not ready for traffic, returning unhealthy status";
return HealthCheckStatus::UNHEALTHY;
}
if (NGEN::Filesystem::exists(rpm_full_load_path)) {
dbgTrace(D_HEALTH_CHECK) << "RPM is fully loaded";
return i_service_controller->getServicesPolicyStatus()
@@ -289,6 +296,24 @@ private:
return HealthCheckStatus::UNHEALTHY;
}
bool
checkReadinessFilesExist()
{
string readiness_dir = readiness_file_path.substr(0, readiness_file_path.find_last_of('/'));
string readiness_filename = NGEN::Filesystem::getFileName(readiness_file_path);
auto directory_files = NGEN::Filesystem::getDirectoryFiles(readiness_dir);
if (!directory_files.ok()) return false;
for (const string& filename : directory_files.unpack()) {
if (NGEN::Strings::startsWith(filename, readiness_filename)) {
return true;
}
}
return false;
}
bool
nginxContainerIsRunning()
{
@@ -304,7 +329,19 @@ private:
return false;
}
return (*maybe_result).find(nginx_container_name) != string::npos;
bool container_running = (*maybe_result).find(nginx_container_name) != string::npos;
if (!container_running) {
dbgTrace(D_HEALTH_CHECK) << "Nginx container is not running";
return false;
}
if (checkReadinessFilesExist()) {
dbgTrace(D_HEALTH_CHECK) << "Readiness file exists on host machine, not ready for traffic";
return false;
}
dbgTrace(D_HEALTH_CHECK) << "Nginx container is running and no readiness files found - ready for traffic";
return true;
}
void

6
components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc
View File

@@ -158,6 +158,12 @@ ManifestDiffCalculator::buildInstallationQueue(
installation_queue.push_back(orchestration_it->second);
}
auto shared_libs_it = new_packages.find("sharedLibs");
if (shared_libs_it != new_packages.end()) {
installation_queue.push_back(shared_libs_it->second);
}
auto wlp_standalone_it = new_packages.find("wlpStandalone");
if (wlp_standalone_it != new_packages.end()){
installation_queue.push_back(wlp_standalone_it->second);

7
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -1535,6 +1535,11 @@ private:
if (i_details_resolver->compareCheckpointVersion(8200, greater_equal<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER82", "true");
}
if (i_details_resolver->compareCheckpointVersion(8200, equal_to<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "true");
} else {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "false");
}
#endif // gaia || smb
if (agent_data_report == curr_agent_data_report) {
@@ -2278,4 +2283,4 @@ OrchestrationComp::preload()
registerExpectedSetting<uint>("successUpgradeInterval");
registerExpectedConfigFile("orchestration", Config::ConfigFileType::Policy);
registerExpectedConfigFile("registration-data", Config::ConfigFileType::Policy);
}
}

23
components/security_apps/prometheus/prometheus_comp.cc
View File

@@ -40,7 +40,13 @@ struct ServiceData
class PrometheusMetricData
{
public:
PrometheusMetricData(const string &n, const string &t, const string &d) : name(n), type(t), description(d) {}
PrometheusMetricData(const string &n, const string &u, const string &t, const string &d)
:
name(n),
unique_name(u),
type(t),
description(d)
{}
void
addElement(const string &labels, const string &value)
@@ -55,7 +61,9 @@ public:
string representative_name = "";
if (!name.empty()) {
auto metric_name = convertMetricName(name);
string metric_name;
if (!unique_name.empty()) metric_name = convertMetricName(unique_name);
if (metric_name.empty()) metric_name = convertMetricName(name);
!metric_name.empty() ? representative_name = metric_name : representative_name = name;
}
@@ -73,6 +81,7 @@ public:
private:
string name;
string unique_name;
string type;
string description;
map<string, string> metric_labels_to_values;
@@ -98,6 +107,7 @@ public:
for(auto &metric : metrics) {
auto &metric_object = getDataObject(
metric.name,
metric.unique_name,
metric.type,
metric.description
);
@@ -107,11 +117,14 @@ public:
private:
PrometheusMetricData &
getDataObject(const string &name, const string &type, const string &description)
getDataObject(const string &name, const string &unique_name, const string &type, const string &description)
{
auto elem = prometheus_metrics.find(name);
auto elem = prometheus_metrics.find(unique_name);
if (elem == prometheus_metrics.end()) {
elem = prometheus_metrics.emplace(name, PrometheusMetricData(name, type, description)).first;
elem = prometheus_metrics.emplace(
unique_name,
PrometheusMetricData(name, unique_name, type, description)
).first;
}
return elem->second;

58
components/security_apps/prometheus/prometheus_metric_names.h
View File

@@ -71,37 +71,37 @@ convertMetricName(const std::string &original_metric_name)
{"responseBodySizeUponTimeoutMaxSample", "response_body_size_max"},
{"responseBodySizeUponTimeoutMinSample", "response_body_size_min"},
// WaapTelemetrics
{"reservedNgenA", "total_requests_counter"},
{"reservedNgenB", "unique_sources_counter"},
{"reservedNgenC", "requests_blocked_by_force_and_exception_counter"},
{"reservedNgenD", "requests_blocked_by_waf_counter"},
{"reservedNgenE", "requests_blocked_by_open_api_counter"},
{"reservedNgenF", "requests_blocked_by_bot_protection_counter"},
{"reservedNgenG", "requests_threat_level_info_and_no_threat_counter"},
{"reservedNgenH", "requests_threat_level_low_counter"},
{"reservedNgenI", "requests_threat_level_medium_counter"},
{"reservedNgenJ", "requests_threat_level_high_counter"},
{"reservedNgenA_WAAP telemetry", "total_requests_counter"},
{"reservedNgenB_WAAP telemetry", "unique_sources_counter"},
{"reservedNgenC_WAAP telemetry", "requests_blocked_by_force_and_exception_counter"},
{"reservedNgenD_WAAP telemetry", "requests_blocked_by_waf_counter"},
{"reservedNgenE_WAAP telemetry", "requests_blocked_by_open_api_counter"},
{"reservedNgenF_WAAP telemetry", "requests_blocked_by_bot_protection_counter"},
{"reservedNgenG_WAAP telemetry", "requests_threat_level_info_and_no_threat_counter"},
{"reservedNgenH_WAAP telemetry", "requests_threat_level_low_counter"},
{"reservedNgenI_WAAP telemetry", "requests_threat_level_medium_counter"},
{"reservedNgenJ_WAAP telemetry", "requests_threat_level_high_counter"},
// WaapTrafficTelemetrics
{"reservedNgenA", "post_requests_counter"},
{"reservedNgenB", "get_requests_counter"},
{"reservedNgenC", "put_requests_counter"},
{"reservedNgenD", "patch_requests_counter"},
{"reservedNgenE", "delete_requests_counter"},
{"reservedNgenF", "other_requests_counter"},
{"reservedNgenG", "2xx_status_code_responses_counter"},
{"reservedNgenH", "4xx_status_code_responses_counter"},
{"reservedNgenI", "5xx_status_code_responses_counter"},
{"reservedNgenJ", "requests_time_latency_average"},
{"reservedNgenA_WAAP traffic telemetry", "post_requests_counter"},
{"reservedNgenB_WAAP traffic telemetry", "get_requests_counter"},
{"reservedNgenC_WAAP traffic telemetry", "put_requests_counter"},
{"reservedNgenD_WAAP traffic telemetry", "patch_requests_counter"},
{"reservedNgenE_WAAP traffic telemetry", "delete_requests_counter"},
{"reservedNgenF_WAAP traffic telemetry", "other_requests_counter"},
{"reservedNgenG_WAAP traffic telemetry", "status_code_2xx_responses_counter"},
{"reservedNgenH_WAAP traffic telemetry", "status_code_4xx_responses_counter"},
{"reservedNgenI_WAAP traffic telemetry", "status_code_5xx_responses_counter"},
{"reservedNgenJ_WAAP traffic telemetry", "requests_time_latency_average"},
// WaapAttackTypesMetrics
{"reservedNgenA", "sql_injection_attacks_type_counter"},
{"reservedNgenB", "vulnerability_scanning_attacks_type_counter"},
{"reservedNgenC", "path_traversal_attacks_type_counter"},
{"reservedNgenD", "ldap_injection_attacks_type_counter"},
{"reservedNgenE", "evasion_techniques_attacks_type_counter"},
{"reservedNgenF", "remote_code_execution_attacks_type_counter"},
{"reservedNgenG", "xml_extern_entity_attacks_type_counter"},
{"reservedNgenH", "cross_site_scripting_attacks_type_counter"},
{"reservedNgenI", "general_attacks_type_counter"},
{"reservedNgenA_WAAP attack type telemetry", "sql_injection_attacks_type_counter"},
{"reservedNgenB_WAAP attack type telemetry", "vulnerability_scanning_attacks_type_counter"},
{"reservedNgenC_WAAP attack type telemetry", "path_traversal_attacks_type_counter"},
{"reservedNgenD_WAAP attack type telemetry", "ldap_injection_attacks_type_counter"},
{"reservedNgenE_WAAP attack type telemetry", "evasion_techniques_attacks_type_counter"},
{"reservedNgenF_WAAP attack type telemetry", "remote_code_execution_attacks_type_counter"},
{"reservedNgenG_WAAP attack type telemetry", "xml_extern_entity_attacks_type_counter"},
{"reservedNgenH_WAAP attack type telemetry", "cross_site_scripting_attacks_type_counter"},
{"reservedNgenI_WAAP attack type telemetry", "general_attacks_type_counter"},
// AssetsMetric
{"numberOfProtectedApiAssetsSample", "api_assets_counter"},
{"numberOfProtectedWebAppAssetsSample", "web_api_assets_counter"},

212
components/security_apps/prometheus/prometheus_ut/prometheus_ut.cc
View File

@@ -1,79 +1,133 @@
#include "prometheus_comp.h"
#include <sstream>
#include <fstream>
#include <vector>
#include "cmock.h"
#include "cptest.h"
#include "maybe_res.h"
#include "debug.h"
#include "config.h"
#include "environment.h"
#include "config_component.h"
#include "agent_details.h"
#include "time_proxy.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_messaging.h"
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_PROMETHEUS);
class PrometheusCompTest : public Test
{
public:
PrometheusCompTest()
{
EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false));
env.preload();
config.preload();
env.init();
EXPECT_CALL(
mock_rest,
addGetCall("metrics", _)
).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
prometheus_comp.init();
}
::Environment env;
ConfigComponent config;
PrometheusComp prometheus_comp;
StrictMock<MockRestApi> mock_rest;
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockMessaging> mock_messaging;
unique_ptr<ServerRest> agent_uninstall;
function<string()> get_metrics_func;
CPTestTempfile status_file;
string registered_services_file_path;
};
TEST_F(PrometheusCompTest, checkAddingMetric)
{
registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json"));
setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services");
string metric_body = "{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"watchdogProcessStartupEventsSum\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n"
" \"value\": \"1534\"\n"
" }\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _))
.Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body)));
string metric_str = "# TYPE nano_service_restarts_counter counter\n"
"nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n";
EXPECT_EQ(metric_str, get_metrics_func());
}
#include "prometheus_comp.h"
#include <sstream>
#include <fstream>
#include <vector>
#include "cmock.h"
#include "cptest.h"
#include "maybe_res.h"
#include "debug.h"
#include "config.h"
#include "environment.h"
#include "config_component.h"
#include "agent_details.h"
#include "time_proxy.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_messaging.h"
using namespace std;
using namespace testing;
USE_DEBUG_FLAG(D_PROMETHEUS);
class PrometheusCompTest : public Test
{
public:
PrometheusCompTest()
{
EXPECT_CALL(mock_rest, mockRestCall(_, "declare-boolean-variable", _)).WillOnce(Return(false));
env.preload();
config.preload();
env.init();
EXPECT_CALL(
mock_rest,
addGetCall("metrics", _)
).WillOnce(DoAll(SaveArg<1>(&get_metrics_func), Return(true)));
prometheus_comp.init();
}
::Environment env;
ConfigComponent config;
PrometheusComp prometheus_comp;
StrictMock<MockRestApi> mock_rest;
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockMessaging> mock_messaging;
unique_ptr<ServerRest> agent_uninstall;
function<string()> get_metrics_func;
CPTestTempfile status_file;
string registered_services_file_path;
};
TEST_F(PrometheusCompTest, checkAddingMetricWithEmptyUniqueName)
{
registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json"));
setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services");
string metric_body = "{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"watchdogProcessStartupEventsSum\",\n"
" \"unique_name\": \"\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n"
" \"value\": \"1534\"\n"
" }\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _))
.Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body)));
string metric_str = "# TYPE nano_service_restarts_counter counter\n"
"nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n";
EXPECT_EQ(metric_str, get_metrics_func());
}
TEST_F(PrometheusCompTest, checkAddingMetricWithoutUniqueName)
{
registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json"));
setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services");
string metric_body = "{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"watchdogProcessStartupEventsSum\",\n"
" \"unique_name\": \"watchdogProcessStartupEventsSum_Bla bla\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n"
" \"value\": \"1534\"\n"
" }\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _))
.Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body)));
string metric_str = "# TYPE nano_service_restarts_counter counter\n"
"nano_service_restarts_counter{method=\"post\",code=\"200\"} 1534\n\n";
EXPECT_EQ(metric_str, get_metrics_func());
}
TEST_F(PrometheusCompTest, checkAddingMetricWithUniqueName)
{
registered_services_file_path = cptestFnameInSrcDir(string("registered_services.json"));
setConfiguration(registered_services_file_path, "orchestration", "Orchestration registered services");
string metric_body = "{\n"
" \"metrics\": [\n"
" {\n"
" \"metric_name\": \"reservedNgenA\",\n"
" \"unique_name\": \"reservedNgenA_WAAP telemetry\",\n"
" \"metric_type\": \"counter\",\n"
" \"metric_description\": \"\",\n"
" \"labels\": \"{method=\\\"post\\\",code=\\\"200\\\"}\",\n"
" \"value\": \"1534\"\n"
" }\n"
" ]\n"
"}";
string message_body;
EXPECT_CALL(mock_messaging, sendSyncMessage(_, "/service-metrics", _, _, _))
.Times(2).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, metric_body)));
string metric_str = "# TYPE total_requests_counter counter\n"
"total_requests_counter{method=\"post\",code=\"200\"} 1534\n\n";
EXPECT_EQ(metric_str, get_metrics_func());
}

30
components/security_apps/rate_limit/rate_limit.cc
View File

@@ -9,7 +9,7 @@
#include "i_mainloop.h"
#include "i_time_get.h"
#include "rate_limit_config.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "http_inspection_events.h"
#include "Waf2Util.h"
#include "generic_rulebase/evaluators/asset_eval.h"
@@ -329,7 +329,7 @@ public:
auto asset_location =
Singleton::Consume<I_GeoLocation>::by<RateLimit>()->lookupLocation(maybe_source_ip.unpack());
if (!asset_location.ok()) {
dbgWarning(D_RATE_LIMIT)
dbgDebug(D_RATE_LIMIT)
<< "Rate limit lookup location failed for source: "
<< source_ip
<< ", Error: "
@@ -400,7 +400,19 @@ public:
if (calcRuleAction(rule) == RateLimitAction::PREVENT) {
dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit";
return DROP;
EventVerdict verdict = DROP;
ScopedContext rate_limit_ctx;
rate_limit_ctx.registerValue<GenericConfigId>(AssetMatcher::ctx_key, asset_id);
auto maybe_rate_limit_config = getConfiguration<RateLimitConfig>("rulebase", "rateLimit");
if (maybe_rate_limit_config.ok()) {
const string &web_user_response_id = maybe_rate_limit_config.unpack().getWebUserResponse();
if (!web_user_response_id.empty()) {
verdict.setWebUserResponseByPractice(web_user_response_id);
dbgTrace(D_RATE_LIMIT) << "Set web user response: " << web_user_response_id;
}
}
return verdict;
}
dbgTrace(D_RATE_LIMIT) << "Received DROP in detect mode, will not block.";
@@ -490,7 +502,7 @@ public:
return;
}
auto maybe_rule_by_ctx = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
auto maybe_rule_by_ctx = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
if (!maybe_rule_by_ctx.ok()) {
dbgWarning(D_RATE_LIMIT)
<< "rule was not found by the given context. Reason: "
@@ -732,9 +744,9 @@ public:
I_EnvDetails* i_env_details = nullptr;
private:
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
RateLimitAction practice_action;
string rate_limit_lua_script_hash;
@@ -761,8 +773,8 @@ RateLimit::~RateLimit() = default;
void
RateLimit::preload()
{
registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
registerExpectedConfigurationWithCache<WaapConfigApplication>("assetId", "WAAP", "WebApplicationSecurity");
registerExpectedConfigurationWithCache<WaapConfigAPI>("assetId", "WAAP", "WebAPISecurity");
registerExpectedConfigFile("waap", Config::ConfigFileType::Policy);
registerExpectedConfiguration<RateLimitConfig>("rulebase", "rateLimit");
registerExpectedConfigFile("accessControlV2", Config::ConfigFileType::Policy);

11
components/security_apps/rate_limit/rate_limit_config.cc
View File

@@ -120,6 +120,17 @@ RateLimitConfig::load(cereal::JSONInputArchive &ar)
dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON config. Error: " << e.what();
ar.setNextName(nullptr);
}
try {
string rate_limit_response;
ar(cereal::make_nvp("webUserResponseId", rate_limit_response));
setWebUserResponse(rate_limit_response);
} catch(const std::exception& e) {
dbgDebug(D_RATE_LIMIT)
<< "No webUserResponseId defined for Rate Limit config. Using default. Error: "
<< e.what();
ar.setNextName(nullptr);
}
}
RateLimitRule

127
components/security_apps/waap/include/buffered_compressed_stream.h Normal file
View File

@@ -0,0 +1,127 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <streambuf>
#include <vector>
#include <memory>
#include "compression_utils.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
// Forward declarations
class WaapComponent;
void yieldIfPossible(const std::string &func, int line);
#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__)
//
// Buffered output stream that compresses and encrypts data when flushing
//
// Usage example:
// std::stringstream ss;
// BufferedCompressedOutputStream compressed_stream(ss);
// compressed_stream << "Hello, World!";
// compressed_stream.flush(); // Data is compressed, encrypted, and written to ss
class BufferedCompressedOutputStream : public std::ostream
{
public:
explicit BufferedCompressedOutputStream(std::ostream &underlying_stream);
~BufferedCompressedOutputStream();
// Manual flush to compress, encrypt and write data
void flush();
void close();
private:
class CompressedBuffer : public std::streambuf, Singleton::Consume<I_Encryptor>
{
public:
explicit CompressedBuffer(std::ostream &underlying_stream);
~CompressedBuffer();
// Public method to flush the buffer
void flushAndClose();
void flushBuffer();
protected:
virtual int overflow(int c) override;
virtual std::streamsize xsputn(const char* s, std::streamsize n) override;
virtual int sync() override;
private:
// Compress and encrypt buffer; is_last indicates final chunk
bool compressAndEncryptBuffer(bool is_last);
std::ostream &m_underlying_stream;
std::vector<char> m_buffer;
static const size_t BUFFER_SIZE = 16 * 1024; // 16KiB
CompressionStream* m_compression_stream;
bool m_closed;
};
std::unique_ptr<CompressedBuffer> m_buffer;
};
// Buffered input stream that decrypts and decompresses data when reading
//
// Usage example:
// std::stringstream ss("encrypted compressed data");
// BufferedCompressedInputStream decompressed_stream(ss);
// std::string line;
// std::getline(decompressed_stream, line); // Data is decrypted and decompressed
class BufferedCompressedInputStream : public std::istream
{
public:
explicit BufferedCompressedInputStream(std::istream &underlying_stream);
~BufferedCompressedInputStream();
private:
class DecompressedBuffer : public std::streambuf
{
public:
explicit DecompressedBuffer(std::istream &underlying_stream);
~DecompressedBuffer();
protected:
virtual int underflow() override;
virtual std::streamsize xsgetn(char* s, std::streamsize n) override;
private:
bool fillBuffer();
bool processNextChunk();
bool decryptChunk(const std::vector<char> &encrypted_chunk, std::vector<char> &decrypted_chunk);
bool decompressChunk(const std::vector<char> &compressed_chunk, std::vector<char> &decompressed_chunk);
std::istream &m_underlying_stream;
std::vector<char> m_buffer; // Output buffer for decompressed data
std::vector<char> m_encrypted_buffer; // Buffer for encrypted data from stream
std::vector<char> m_compressed_buffer; // Buffer for decrypted but still compressed data
std::vector<char> m_decompressed_buffer; // Buffer for decompressed data chunks
size_t m_decompressed_pos; // Current position in decompressed buffer
static const size_t OUTPUT_BUFFER_SIZE = 64 * 1024; // 64KiB output buffer
static const size_t CHUNK_SIZE = 16 * 1024; // 16KiB chunks for processing
CompressionStream* m_compression_stream;
bool m_eof_reached;
bool m_stream_finished; // Whether we've finished processing the entire stream
};
std::unique_ptr<DecompressedBuffer> m_buffer;
};

53
components/security_apps/waap/include/i_serialize.h
View File

@@ -12,6 +12,7 @@
// limitations under the License.
#pragma once
#include <chrono>
#include <fstream>
#include "i_time_get.h"
@@ -19,17 +20,23 @@
#include "i_messaging.h"
#include "i_mainloop.h"
#include "i_agent_details.h"
#include "compression_utils.h"
#include <sstream>
#include <vector>
static const uint max_send_obj_retries = 3;
static const std::chrono::microseconds wait_next_attempt(5000000);
USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
// Forward declarations
class WaapComponent;
class RestGetFile : public ClientRest
{
public:
// decrypts and load json
bool loadJson(const std::string& json);
bool loadJson(const std::string &json);
// gen json and encrypt
Maybe<std::string> genJson() const;
};
@@ -47,10 +54,10 @@ public:
// parses xml instead of json
// extracts a file list in <Contents><Key>
bool loadJson(const std::string& xml);
bool loadJson(const std::string &xml);
const std::vector<FileMetaData>& getFilesMetadataList() const;
const std::vector<std::string>& getFilesList() const;
const std::vector<FileMetaData> &getFilesMetadataList() const;
const std::vector<std::string> &getFilesList() const;
private:
RestParam<std::vector<FileMetaData>> files;
@@ -59,18 +66,19 @@ private:
class I_Serializable {
public:
virtual void serialize(std::ostream& stream) = 0;
virtual void deserialize(std::istream& stream) = 0;
virtual void serialize(std::ostream &stream) = 0;
virtual void deserialize(std::istream &stream) = 0;
};
class I_RemoteSyncSerialize {
public:
virtual bool postData() = 0;
virtual void pullData(const std::vector<std::string>& files) = 0;
virtual void pullData(const std::vector<std::string> &files) = 0;
virtual void processData() = 0;
virtual void postProcessedData() = 0;
virtual void pullProcessedData(const std::vector<std::string>& files) = 0;
virtual void updateState(const std::vector<std::string>& files) = 0;
virtual void pullProcessedData(const std::vector<std::string> &files) = 0;
virtual void updateState(const std::vector<std::string> &files) = 0;
virtual Maybe<std::string> getRemoteStateFilePath() const { return genError("No remote state file path defined"); }
};
class I_Backup {
@@ -86,7 +94,7 @@ class SerializeToFileBase :
public I_Serializable
{
public:
SerializeToFileBase(std::string filePath);
SerializeToFileBase(const std::string &filePath);
virtual ~SerializeToFileBase();
virtual void saveData();
@@ -96,13 +104,13 @@ protected:
// saved file name for testing
std::string m_filePath;
private:
void loadFromFile(std::string filePath);
void loadFromFile(const std::string &filePath); // Updated to match implementation
};
class SerializeToFilePeriodically : public SerializeToFileBase
{
public:
SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, std::string filePath);
SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, const std::string &filePath);
virtual ~SerializeToFilePeriodically();
void setInterval(std::chrono::seconds newInterval);
@@ -122,16 +130,16 @@ class SerializeToLocalAndRemoteSyncBase : public I_RemoteSyncSerialize, public S
public:
SerializeToLocalAndRemoteSyncBase(std::chrono::minutes interval,
std::chrono::seconds waitForSync,
const std::string& filePath,
const std::string& remotePath,
const std::string& assetId,
const std::string& owner);
const std::string &filePath,
const std::string &remotePath,
const std::string &assetId,
const std::string &owner);
virtual ~SerializeToLocalAndRemoteSyncBase();
virtual void restore();
virtual void syncWorker();
bool shouldNotSync() const;
void setInterval(std::chrono::seconds newInterval);
std::chrono::seconds getIntervalDuration() const;
void setRemoteSyncEnabled(bool enabled);
@@ -143,7 +151,7 @@ protected:
std::string getUri();
size_t getIntervalsCount();
void incrementIntervalsCount();
bool isBase();
bool isBase() const;
template<typename T>
bool sendObject(T &obj, HTTPMethod method, std::string uri)
@@ -242,7 +250,7 @@ protected:
}
template<typename T>
bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, std::string uri)
bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, const std::string &uri)
{
I_MainLoop *mainloop= Singleton::Consume<I_MainLoop>::by<WaapComponent>();
for (uint i = 0; i < max_send_obj_retries; i++)
@@ -270,10 +278,16 @@ protected:
private:
bool localSyncAndProcess();
void updateStateFromRemoteService();
Maybe<std::string> getStateTimestampByListing();
bool checkAndUpdateStateTimestamp(const std::string& currentStateTimestamp);
RemoteFilesList getProcessedFilesList();
RemoteFilesList getRemoteProcessedFilesList();
std::string getLearningHost();
std::string getSharedStorageHost();
std::string getStateTimestampPath();
Maybe<std::string> getStateTimestamp();
Maybe<void> updateStateFromRemoteFile();
bool shouldSendSyncNotification() const;
I_MainLoop* m_pMainLoop;
std::chrono::microseconds m_waitForSync;
@@ -287,3 +301,4 @@ private:
Maybe<std::string> m_shared_storage_host;
Maybe<std::string> m_learning_host;
};

2
components/security_apps/waap/include/i_transaction.h
View File

@@ -89,7 +89,7 @@ public:
virtual double getOtherModelScore() const = 0;
virtual const std::vector<double> getScoreArray() const = 0;
virtual Waap::CSRF::State& getCsrfState() = 0;
virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;
virtual ServiceVerdict getUserLimitVerdict() = 0;
virtual const std::string getUserLimitVerdictStr() const = 0;
virtual const std::string getViolatedUserLimitTypeStr() const = 0;
virtual void checkShouldInject() = 0;

8
components/security_apps/waap/include/i_waapConfig.h
View File

@@ -40,6 +40,13 @@ enum class AttackMitigationMode
PREVENT,
UNKNOWN
};
enum class WaapConfigType {
Unknown = 0,
Application,
API
};
class IWaapConfig {
public:
virtual const std::string& get_AssetId() const = 0;
@@ -65,6 +72,7 @@ public:
virtual const std::shared_ptr<Waap::RateLimiting::Policy>& get_ErrorLimitingPolicy() const = 0;
virtual const std::shared_ptr<Waap::SecurityHeaders::Policy>& get_SecurityHeadersPolicy() const = 0;
virtual const std::shared_ptr<Waap::UserLimits::Policy>& get_UserLimitsPolicy() const = 0;
virtual WaapConfigType getType() const = 0;
virtual void printMe(std::ostream& os) const = 0;
};

4
components/security_apps/waap/reputation/reputation_features_agg.cc
View File

@@ -30,7 +30,7 @@ template <typename EventType>
class DefaultListener : public Listener<EventType>
{
public:
DefaultListener(EventVerdict defaultVerdict = EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT))
DefaultListener(EventVerdict defaultVerdict = EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT))
:
m_default_verdict(defaultVerdict)
{}
@@ -58,7 +58,7 @@ class ReputationFeaturesAgg::Impl
public:
Impl()
:
DefaultListener<ResponseCodeEvent>(EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT)),
DefaultListener<ResponseCodeEvent>(EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT)),
m_agg_entries()
{
}

35
components/security_apps/waap/waap_clib/AssertionRegexes.h Normal file
View File

@@ -0,0 +1,35 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ASSERTION_REGEXES_H__
#define __ASSERTION_REGEXES_H__
#include <boost/regex.hpp>
namespace Waap {
namespace AssertionRegexes {
// Static const boost regexes used in processAssertions() function
// These regexes detect various assertion patterns in regex strings
// The patterns are in a separate file to avoid this codestyle checker issue:
// "error T009: comma should be followed by whitespace"
static const boost::regex reStartNonWordBehind(R"(\(\?<!\\w\))"); // (?<!\w)
static const boost::regex reEndNonWordAhead(R"(\(\?!\\w\))"); // (?!\w)
static const boost::regex reEndNonWordSpecial(R"(\(\?=\[\^\\w\?<>:=\]\|\$\))"); // (?=[^\w?<>:=]|$)
static const boost::regex rePathTraversalStart(R"(\(\?<!\[\\\.\,:\]\))"); // (?<![\.,:])
static const boost::regex rePathTraversalEnd(R"(\(\?!\[\\\.\,:\]\))"); // (?![\.,:])
} // namespace AssertionRegexes
} // namespace Waap
#endif // __ASSERTION_REGEXES_H__

8
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -93,6 +93,14 @@ add_library(waap_clib
ParserBinaryFile.cc
RegexComparator.cc
RequestsMonitor.cc
WaapHyperscanEngine.cc
buffered_compressed_stream.cc
UnifiedIndicatorsContainer.cc
)
add_library(
UnifiedIndicatorsContainer
UnifiedIndicatorsContainer.cc
)
add_definitions("-Wno-unused-function")

652
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

File diff suppressed because it is too large Load Diff

22
components/security_apps/waap/waap_clib/ConfidenceCalculator.h
View File

@@ -66,6 +66,8 @@ struct ConfidenceCalculatorParams
friend std::ostream & operator<<(std::ostream &os, const ConfidenceCalculatorParams &ccp);
};
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
class ConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase
{
public:
@@ -106,17 +108,16 @@ public:
bool reset(ConfidenceCalculatorParams &params);
virtual bool postData();
virtual void pullData(const std::vector<std::string>& files);
virtual void pullData(const std::vector<std::string> &files);
virtual void processData();
virtual void postProcessedData();
virtual void pullProcessedData(const std::vector<std::string>& files);
virtual void updateState(const std::vector<std::string>& files);
virtual void pullProcessedData(const std::vector<std::string> &files);
virtual void updateState(const std::vector<std::string> &files);
virtual Maybe<std::string> getRemoteStateFilePath() const override;
virtual void serialize(std::ostream &stream);
virtual void deserialize(std::istream &stream);
Maybe<void> writeToFile(const std::string& path, const std::vector<unsigned char>& data);
void mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull);
bool is_confident(const Key &key, const Val &value) const;
@@ -135,6 +136,13 @@ public:
static void mergeConfidenceSets(ConfidenceSet &confidence_set,
const ConfidenceSet &confidence_set_to_merge,
size_t &last_indicators_update);
// Methods for conditional parameter tracking (INXT-46771)
void setIndicatorTrackingKeys(const std::vector<std::string>& keys);
void markKeyAsConfident(const Key &key);
bool shouldTrackParameter(const Key &key, const Val &value);
void extractLowConfidenceKeys(const ConfidenceLevels& confidence_levels);
private:
void loadVer0(cereal::JSONInputArchive &archive);
void loadVer1(cereal::JSONInputArchive &archive);
@@ -178,4 +186,8 @@ private:
I_MainLoop *m_mainLoop;
I_MainLoop::RoutineID m_routineId;
std::vector<std::string> m_filesToRemove;
// Additional member variables for conditional tracking (INXT-46771)
std::unordered_set<std::string> m_indicator_tracking_keys;
bool m_tracking_keys_received;
};

10
components/security_apps/waap/waap_clib/ConfidenceFile.cc
View File

@@ -29,9 +29,15 @@ Maybe<ConfidenceCalculator::ConfidenceLevels> ConfidenceFileDecryptor::getConfid
return genError("failed to get confidence levels");
}
Maybe<std::vector<std::string>> ConfidenceFileDecryptor::getTrackingKeys() const
{
if (!tracking_keys.get().empty()) return tracking_keys.get();
return genError("failed to get tracking keys");
}
ConfidenceFileEncryptor::ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet& _confidence_set,
const ConfidenceCalculator::ConfidenceLevels& _confidence_levels) :
ConfidenceFileEncryptor::ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet &_confidence_set,
const ConfidenceCalculator::ConfidenceLevels &_confidence_levels) :
confidence_set(_confidence_set), confidence_levels(_confidence_levels)
{
}

7
components/security_apps/waap/waap_clib/ConfidenceFile.h
View File

@@ -23,17 +23,20 @@ public:
getConfidenceSet() const;
Maybe<ConfidenceCalculator::ConfidenceLevels>
getConfidenceLevels() const;
Maybe<std::vector<std::string>>
getTrackingKeys() const;
private:
S2C_PARAM(ConfidenceCalculator::ConfidenceSet, confidence_set);
S2C_OPTIONAL_PARAM(ConfidenceCalculator::ConfidenceLevels, confidence_levels);
S2C_OPTIONAL_PARAM(std::vector<std::string>, tracking_keys);
};
class ConfidenceFileEncryptor : public RestGetFile
{
public:
ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet& _confidence_set,
const ConfidenceCalculator::ConfidenceLevels& _confidence_levels);
ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet &_confidence_set,
const ConfidenceCalculator::ConfidenceLevels &_confidence_levels);
private:
C2S_PARAM(ConfidenceCalculator::ConfidenceSet, confidence_set);

6
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -414,7 +414,9 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
if (valueStats.canSplitPipe || valueStats.canSplitSemicolon) {
std::string key = IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction);
m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction);
if (m_pWaapAssetState->m_filtersMngr != nullptr) {
m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction);
}
}
// Detect and decode UTF-16 data
@@ -1090,7 +1092,7 @@ DeepParser::createInternalParser(
int offset = -1;
auto pWaapAssetState = m_pTransaction->getAssetState();
std::shared_ptr<Signatures> signatures = m_pWaapAssetState->getSignatures();
if (pWaapAssetState != nullptr) {
if (pWaapAssetState != nullptr && pWaapAssetState->m_filtersMngr != nullptr) {
// Find out learned type
std::set<std::string> paramTypes = pWaapAssetState->m_filtersMngr->getParameterTypes(
IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction)

9
components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc
View File

@@ -14,6 +14,7 @@
#include "i_indicatorsFilter.h"
#include "IndicatorsFilterBase.h"
#include "Waf2Engine.h"
#include "maybe_res.h"
IndicatorFilterBase::IndicatorFilterBase(const std::string& confidence_path,
const std::string& trusted_path,
@@ -113,12 +114,12 @@ bool IndicatorFilterBase::isTrustedSourceOfType(const std::string& source,
}
std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction)
Maybe<std::string> IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction)
{
if (m_policy == nullptr)
{
dbgTrace(D_WAAP) << "Policy for trusted sources is not set";
return "";
return genError<std::string>("Policy for trusted sources is not set");
}
auto trustedTypes = m_policy->getTrustedTypes();
std::string xFwdVal;
@@ -171,7 +172,7 @@ std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction
}
}
return "";
return genError<std::string>("No trusted source found");
}
void IndicatorFilterBase::registerKeyword(const std::string& key,
@@ -191,3 +192,5 @@ void IndicatorFilterBase::registerKeyword(const std::string& key,
m_trusted_confidence_calc.log(key, keyword, trusted_src);
}
}
//TODO: update states function (use getRemoteStateFilePath)

3
components/security_apps/waap/waap_clib/IndicatorsFilterBase.h
View File

@@ -19,6 +19,7 @@
#include "TrustedSourcesConfidence.h"
#include "ConfidenceCalculator.h"
#include "TuningDecisions.h"
#include "maybe_res.h"
class IndicatorFilterBase : public I_IndicatorsFilter
{
@@ -39,8 +40,8 @@ public:
bool setTrustedSrcParameter(std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter> policy);
void reset();
Maybe<std::string> getTrustedSource(IWaf2Transaction* pTransaction);
protected:
std::string getTrustedSource(IWaf2Transaction* pTransaction);
void registerKeyword(const std::string& key,
const std::string& keyword,
const std::string& source,

279
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc
View File

@@ -19,44 +19,168 @@
#include "FpMitigation.h"
#include "Waf2Engine.h"
#include "WaapKeywords.h"
#include "config.h"
IndicatorsFiltersManager::IndicatorsFiltersManager(const std::string& remotePath, const std::string &assetId,
USE_DEBUG_FLAG(D_WAAP_LEARN);
static constexpr int DEFAULT_SOURCES_LIMIT = 1000;
using namespace std;
// // Helper class for posting unified indicators
// class UnifiedIndicatorsLogPost : public RestGetFile {
// public:
// UnifiedIndicatorsLogPost(shared_ptr<UnifiedIndicatorsContainer> container_ptr)
// {
// window_logger = move(*container_ptr);
// }
// private:
// C2S_PARAM(UnifiedIndicatorsContainer, window_logger);
// };
IndicatorsFiltersManager::IndicatorsFiltersManager(const string& remotePath, const string &assetId,
I_WaapAssetState* pWaapAssetState)
:
SerializeToFileBase(pWaapAssetState->getWaapDataDir() + "/6.data"),
SerializeToLocalAndRemoteSyncBase(
chrono::minutes(120),
chrono::seconds(300),
pWaapAssetState->getWaapDataDir() + "/6.data",
(remotePath == "") ? remotePath : remotePath + "/CentralizedData",
assetId,
"IndicatorsFiltersManager"
),
m_pWaapAssetState(pWaapAssetState),
m_ignoreSources(pWaapAssetState->getWaapDataDir(), remotePath, assetId),
m_tuning(remotePath),
m_matchedOverrideKeywords()
m_matchedOverrideKeywords(),
m_isLeading(getSettingWithDefault<bool>(true, "features", "learningLeader")),
m_sources_limit(DEFAULT_SOURCES_LIMIT),
m_uniqueSources(),
m_unifiedIndicators(make_shared<UnifiedIndicatorsContainer>())
{
restore();
m_keywordsFreqFilter = std::make_unique<KeywordIndicatorFilter>(
m_keywordsFreqFilter = make_unique<KeywordIndicatorFilter>(
pWaapAssetState->getWaapDataDir(),
remotePath,
assetId,
&m_ignoreSources,
&m_tuning);
m_typeFilter = std::make_unique<TypeIndicatorFilter>(pWaapAssetState, remotePath, assetId, &m_tuning);
m_typeFilter = make_unique<TypeIndicatorFilter>(pWaapAssetState, remotePath, assetId, &m_tuning);
m_uniqueSources.reserve(m_sources_limit);
registerConfigLoadCb([this](){
updateSourcesLimit();
updateLearningLeaderFlag();
});
}
IndicatorsFiltersManager::~IndicatorsFiltersManager()
{
}
void IndicatorsFiltersManager::registerKeywords(const std::string& key,
bool IndicatorsFiltersManager::shouldRegister(
const string& key,
const Waap::Keywords::KeywordsSet& keywords,
const IWaf2Transaction* pTransaction)
{
// Check if the learning leader flag is true
if (!m_isLeading) {
dbgDebug(D_WAAP_LEARN) << "Learning leader flag is false. Skipping source ID assertion.";
return false;
}
// if key needs tracking
if (m_keywordsFreqFilter->shouldTrack(key, keywords) ||
m_typeFilter->shouldTrack(key, pTransaction)) {
dbgDebug(D_WAAP_LEARN) << "Key '" << key << "' needs tracking.";
} else {
dbgTrace(D_WAAP_LEARN) << "Key '" << key << "' does not need tracking.";
return false;
}
auto sourceId = pTransaction->getSourceIdentifier();
// Check if the database has reached its limit and source is unknown
if (m_uniqueSources.size() >= static_cast<size_t>(m_sources_limit) &&
m_uniqueSources.find(sourceId) == m_uniqueSources.end() ) {
dbgDebug(D_WAAP_LEARN) << "Database limit reached. Cannot insert new source ID '" << sourceId << "'.";
return false;
}
// Insert the sourceId into the database when did not reached the limit
// If limit is reached, we know that sourceId is already in the database
auto insertResult = m_uniqueSources.insert(sourceId);
if (insertResult.second) {
dbgDebug(D_WAAP_LEARN) << "Inserted new source ID '" << sourceId << "' into the database.";
} else {
dbgTrace(D_WAAP_LEARN) << "source ID '" << sourceId << "' exists in database.";
}
return true;
}
void IndicatorsFiltersManager::registerKeywords(
const string& key,
Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pWaapTransaction)
{
// Check should register - if false, do not collect data
if (!shouldRegister(key, keywords, pWaapTransaction)) {
return;
}
const std::string& sourceId = pWaapTransaction->getSourceIdentifier();
if (m_tuning.getDecision(pWaapTransaction->getLastScanParamName(), PARAM_NAME) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getLastScanSample(), PARAM_VALUE) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getUri(), URL) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getSourceIdentifier(), SOURCE) == MALICIOUS)
m_tuning.getDecision(sourceId, SOURCE) == MALICIOUS)
{
dbgDebug(D_WAAP_LEARN) << "Skipping registration due to tuning decision (malicious)";
return;
}
// TODO: add configuration to choose central logging and return, else do legacy
if(getProfileAgentSettingWithDefault<bool>(false, "agent.learning.centralLogging")) {
dbgDebug(D_WAAP_LEARN) << "Central logging is enabled.";
// Build unified entry
UnifiedIndicatorsContainer::Entry entry;
entry.key = key;
entry.sourceId = pWaapTransaction->getSourceIdentifier();
// Use the new getTrustedSource method for proper trusted source checking
if (m_keywordsFreqFilter) {
auto trustedSourceResult = m_keywordsFreqFilter->getTrustedSource(pWaapTransaction);
entry.isTrusted = trustedSourceResult.ok();
if (entry.isTrusted) {
dbgDebug(D_WAAP_LEARN) << "Entry is from trusted source: " << trustedSourceResult.unpack();
}
} else {
entry.isTrusted = false;
}
for (const auto &kw : keywords) {
entry.indicators.push_back(kw);
}
// Add parameter types as TYPE indicators if applicable (skip url# keys)
if (key.rfind("url#", 0) != 0) {
string sample = pWaapTransaction->getLastScanSample();
auto sampleTypes = m_pWaapAssetState->getSampleType(sample);
entry.types.insert(entry.types.end(), sampleTypes.begin(), sampleTypes.end());
}
// Push to unified container
m_unifiedIndicators->addEntry(entry);
return;
}
dbgTrace(D_WAAP_LEARN) << "Central logging is disabled. Using legacy filters.";
// Legacy behavior (optional): keep existing filters updates for backward compatibility
if (!keywords.empty())
{
m_ignoreSources.log(pWaapTransaction->getSourceIdentifier(), key, keywords);
m_ignoreSources.log(sourceId, key, keywords);
}
m_keywordsFreqFilter->registerKeywords(key, keywords, pWaapTransaction);
if (key.rfind("url#", 0) == 0)
{
@@ -73,7 +197,7 @@ void IndicatorsFiltersManager::registerKeywords(const std::string& key,
}
}
bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const std::string &keyword) const
bool IndicatorsFiltersManager::shouldFilterKeyword(const string &key, const string &keyword) const
{
bool shouldFilter = false;
if (m_keywordsFreqFilter != nullptr)
@@ -99,14 +223,14 @@ bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const
return shouldFilter;
}
void IndicatorsFiltersManager::serialize(std::ostream& stream)
void IndicatorsFiltersManager::serialize(ostream& stream)
{
cereal::JSONOutputArchive archive(stream);
archive(cereal::make_nvp("version", 1), cereal::make_nvp("trustedSrcParams", m_trustedSrcParams));
}
void IndicatorsFiltersManager::deserialize(std::istream& stream)
void IndicatorsFiltersManager::deserialize(istream& stream)
{
cereal::JSONInputArchive archive(stream);
@@ -116,10 +240,10 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream)
{
archive(cereal::make_nvp("version", version));
}
catch (std::runtime_error & e) {
catch (runtime_error & e) {
archive.setNextName(nullptr);
version = 0;
dbgDebug(D_WAAP) << "Can't load file version: " << e.what();
dbgDebug(D_WAAP_LEARN) << "Can't load file version: " << e.what();
}
switch (version)
@@ -131,12 +255,12 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream)
archive(cereal::make_nvp("trustedSrcParams", m_trustedSrcParams));
break;
default:
dbgWarning(D_WAAP) << "unknown file format version: " << version;
dbgWarning(D_WAAP_LEARN) << "unknown file format version: " << version;
break;
}
}
std::set<std::string> IndicatorsFiltersManager::getParameterTypes(const std::string& canonicParam) const
set<string> IndicatorsFiltersManager::getParameterTypes(const string& canonicParam) const
{
return m_typeFilter->getParamTypes(canonicParam);
}
@@ -166,18 +290,18 @@ bool IndicatorsFiltersManager::loadPolicy(IWaapConfig* pConfig)
}
else
{
dbgWarning(D_WAAP) << "Failed to get configuration";
dbgWarning(D_WAAP_LEARN) << "Failed to get configuration";
}
return pConfig != NULL;
}
void IndicatorsFiltersManager::filterVerbose(const std::string &param,
std::vector<std::string>& filteredKeywords,
std::map<std::string, std::vector<std::string>>& filteredKeywordsVerbose)
void IndicatorsFiltersManager::filterVerbose(const string &param,
vector<string>& filteredKeywords,
map<string, vector<string>>& filteredKeywordsVerbose)
{
static std::string typeFilterName = "type indicators filter";
static std::string keywordsFilterName = "keywords frequency indicators filter";
static string typeFilterName = "type indicators filter";
static string keywordsFilterName = "keywords frequency indicators filter";
filteredKeywordsVerbose[typeFilterName];
filteredKeywordsVerbose[keywordsFilterName];
auto types = getParameterTypes(param);
@@ -209,12 +333,12 @@ void IndicatorsFiltersManager::reset()
}
std::string IndicatorsFiltersManager::extractUri(const std::string& referer, const IWaf2Transaction* pTransaction)
string IndicatorsFiltersManager::extractUri(const string& referer, const IWaf2Transaction* pTransaction)
{
std::string url;
string url;
size_t pos = referer.find("://");
if (pos == std::string::npos || (pos + 3) > referer.size())
if (pos == string::npos || (pos + 3) > referer.size())
{
url = referer;
}
@@ -223,11 +347,11 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
url = referer.substr(pos + 3);
}
pos = url.find('/');
if (pos == std::string::npos)
if (pos == string::npos)
{
return url;
}
std::string host = url.substr(0, pos);
string host = url.substr(0, pos);
if (host == pTransaction->getHdrContent("host"))
{
return url.substr(pos);
@@ -235,13 +359,13 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
return url;
}
std::string IndicatorsFiltersManager::generateKey(const std::string& location,
const std::string& param_name,
string IndicatorsFiltersManager::generateKey(const string& location,
const string& param_name,
const IWaf2Transaction* pTransaction)
{
std::string key = location;
static const std::string delim = "#";
std::string param = normalize_param(param_name);
string key = location;
static const string delim = "#";
string param = normalize_param(param_name);
if (location == "header" || location == "cookie" || location == "url_param")
{
@@ -268,8 +392,8 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location,
}
else if (location == "referer")
{
std::string referer = pTransaction->getHdrContent("referer");
std::string uri = extractUri(referer, pTransaction);
string referer = pTransaction->getHdrContent("referer");
string uri = extractUri(referer, pTransaction);
key = "url" + delim + normalize_uri(uri);
}
else
@@ -279,10 +403,10 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location,
return key;
}
std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& canonicKey, IWaf2Transaction* pTransaction)
string IndicatorsFiltersManager::getLocationFromKey(const string& canonicKey, IWaf2Transaction* pTransaction)
{
std::vector<std::string> known_locations = { "header", "cookie", "url", "body", "referer", "url_param" };
std::string delim = "#";
vector<string> known_locations = { "header", "cookie", "url", "body", "referer", "url_param" };
string delim = "#";
for (auto location : known_locations)
{
if (canonicKey.find(location + delim) == 0)
@@ -294,9 +418,9 @@ std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& cano
}
void IndicatorsFiltersManager::filterKeywords(
const std::string &key,
const string &key,
Waap::Keywords::KeywordsSet& keywords,
std::vector<std::string>& filteredKeywords)
vector<string>& filteredKeywords)
{
for (auto keyword = keywords.begin(); keyword != keywords.end(); )
{
@@ -313,10 +437,15 @@ void IndicatorsFiltersManager::filterKeywords(
}
void IndicatorsFiltersManager::pushSample(
const std::string& key,
const std::string& sample,
const string& key,
const string& sample,
IWaf2Transaction* pTransaction)
{
// Check learning leader flag - if false, do not collect data
if (!m_isLeading) {
return;
}
if (key.rfind("url#", 0) == 0)
{
return;
@@ -324,7 +453,75 @@ void IndicatorsFiltersManager::pushSample(
m_typeFilter->registerKeywords(key, sample, pTransaction);
}
std::set<std::string> & IndicatorsFiltersManager::getMatchedOverrideKeywords(void)
set<string> & IndicatorsFiltersManager::getMatchedOverrideKeywords(void)
{
return m_matchedOverrideKeywords;
}
void IndicatorsFiltersManager::updateLearningLeaderFlag() {
m_isLeading = getSettingWithDefault<bool>(true, "features", "learningLeader");
dbgDebug(D_WAAP_LEARN) << "Updating learning leader flag from configuration: " << (m_isLeading ? "true" : "false");
}
void IndicatorsFiltersManager::updateSourcesLimit()
{
int new_limit = getProfileAgentSettingWithDefault<int>(DEFAULT_SOURCES_LIMIT, "agent.learning.sourcesLimit");
if (new_limit != m_sources_limit) {
m_sources_limit = new_limit;
m_uniqueSources.reserve(m_sources_limit);
}
}
bool IndicatorsFiltersManager::postData()
{
dbgDebug(D_WAAP_LEARN) << "Posting indicators data";
// Example: post unified indicators data if present
if (m_unifiedIndicators->getKeyCount() == 0) {
dbgDebug(D_WAAP_LEARN) << "No unified indicators to post, skipping";
return true; // Nothing to post
}
// Post unified indicators using REST client with C2S_PARAM
UnifiedIndicatorsLogPost logPost(m_unifiedIndicators);
string postUrl = getPostDataUrl();
dbgTrace(D_WAAP_LEARN) << "Posting unified indicators to: " << postUrl;
bool ok = sendNoReplyObjectWithRetry(logPost, HTTPMethod::PUT, postUrl);
if (!ok) {
dbgError(D_WAAP_LEARN) << "Failed to post unified indicators to: " << postUrl;
}
m_unifiedIndicators = make_shared<UnifiedIndicatorsContainer>();
m_uniqueSources.clear();
return ok;
}
void IndicatorsFiltersManager::pullData(const vector<string>& files)
{
// Phase 2 : backup sync flow
// Add logic for pulling data from a remote service
}
void IndicatorsFiltersManager::processData()
{
// Phase 2 : backup sync flow
// Add logic for processing pulled data
// call filters with ptr to unified data to process data from m_unifiedIndicators
}
void IndicatorsFiltersManager::postProcessedData()
{
// Add logic for posting processed data to a remote service
}
void IndicatorsFiltersManager::pullProcessedData(const vector<string>& files)
{
// Add logic for pulling processed data from a remote service
}
// TODO: Phase 3 implement getRemoteStateFilePath to return the base dir
void IndicatorsFiltersManager::updateState(const vector<string>& files)
{
// files is a list of single file base dir
// TODO phase 3: call each filter to update internal states
}

51
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h
View File

@@ -24,48 +24,73 @@
#include <cereal/cereal.hpp>
#include <cereal/types/memory.hpp>
#include <cereal/archives/json.hpp>
#include "UnifiedIndicatorsContainer.h"
using namespace Waap::Parameters;
class IWaf2Transaction;
struct Waf2ScanResult;
class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToFileBase
class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToLocalAndRemoteSyncBase
{
public:
IndicatorsFiltersManager(const std::string &remotePath, const std::string &assetId,
I_WaapAssetState* pWaapAssetState);
~IndicatorsFiltersManager();
virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
virtual void registerKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords,
IWaf2Transaction* pWaapTransaction);
virtual bool shouldFilterKeyword(const std::string &key, const std::string &keyword) const;
virtual void filterKeywords(const std::string &key, Waap::Keywords::KeywordsSet& keywords,
std::vector<std::string>& filteredKeywords);
virtual void filterKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords,
std::vector<std::string> &filteredKeywords);
std::set<std::string> &getMatchedOverrideKeywords(void);
void pushSample(const std::string& key, const std::string& sample, IWaf2Transaction* pTransaction);
void pushSample(const std::string &key, const std::string &sample, IWaf2Transaction* pTransaction);
bool loadPolicy(IWaapConfig* pConfig);
void reset();
void filterVerbose(const std::string &param,
std::vector<std::string>& filteredKeywords,
std::vector<std::string> &filteredKeywords,
std::map<std::string, std::vector<std::string>>& filteredKeywordsVerbose);
static std::string getLocationFromKey(const std::string& canonicKey, IWaf2Transaction* pTransaction);
static std::string generateKey(const std::string& location,
const std::string& param,
static std::string getLocationFromKey(const std::string &canonicKey, IWaf2Transaction* pTransaction);
static std::string generateKey(const std::string &location,
const std::string &param,
const IWaf2Transaction* pTransaction);
virtual void serialize(std::ostream& stream);
virtual void deserialize(std::istream& stream);
virtual void serialize(std::ostream &stream);
virtual void deserialize(std::istream &stream);
virtual std::set<std::string> getParameterTypes(const std::string& canonicParam) const;
virtual std::set<std::string> getParameterTypes(const std::string &canonicParam) const;
// New required functions from SerializeToLocalAndRemoteSyncBase
virtual bool postData() override;
virtual void pullData(const std::vector<std::string>& files) override;
virtual void processData() override;
virtual void postProcessedData() override;
virtual void pullProcessedData(const std::vector<std::string>& files) override;
virtual void updateState(const std::vector<std::string>& files) override;
// Getter for unified indicators (for testing)
const UnifiedIndicatorsContainer& getUnifiedIndicators() const { return *m_unifiedIndicators; }
private:
static std::string extractUri(const std::string& referer, const IWaf2Transaction* pTransaction);
static std::string extractUri(const std::string &referer, const IWaf2Transaction* pTransaction);
void updateLearningLeaderFlag();
bool shouldRegister(
const std::string& key,
const Waap::Keywords::KeywordsSet& keywords,
const IWaf2Transaction* pTransaction
);
void updateSourcesLimit();
std::unique_ptr<KeywordIndicatorFilter> m_keywordsFreqFilter;
std::unique_ptr<TypeIndicatorFilter> m_typeFilter;
I_WaapAssetState* m_pWaapAssetState;
std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter> m_trustedSrcParams;
ScannerDetector m_ignoreSources;
TuningDecision m_tuning;
std::set<std::string> m_matchedOverrideKeywords;
bool m_isLeading;
int m_sources_limit = 0;
std::unordered_set<std::string> m_uniqueSources;
std::shared_ptr<UnifiedIndicatorsContainer> m_unifiedIndicators;
};

311
components/security_apps/waap/waap_clib/KeyStack.cc
View File

@@ -20,56 +20,131 @@
USE_DEBUG_FLAG(D_WAAP);
KeyStack::KeyStack(const char* name)
:m_name(name), m_nameDepth(0) {
:m_name(name), m_nameDepth(0), m_total_length(0), m_using_buffer(true),
m_str_cache_valid(false), m_first_cache_valid(false) {
m_buffer[0] = '\0';
m_positions.reserve(16); // Reserve reasonable capacity
m_lengths.reserve(16);
m_fallback_stack.reserve(16);
}
void KeyStack::push(const char* subkey, size_t subkeySize, bool countDepth) {
m_stack.push_back(m_key.size());
void
KeyStack::push(const char *subkey, size_t subkeySize, bool countDepth)
{
bool dot_needed = !m_positions.empty() && subkey != nullptr && subkeySize > 0;
if (m_using_buffer) {
// Calculate space needed: subkey + dot (if not first) + null terminator
size_t dot_size = dot_needed ? 1 : 0;
size_t needed_space = subkeySize + dot_size + 1; // +1 for null terminator
// Prefix all subkeys (except the first) with '.'
if (!m_key.empty()) {
m_key += '.';
if (can_fit_in_buffer(needed_space)) {
// Fast path: use fixed buffer
if (dot_needed) {
m_buffer[m_total_length] = '.';
m_total_length++;
}
m_positions.push_back(m_total_length);
m_lengths.push_back(subkeySize);
memcpy(m_buffer + m_total_length, subkey, subkeySize);
m_total_length += subkeySize;
m_buffer[m_total_length] = '\0';
} else {
// Switch to fallback mode
switch_to_fallback();
// Continue with fallback logic below
}
}
m_key += std::string(subkey, subkeySize);
if (!m_using_buffer) {
// Slow path: use dynamic string
m_fallback_stack.push_back(m_fallback_key.size());
if (dot_needed) {
m_fallback_key.append(1, '.');
}
m_fallback_key.append(subkey, subkeySize);
}
if (countDepth) {
m_nameDepth++;
}
// Invalidate cache since key structure changed
invalidate_cache();
dbgTrace(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::push(): '"
<< std::string(subkey, subkeySize)
<< "' => full_key='"
<< std::string(m_key.data(), m_key.size())
<< c_str()
<< "'";
}
void KeyStack::pop(const char* log, bool countDepth) {
// Keep depth balanced even if m_key[] buffer is full
if (m_key.empty() || m_stack.empty()) {
dbgDebug(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! "
<< log;
return;
if (m_using_buffer) {
if (m_positions.empty()) {
dbgDebug(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! "
<< log;
return;
}
// Remove last subkey from buffer
m_total_length = m_positions.back();
// Only remove dot if:
// 1. There are multiple elements (not the first)
// 2. The element being popped had content (length > 0, meaning a dot was added)
// 3. The character before current position is actually a dot (safety check)
if (m_positions.size() > 1 && m_lengths.back() > 0) {
if (m_total_length > 0 && m_buffer[m_total_length - 1] == '.') {
m_total_length -= 1; // Remove the dot
}
} else if (m_positions.size() == 1) {
m_total_length = 0; // First element, no dot to remove
}
m_positions.pop_back();
m_lengths.pop_back();
m_buffer[m_total_length] = '\0';
} else {
// Fallback mode
if (m_fallback_key.empty() || m_fallback_stack.empty()) {
dbgDebug(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! "
<< log;
return;
}
// Remove last subkey
m_fallback_key.erase(m_fallback_stack.back());
m_fallback_stack.pop_back();
// Try to switch back to buffer if possible
if (m_fallback_key.size() + 1 < MAX_KEY_SIZE) {
rebuild_buffer_from_fallback();
}
}
if (countDepth) {
m_nameDepth--;
}
// Remove last subkey.
m_key.erase(m_stack.back());
m_stack.pop_back();
// Invalidate cache since key structure changed
invalidate_cache();
dbgTrace(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): full_key='"
<< std::string(m_key.data(), (int)m_key.size())
<< c_str()
<< "': pop_key="
<< log
<< "'";
@@ -81,6 +156,200 @@ void KeyStack::print(std::ostream &os) const
<< "KeyStack("
<< m_name
<< ")::show(): full_key='"
<< std::string(m_key.data(), (int)m_key.size())
<< c_str()
<< "'";
}
void KeyStack::clear() {
if (m_using_buffer) {
m_positions.clear();
m_lengths.clear();
m_total_length = 0;
m_buffer[0] = '\0';
} else {
m_fallback_key.clear();
m_fallback_stack.clear();
m_using_buffer = true;
m_total_length = 0;
m_buffer[0] = '\0';
}
m_nameDepth = 0;
invalidate_cache();
}
size_t KeyStack::size() const {
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
return 0;
}
// Return size from second subkey onwards
return m_total_length - m_positions[1];
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
return 0;
}
// m_fallback_stack[1] points to the dot preceding the 2nd subkey.
// Exclude the dot itself from the reported size.
if (m_fallback_stack[1] + 1 >= m_fallback_key.size()) {
return 0; // Defensive: nothing after the dot
}
return m_fallback_key.size() - (m_fallback_stack[1] + 1);
}
}
const char* KeyStack::c_str() const {
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
return "";
}
// Return pointer to second subkey (skip first + dot)
return m_buffer + m_positions[1];
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
return "";
}
// m_fallback_stack[1] points to the dot. Skip it for consistency with buffer mode.
static thread_local std::string temp_result;
size_t start = m_fallback_stack[1] + 1;
if (start >= m_fallback_key.size()) {
temp_result.clear();
} else {
temp_result = m_fallback_key.substr(start);
}
return temp_result.c_str();
}
}
const std::string KeyStack::str() const {
if (m_str_cache_valid) {
return m_cached_str;
}
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
m_cached_str = std::string();
} else {
// Return string from second subkey onwards
m_cached_str = std::string(m_buffer + m_positions[1], m_total_length - m_positions[1]);
}
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
m_cached_str = std::string();
} else {
size_t start = m_fallback_stack[1] + 1; // Skip the dot
if (start >= m_fallback_key.size()) {
m_cached_str.clear();
} else {
m_cached_str = m_fallback_key.substr(start);
}
}
}
m_str_cache_valid = true;
return m_cached_str;
}
const std::string KeyStack::first() const {
if (m_first_cache_valid) {
return m_cached_first;
}
if (m_using_buffer) {
if (m_positions.empty()) {
m_cached_first = std::string();
} else if (m_positions.size() == 1) {
// Only one subkey, return the whole buffer content
m_cached_first = std::string(m_buffer, m_lengths[0]);
} else {
// Multiple subkeys, return first one
m_cached_first = std::string(m_buffer + m_positions[0], m_lengths[0]);
}
} else {
// Fallback mode
if (m_fallback_stack.empty()) {
m_cached_first = std::string();
} else if (m_fallback_stack.size() == 1) {
m_cached_first = m_fallback_key;
} else {
// m_fallback_stack[1] points to the dot; substring up to dot (exclude it)
size_t dot_pos = m_fallback_stack[1];
if (dot_pos == 0 || dot_pos > m_fallback_key.size()) {
m_cached_first.clear();
} else {
m_cached_first = m_fallback_key.substr(0, dot_pos);
}
}
}
m_first_cache_valid = true;
return m_cached_first;
}
bool KeyStack::can_fit_in_buffer(size_t additional_size) const {
return (m_total_length + additional_size) < MAX_KEY_SIZE;
}
void KeyStack::switch_to_fallback() {
// Copy buffer content to fallback string
m_fallback_key.assign(m_buffer, m_total_length);
// Convert positions to stack format used by fallback
m_fallback_stack.clear();
for (size_t i = 0; i < m_positions.size(); ++i) {
if (i == 0) {
m_fallback_stack.push_back(0);
} else {
// Position after dot
m_fallback_stack.push_back(m_positions[i] - 1);
}
}
m_using_buffer = false;
invalidate_cache();
}void KeyStack::rebuild_buffer_from_fallback() {
if (m_fallback_key.size() + 1 >= MAX_KEY_SIZE) {
return; // Still too big for buffer
}
// Copy fallback content back to buffer
memcpy(m_buffer, m_fallback_key.c_str(), m_fallback_key.size());
m_total_length = m_fallback_key.size();
m_buffer[m_total_length] = '\0';
// Rebuild positions and lengths by parsing the buffer
m_positions.clear();
m_lengths.clear();
size_t pos = 0;
while (pos < m_total_length) {
m_positions.push_back(pos);
// Find length of current subkey
size_t start = pos;
while (pos < m_total_length && m_buffer[pos] != '.') {
pos++;
}
m_lengths.push_back(pos - start);
if (pos < m_total_length) {
pos++; // Skip the dot
}
}
// Clear fallback data
m_fallback_key.clear();
m_fallback_stack.clear();
m_using_buffer = true;
invalidate_cache();
}
void KeyStack::invalidate_cache() {
m_str_cache_valid = false;
m_first_cache_valid = false;
}

77
components/security_apps/waap/waap_clib/KeyStack.h
View File

@@ -21,62 +21,49 @@
// Represent string (key) that is concatenation of substrings (subkeys) separated by '.' character.
// Mostly emulates API of C++ std::string class, with addition of push() and pop() methods
// that append individual subkey and delete last subkey from the string efficiently.
// Uses fixed buffer for performance with fallback to dynamic string for long keys.
class KeyStack {
public:
KeyStack(const char *name);
void push(const char *subkey, size_t subkeySize, bool countDepth=true);
void pop(const char* log, bool countDepth=true);
bool empty() const { return m_key.empty(); }
void clear() { m_key.clear(); m_stack.clear(); }
bool empty() const { return m_using_buffer ? m_positions.empty() : m_fallback_key.empty(); }
void clear();
void print(std::ostream &os) const;
size_t depth() const { return m_nameDepth; }
size_t size() const {
return str().size();
}
const char *c_str() const {
// If pushed none - return empty string.
// If pushed once - still return empty string (the once-pushed subkey will only be returned
// by the first() method.
// If pushed twice or more - return all subkeys starting from the second one.
// Also, even if pushed 2 or more times, but pushed empty strings as subkeys,
// then it could happen that m_key is still empty, in which case we should still return empty string.
if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) {
return "";
}
size_t size() const;
const char *c_str() const;
const std::string str() const;
const std::string first() const;
return m_key.c_str() + m_stack[1] + 1;
}
const std::string str() const {
// If pushed none - return empty string.
// If pushed once - still return empty string (the once-pushed subkey will only be returned
// by the first() method.
// If pushed twice or more - return all subkeys starting from the second one.
// Also, even if pushed 2 or more times, but pushed empty strings as subkeys,
// then it could happen that m_key is still empty, in which case we should still return empty string.
if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) {
return "";
}
return m_key.substr(m_stack[1] + 1);
}
const std::string first() const {
if (m_stack.size() == 0) {
return "";
}
else if (m_stack.size() == 1) {
return m_key;
}
else {
// m_stack.size() > 1, so m_stack[1] is valid
return m_key.substr(0, m_stack[1]);
}
}
private:
static const size_t MAX_KEY_SIZE = 1024;
const char *m_name;
std::string m_key;
std::vector<size_t> m_stack; // position of individual key name starts in m_key,
// used to backtrack 1 key at a time.
int m_nameDepth;
// Fixed buffer approach for common case (fast path)
char m_buffer[MAX_KEY_SIZE];
std::vector<size_t> m_positions; // Start positions of each subkey in buffer
std::vector<size_t> m_lengths; // Length of each subkey
size_t m_total_length;
bool m_using_buffer;
// Fallback to dynamic approach for long keys (slow path)
std::string m_fallback_key;
std::vector<size_t> m_fallback_stack;
// Caching for frequently accessed methods
mutable std::string m_cached_str;
mutable std::string m_cached_first;
mutable bool m_str_cache_valid;
mutable bool m_first_cache_valid;
// Helper methods
void switch_to_fallback();
void rebuild_buffer_from_fallback();
bool can_fit_in_buffer(size_t additional_size) const;
void invalidate_cache();
};
#endif // __KEYSTACK_H__0a8039e6

16
components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc
View File

@@ -12,11 +12,11 @@
// limitations under the License.
#include "KeywordIndicatorFilter.h"
#include "i_transaction.h"
#include "waap.h"
#include "WaapConfigApi.h"
#include "WaapConfigApplication.h"
#include "FpMitigation.h"
#include "i_transaction.h"
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/trim.hpp>
@@ -98,12 +98,24 @@ bool KeywordIndicatorFilter::loadParams(std::shared_ptr<Waap::Parameters::WaapPa
return m_confidence_calc.reset(params);
}
bool KeywordIndicatorFilter::shouldTrack(const std::string& key, const Waap::Keywords::KeywordsSet& keywords)
{
for (const auto& keyword : keywords)
{
if (m_confidence_calc.shouldTrackParameter(key, keyword))
{
return true;
}
}
return m_confidence_calc.shouldTrackParameter(key, "");
}
void KeywordIndicatorFilter::registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction)
{
std::string source(pTransaction->getSourceIdentifier());
std::string trusted_source = getTrustedSource(pTransaction);
auto trusted_source_maybe = getTrustedSource(pTransaction);
std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : "";
if (keywords.empty())
{
registerSource(key, source);

2
components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h
View File

@@ -36,10 +36,12 @@ public:
size_t minIntervals = CONFIDENCE_MIN_INTERVALS,
std::chrono::minutes intervalDuration = CONFIDENCE_WINDOW_INTERVAL,
double ratioThreshold = CONFIDENCE_THRESHOLD);
~KeywordIndicatorFilter();
virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction);
bool shouldTrack(const std::string& key, const Waap::Keywords::KeywordsSet& keywords);
virtual bool shouldFilterKeyword(const std::string &key, const std::string &keyword) const;

9
components/security_apps/waap/waap_clib/KeywordTypeValidator.cc
View File

@@ -19,7 +19,7 @@
USE_DEBUG_FLAG(D_WAAP);
KeywordTypeValidator::KeywordTypeValidator(const std::string& mapFilePath) :
KeywordTypeValidator::KeywordTypeValidator(const std::string &mapFilePath) :
SerializeToFileBase(mapFilePath),
m_serializedData(),
m_keywordTypeMap(m_serializedData.m_keywordTypeMap)
@@ -32,7 +32,7 @@ KeywordTypeValidator::~KeywordTypeValidator()
}
void KeywordTypeValidator::serialize(std::ostream& stream)
void KeywordTypeValidator::serialize(std::ostream &stream)
{
(void)stream;
}
@@ -42,7 +42,7 @@ void KeywordTypeValidator::saveData()
// do not override existing file
}
void KeywordTypeValidator::deserialize(std::istream& stream)
void KeywordTypeValidator::deserialize(std::istream &stream)
{
try
{
@@ -54,6 +54,7 @@ void KeywordTypeValidator::deserialize(std::istream& stream)
}
catch (std::runtime_error & e) {
dbgWarning(D_WAAP) << "failed to deserialize keyword types validator file. Error: " << e.what();
throw e;
}
}
@@ -62,7 +63,7 @@ void KeywordTypeValidator::operator=(const KeywordTypeValidator &other) {
m_serializedData.m_keywordTypeMap = other.m_serializedData.m_keywordTypeMap;
}
bool KeywordTypeValidator::isKeywordOfType(const std::string& keyword, ParamType type) const
bool KeywordTypeValidator::isKeywordOfType(const std::string &keyword, ParamType type) const
{
auto keywordEntry = m_keywordTypeMap.find(keyword);
if (keywordEntry != m_keywordTypeMap.end())

230
components/security_apps/waap/waap_clib/ParserGzip.cc
View File

@@ -1,115 +1,115 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserGzip.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_GZIP);
const std::string ParserGzip::m_parserName = "ParserGzip";
ParserGzip::ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth)
:m_receiver(receiver), m_key("gzip"), m_state(s_start), m_stream(nullptr) {
}
ParserGzip::~ParserGzip() {
if (m_stream != nullptr) {
finiCompressionStream(m_stream);
m_stream = nullptr;
}
}
size_t ParserGzip::push(const char *buf, size_t len) {
dbgTrace(D_WAAP_PARSER_GZIP) << "len=" << (unsigned long int)len << ")";
if (len == 0) {
dbgTrace(D_WAAP_PARSER_GZIP) << "end of data signal! m_state=" << m_state;
// flush
if (m_state != s_start) { // only emit if at least something was pushed
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
}
}
return 0;
}
DecompressionResult res;
switch (m_state) {
case s_start:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_start";
if (m_receiver.onKey(m_key.data(), m_key.size()) != 0) {
m_state = s_error;
return 0;
}
m_stream = initCompressionStream();
m_state = s_forward;
// fallthrough //
CP_FALL_THROUGH;
case s_forward:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_forward";
res = decompressData(
m_stream,
len,
reinterpret_cast<const unsigned char *>(buf));
dbgTrace(D_WAAP_PARSER_GZIP) << "res: " << res.ok
<< ", size: " << res.num_output_bytes
<< ", is last: " << res.is_last_chunk;
if (!res.ok) {
m_state = s_error;
break;
}
if (res.num_output_bytes != 0 &&
m_receiver.onValue(reinterpret_cast<const char *>(res.output), res.num_output_bytes) != 0) {
m_state = s_error;
break;
}
if (res.is_last_chunk) {
m_state = s_done;
break;
}
break;
case s_done:
if (len > 0) {
dbgTrace(D_WAAP_PARSER_GZIP) << " unexpected data after completion, len=" << len;
m_state = s_error;
return 0; // Return 0 to indicate error
}
break;
case s_error:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_error";
return 0;
}
return len;
}
void ParserGzip::finish() {
push(NULL, 0);
if (m_state != s_done) {
m_state = s_error;
return;
}
}
const std::string &
ParserGzip::name() const {
return m_parserName;
}
bool ParserGzip::error() const {
return m_state == s_error;
}
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserGzip.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_GZIP);
const std::string ParserGzip::m_parserName = "ParserGzip";
ParserGzip::ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth)
:m_receiver(receiver), m_key("gzip"), m_state(s_start), m_stream(nullptr) {
}
ParserGzip::~ParserGzip() {
if (m_stream != nullptr) {
finiCompressionStream(m_stream);
m_stream = nullptr;
}
}
size_t ParserGzip::push(const char *buf, size_t len) {
dbgTrace(D_WAAP_PARSER_GZIP) << "len=" << (unsigned long int)len << ")";
if (len == 0) {
dbgTrace(D_WAAP_PARSER_GZIP) << "end of data signal! m_state=" << m_state;
// flush
if (m_state != s_start) { // only emit if at least something was pushed
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
}
}
return 0;
}
DecompressionResult res;
switch (m_state) {
case s_start:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_start";
if (m_receiver.onKey(m_key.data(), m_key.size()) != 0) {
m_state = s_error;
return 0;
}
m_stream = initCompressionStream();
m_state = s_forward;
// fallthrough //
CP_FALL_THROUGH;
case s_forward:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_forward";
res = decompressData(
m_stream,
len,
reinterpret_cast<const unsigned char *>(buf));
dbgTrace(D_WAAP_PARSER_GZIP) << "res: " << res.ok
<< ", size: " << res.num_output_bytes
<< ", is last: " << res.is_last_chunk;
if (!res.ok) {
m_state = s_error;
break;
}
if (res.num_output_bytes != 0 &&
m_receiver.onValue(reinterpret_cast<const char *>(res.output), res.num_output_bytes) != 0) {
m_state = s_error;
break;
}
if (res.is_last_chunk) {
m_state = s_done;
break;
}
break;
case s_done:
if (len > 0) {
dbgTrace(D_WAAP_PARSER_GZIP) << " unexpected data after completion, len=" << len;
m_state = s_error;
return 0; // Return 0 to indicate error
}
break;
case s_error:
dbgTrace(D_WAAP_PARSER_GZIP) << "s_error";
return 0;
}
return len;
}
void ParserGzip::finish() {
push(NULL, 0);
if (m_state != s_done) {
m_state = s_error;
return;
}
}
const std::string &
ParserGzip::name() const {
return m_parserName;
}
bool ParserGzip::error() const {
return m_state == s_error;
}

92
components/security_apps/waap/waap_clib/ParserGzip.h
View File

@@ -1,46 +1,46 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_GZIP_H_
#define __PARSER_GZIP_H_
#include "ParserBase.h"
#include <string.h>
#include "compression_utils.h"
class ParserGzip : public ParserBase {
public:
ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserGzip();
size_t push(const char *data, size_t data_len);
void finish();
virtual const std::string &name() const;
bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
s_start,
s_forward,
s_done,
s_error
};
IParserStreamReceiver &m_receiver;
std::string m_key;
state m_state;
CompressionStream * m_stream;
static const std::string m_parserName;
};
#endif // __PARSER_GZIP_H_
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_GZIP_H_
#define __PARSER_GZIP_H_
#include "ParserBase.h"
#include <string.h>
#include "compression_utils.h"
class ParserGzip : public ParserBase {
public:
ParserGzip(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserGzip();
size_t push(const char *data, size_t data_len);
void finish();
virtual const std::string &name() const;
bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
s_start,
s_forward,
s_done,
s_error
};
IParserStreamReceiver &m_receiver;
std::string m_key;
state m_state;
CompressionStream * m_stream;
static const std::string m_parserName;
};
#endif // __PARSER_GZIP_H_

2
components/security_apps/waap/waap_clib/ParserUrlEncode.cc
View File

@@ -92,7 +92,7 @@ ParserUrlEncode::push(const char *buf, size_t len)
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32) {
if (static_cast<unsigned char>(c) < 32u || static_cast<unsigned char>(c) > 126u) {
dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c;
m_state = s_error;
return i;

256
components/security_apps/waap/waap_clib/ScannerDetector.cc
View File

@@ -19,17 +19,20 @@ USE_DEBUG_FLAG(D_WAAP);
#define SYNC_WAIT_TIME std::chrono::seconds(300) // 5 minutes in seconds
#define INTERVAL std::chrono::minutes(120)
#define EQUAL_VALUES_COUNT_THRESHOLD 2
#define MAX_RETENTION 5
#define MAX_RETENTION 2
#define DEFAULT_MAX_SOURCES 256
ScannerDetector::ScannerDetector(const std::string& localPath, const std::string& remotePath,
ScannerDetector::ScannerDetector(const std::string &localPath, const std::string &remotePath,
const std::string &assetId) :
SerializeToLocalAndRemoteSyncBase(INTERVAL, SYNC_WAIT_TIME,
localPath + "/11.data",
(remotePath == "") ? remotePath : remotePath + "/ScannersDetector",
assetId,
"ScannerDetector")
"ScannerDetector"),
m_current_accumulator(std::make_shared<SourceKeyValsMap>()),
m_maxSources(getProfileAgentSettingWithDefault<uint>(-1, "scannerDetector.maxSources"))
{
m_sources_monitor.push_front(SourceKeyValsMap());
dbgTrace(D_WAAP) << "ScannerDetector constructor: m_maxSources set to " << m_maxSources;
}
bool ScannerDetector::ready()
@@ -47,25 +50,76 @@ std::vector<std::string>* ScannerDetector::getSourcesToIgnore()
return &m_sources;
}
void ScannerDetector::log(const std::string& source, const std::string& key, Waap::Keywords::KeywordsSet& keywords)
void ScannerDetector::log(
const std::string &source,
const std::string &key,
Waap::Keywords::KeywordsSet &keywords)
{
m_sources_monitor.front()[source][key].insert(keywords.begin(), keywords.end());
if (m_maxSources == uint(-1)) {
m_maxSources = getProfileAgentSettingWithDefault<uint>(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources");
dbgTrace(D_WAAP) << "log: m_maxSources set to " << m_maxSources;
}
// Add to accumulator for processing - same as original
(*m_current_accumulator)[source][key].insert(keywords.begin(), keywords.end());
// Optimized O(1) cache update - just add the key directly to the source cache
auto currentTime = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto cache_it = m_sourceCache.find(source);
if (cache_it != m_sourceCache.end()) {
// Source exists, just add the key - O(1) operation
cache_it->second.keys.insert(key);
cache_it->second.lastUpdate = currentTime;
cache_it->second.accessCount++;
// Move to front of LRU - O(1) operation
auto lru_it = m_lruMap.find(source);
if (lru_it != m_lruMap.end()) {
m_lruOrder.erase(lru_it->second);
m_lruOrder.push_front(source);
m_lruMap[source] = m_lruOrder.begin();
}
dbgTrace(D_WAAP) << "log: Updated existing source " << source << " with key " << key;
return;
}
// New source - check if cache is full
if (m_sourceCache.size() >= m_maxSources) {
evictLeastImportantSource();
}
// Add new source - O(1) operations
SourceInfo newSource(source, currentTime);
newSource.keys.insert(key);
m_sourceCache[source] = newSource;
// Add to front of LRU list - O(1) operation
m_lruOrder.push_front(source);
m_lruMap[source] = m_lruOrder.begin();
dbgTrace(D_WAAP) << "log: Added new source " << source << " with key " << key
<< " (cache size: " << m_sourceCache.size() << ")";
}
void ScannerDetector::loadParams(std::shared_ptr<Waap::Parameters::WaapParameters> pParams)
{
std::string interval = pParams->getParamVal("learnIndicators.intervalDuration",
std::to_string(INTERVAL.count()));
std::to_string(INTERVAL.count()));
setInterval(std::chrono::minutes(std::stoul(interval)));
std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true");
setRemoteSyncEnabled(!boost::iequals(remoteSyncStr, "false"));
m_maxSources = getProfileAgentSettingWithDefault<uint>(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources");
dbgTrace(D_WAAP) << "loadParams: m_maxSources set to " << m_maxSources;
}
class SourcesMonitorPost : public RestGetFile
{
public:
SourcesMonitorPost(ScannerDetector::SourceKeyValsMap& _monitor)
: monitor(_monitor)
SourcesMonitorPost(ScannerDetector::SourceKeyValsMap &_monitor)
: monitor(std::move(_monitor))
{
}
@@ -92,31 +146,31 @@ private:
bool ScannerDetector::postData()
{
m_sources_monitor_backup = m_sources_monitor.front();
m_sources_monitor.push_front(SourceKeyValsMap());
std::string url = getPostDataUrl();
dbgTrace(D_WAAP) << "Sending the data to: " << url;
SourcesMonitorPost currentWindow(m_sources_monitor_backup);
bool ok = sendNoReplyObjectWithRetry(currentWindow,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP) << "Failed to post collected data to: " << url;
if (m_current_accumulator->empty()) {
dbgDebug(D_WAAP) << "No data to post, skipping";
return true;
}
SourcesMonitorPost postMonitor(*m_current_accumulator);
bool ok = sendNoReplyObjectWithRetry(postMonitor,
HTTPMethod::PUT,
getPostDataUrl());
if (ok) {
m_current_accumulator = std::make_shared<SourceKeyValsMap>();
}
return ok;
}
void ScannerDetector::pullData(const std::vector<std::string>& files)
void ScannerDetector::pullData(const std::vector<std::string> &files)
{
std::string url = getPostDataUrl();
std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
dbgTrace(D_WAAP) << "pulling files, skipping: " << sentFile;
for (auto file : files)
for (const auto &file : files) // Use const reference
{
if (file == sentFile)
{
if (file == sentFile) {
continue;
}
dbgTrace(D_WAAP) << "Pulling the file: " << file;
@@ -131,113 +185,125 @@ void ScannerDetector::pullData(const std::vector<std::string>& files)
}
SourceKeyValsMap remoteMonitor = getMonitor.getSourcesMonitor().unpack();
for (const auto& srcData : remoteMonitor)
{
for (const auto& keyData : srcData.second)
{
m_sources_monitor_backup[srcData.first][keyData.first].insert(
keyData.second.begin(),
keyData.second.end());
}
}
// update the sources monitor in previous "time window"
auto temp = m_sources_monitor.front();
m_sources_monitor.pop_front();
m_sources_monitor.pop_front();
m_sources_monitor.push_front(m_sources_monitor_backup);
m_sources_monitor.push_front(temp);
mergeMonitors(*m_current_accumulator, remoteMonitor);
}
}
void ScannerDetector::postProcessedData()
{
// Empty implementation as in original
}
void ScannerDetector::updateState(const std::vector<std::string>&)
{
// Empty implementation as in original
}
void ScannerDetector::pullProcessedData(const std::vector<std::string>& files)
void ScannerDetector::pullProcessedData(const std::vector<std::string> &files)
{
(void)files;
}
void ScannerDetector::mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom)
void ScannerDetector::evictLeastImportantSource()
{
for (const auto& srcData : mergeFrom)
{
for (const auto& keyData : srcData.second)
{
dbgTrace(D_WAAP) << "merging src: " << srcData.first << ", key: " << keyData.first <<
", keywords: " << Waap::Util::setToString(keyData.second);
mergeTo[srcData.first][keyData.first].insert(keyData.second.begin(), keyData.second.end());
if (m_lruOrder.empty()) {
return;
}
// Enhanced eviction: scan last N sources in LRU and evict the one with the smallest key count
constexpr size_t N = 10; // Number of candidates to consider
auto it = m_lruOrder.rbegin();
auto it_end = m_lruOrder.rend();
size_t checked = 0;
std::string evictCandidate;
size_t minKeyCount = std::numeric_limits<size_t>::max();
auto candidateIt = m_lruOrder.rbegin();
for (; it != it_end && checked < N; ++it, ++checked) {
const std::string &source = *it;
auto cacheIt = m_sourceCache.find(source);
size_t keyCount = (cacheIt != m_sourceCache.end()) ? cacheIt->second.keys.size() : 0;
if (keyCount < minKeyCount) {
minKeyCount = keyCount;
evictCandidate = source;
candidateIt = it;
}
}
if (evictCandidate.empty()) {
// fallback to classic LRU
evictCandidate = m_lruOrder.back();
candidateIt = m_lruOrder.rbegin();
}
// Remove from all data structures - O(1) operations
m_sourceCache.erase(evictCandidate);
m_lruMap.erase(evictCandidate);
// Erase from m_lruOrder using base iterator
m_lruOrder.erase(std::next(candidateIt).base());
// Remove evicted source from current accumulator
m_current_accumulator->erase(evictCandidate);
dbgTrace(D_WAAP) << "evictLeastImportantSource: Evicted " << evictCandidate
<< " (key count: " << minKeyCount << ", cache size: " << m_sourceCache.size() << ")";
}
void ScannerDetector::mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom)
{
for (const auto &sourceEntry : mergeFrom) { // Use const reference
const std::string &source = sourceEntry.first;
for (const auto &keyEntry : sourceEntry.second) { // Use const reference
const std::string &key = keyEntry.first;
for (const auto &value : keyEntry.second) { // Use const reference
mergeTo[source][key].insert(value);
}
}
}
}
void ScannerDetector::processData()
{
if (m_sources_monitor_backup.empty())
{
m_sources_monitor_backup = m_sources_monitor.front();
m_sources_monitor.push_front(SourceKeyValsMap());
dbgTrace(D_WAAP) << "processData: Processing accumulated sources";
// Move current data to monitor deque for analysis
if (!m_current_accumulator->empty()) {
m_sources_monitor.push_front(m_current_accumulator);
m_current_accumulator = std::make_shared<SourceKeyValsMap>();
}
if (m_sources_monitor.size() > 2)
{
auto monitorItr = m_sources_monitor.begin()++;
for (monitorItr++; monitorItr != m_sources_monitor.end(); monitorItr++)
{
mergeMonitors(m_sources_monitor_backup, *monitorItr);
}
// Merge all monitors into a single monitor, but only include cached sources
SourceKeyValsMap mergedMonitor;
for (const auto &monitor : m_sources_monitor) {
mergeMonitors(mergedMonitor, *monitor);
}
if (m_sources_monitor.size() == MAX_RETENTION) {
m_sources_monitor.pop_back(); // Keep only the latest MAX_RETENTION cycles
}
// Analyze cached sources to identify scanners
m_sources.clear();
for (auto source : m_sources_monitor_backup)
{
if (source.second.size() <= 2)
{
continue;
}
std::map<std::string, std::set<std::string>>& keyVals = source.second;
for (auto key = keyVals.begin(); key != keyVals.end(); key++)
{
auto otherKey = key;
int counter = 0;
for (++otherKey; otherKey != keyVals.end(); otherKey++)
{
if (key->second != otherKey->second)
{
continue;
}
dbgTrace(D_WAAP) << "source monitor: src: " << source.first << ", key_1: " << key->first << ", key_2: "
<< otherKey->first << ", vals: " << Waap::Util::setToString(key->second);
counter++;
}
if (counter >= EQUAL_VALUES_COUNT_THRESHOLD)
{
dbgDebug(D_WAAP) << "source: " << source.first << " will be ignored";
m_sources.push_back(source.first);
break;
}
// Simple threshold-based scanner detection
const uint SCANNER_KEY_THRESHOLD = 3;
for (const auto &sourceInfo : mergedMonitor) {
const std::string &source = sourceInfo.first;
const auto &keys = sourceInfo.second;
if (keys.size() >= SCANNER_KEY_THRESHOLD) {
dbgTrace(D_WAAP) << "processData: Source " << source
<< " flagged as scanner (keyCount=" << keys.size() << ")";
m_sources.push_back(source);
}
}
if (m_sources_monitor.size() > MAX_RETENTION)
{
m_sources_monitor.pop_back();
}
m_sources_monitor_backup.clear();
dbgTrace(D_WAAP) << "processData: Found " << m_sources.size() << " scanners out of "
<< m_sourceCache.size() << " sources in cache";
m_lastSync = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
}
void ScannerDetector::serialize(std::ostream& stream)
void ScannerDetector::serialize(std::ostream &stream)
{
(void)stream;
}
void ScannerDetector::deserialize(std::istream& stream)
void ScannerDetector::deserialize(std::istream &stream)
{
(void)stream;
}

69
components/security_apps/waap/waap_clib/ScannersDetector.h
View File

@@ -11,45 +11,84 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __SCANNERS_DETECTOR_H__
#define __SCANNERS_DETECTOR_H__
#ifndef __OPTIMIZED_SCANNERS_DETECTOR_H__
#define __OPTIMIZED_SCANNERS_DETECTOR_H__
#include "WaapKeywords.h"
#include "i_serialize.h"
#include "i_ignoreSources.h"
#include "WaapParameters.h"
#include <chrono>
#include <unordered_map>
#include <unordered_set>
#include <list>
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
class ScannerDetector : public SerializeToLocalAndRemoteSyncBase, public I_IgnoreSources
{
public:
typedef std::map<std::string, std::map<std::string, std::set<std::string>>> SourceKeyValsMap;
ScannerDetector(const std::string& localPath, const std::string& remotePath = "", const std::string &assetId = "");
struct SourceInfo {
std::string source;
std::unordered_set<std::string> keys;
std::chrono::microseconds lastUpdate;
uint32_t accessCount; // Track access frequency for LFU eviction
// Default constructor for container requirements
SourceInfo() : lastUpdate(std::chrono::microseconds(0)), accessCount(0) {}
SourceInfo(const std::string &src, std::chrono::microseconds time)
: source(src), lastUpdate(time), accessCount(1) {}
uint getKeyCount() const { return keys.size(); }
void updateKeys(const std::set<std::string> &newKeys) {
keys.clear();
keys.insert(newKeys.begin(), newKeys.end());
lastUpdate = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch());
accessCount++;
}
};
ScannerDetector(
const std::string &localPath,
const std::string &remotePath = "",
const std::string &assetId = "");
virtual bool ready();
virtual std::vector<std::string>* getSourcesToIgnore();
void log(const std::string& source, const std::string& key, Waap::Keywords::KeywordsSet& keywords);
void log(const std::string &source, const std::string &key, Waap::Keywords::KeywordsSet &keywords);
void loadParams(std::shared_ptr<Waap::Parameters::WaapParameters> pParams);
virtual bool postData();
virtual void pullData(const std::vector<std::string>& files);
virtual void pullData(const std::vector<std::string> &files);
virtual void processData();
virtual void postProcessedData();
virtual void pullProcessedData(const std::vector<std::string>& files);
virtual void updateState(const std::vector<std::string>& files);
virtual void pullProcessedData(const std::vector<std::string> &files);
virtual void updateState(const std::vector<std::string> &files);
virtual void serialize(std::ostream& stream);
virtual void deserialize(std::istream& stream);
virtual void serialize(std::ostream &stream);
virtual void deserialize(std::istream &stream);
private:
void mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom);
std::list<SourceKeyValsMap> m_sources_monitor; // list of map source -> key -> set of indicators
SourceKeyValsMap m_sources_monitor_backup; // stores data of the last window to process
void evictLeastImportantSource();
void mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom);
// Optimized data structures
std::unordered_map<std::string, SourceInfo> m_sourceCache;
std::list<std::string> m_lruOrder;
std::unordered_map<std::string, std::list<std::string>::iterator> m_lruMap;
// Original data structures for compatibility
std::shared_ptr<SourceKeyValsMap> m_current_accumulator;
std::deque<std::shared_ptr<SourceKeyValsMap>> m_sources_monitor;
SourceKeyValsMap m_sources_monitor_backup;
std::vector<std::string> m_sources;
std::chrono::microseconds m_lastSync;
uint m_maxSources;
};
#endif

1
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -113,6 +113,7 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
restore();
}
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores) :
m_scoreTrigger(0),
m_fpStore(),

429
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -16,6 +16,7 @@
#include "Waf2Util.h"
#include "WaapAssetState.h"
#include "i_instance_awareness.h"
#include "buffered_compressed_stream.h"
#include <boost/regex.hpp>
#include <sstream>
#include <fstream>
@@ -33,7 +34,7 @@ USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
namespace ch = std::chrono;
using namespace std;
typedef ch::duration<size_t, std::ratio<86400>> days;
typedef ch::duration<size_t, ratio<86400>> days;
// Define interval between successful sync times
static const ch::minutes assetSyncTimeSliceLength(10);
@@ -44,26 +45,40 @@ static const string defaultSharedStorageHost = "appsec-shared-storage-svc";
#define SHARED_STORAGE_HOST_ENV_NAME "SHARED_STORAGE_HOST"
#define LEARNING_HOST_ENV_NAME "LEARNING_HOST"
void yieldIfPossible(const string& func, int line)
bool RestGetFile::loadJson(const string &json)
{
// Check if we are in the main loop
if (Singleton::exists<I_MainLoop>() &&
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->getCurrentRoutineId().ok())
{
// If we are not in the main loop, yield to allow other routines to run
// This is important for the main loop to be able to process other events
// and avoid blocking the entire system.
dbgTrace(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line;
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
// Try streaming approach first - handles both encryption and compression
try {
dbgTrace(D_WAAP_SERIALIZE) << "Attempting to use streaming approach for JSON loading, data size: "
<< json.size() << " bytes";
stringstream json_stream(json);
// if input json is big then yield to allow other routines to run
if (json.size() > 1000000) {
dbgTrace(D_WAAP_SERIALIZE) << "Input JSON is large, yielding to allow other routines to run";
YIELD_IF_POSSIBLE();
}
BufferedCompressedInputStream decompressed_stream(json_stream);
{
cereal::JSONInputArchive json_archive(decompressed_stream);
load(json_archive);
}
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded JSON using streaming approach";
return true;
}
catch (const exception &e) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach: " << e.what()
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
}
catch (...) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach"
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
}
}
#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__)
bool RestGetFile::loadJson(const string& json)
{
// Legacy approach: manual decryption and decompression
string json_str;
json_str = json;
if (!Waap::Util::isGzipped(json_str))
{
@@ -87,7 +102,7 @@ bool RestGetFile::loadJson(const string& json)
finiCompressionStream(compression_stream);
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Yielded after decompression in loadJson, decompressed size: "
dbgTrace(D_WAAP_SERIALIZE) << "Yielded after legacy decompression in loadJson, decompressed size: "
<< json_str.size() << " bytes";
return ClientRest::loadJson(json_str);
@@ -95,76 +110,42 @@ bool RestGetFile::loadJson(const string& json)
Maybe<string> RestGetFile::genJson() const
{
Maybe<string> json = ClientRest::genJson();
YIELD_IF_POSSIBLE();
if (json.ok())
stringstream output_stream;
try
{
string data = json.unpack();
// Get chunk size from profile settings for compression chunks
const size_t COMPRESSED_CHUNK_SIZE = static_cast<size_t>(
getProfileAgentSettingWithDefault<uint>(64 * 1024, "appsecLearningSettings.compressionChunkSize"));
auto compression_stream = initCompressionStream();
size_t offset = 0;
std::vector<unsigned char> compressed_data;
bool ok = true;
size_t chunk_count = 0;
// Process data in chunks for compression
while (offset < data.size()) {
size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
bool is_last = (offset + chunk_size >= data.size());
CompressionResult chunk_res = compressData(
compression_stream,
CompressionType::GZIP,
static_cast<uint32_t>(chunk_size),
reinterpret_cast<const unsigned char *>(data.c_str() + offset),
is_last ? 1 : 0
);
if (!chunk_res.ok) {
ok = false;
break;
}
if (chunk_res.output && chunk_res.num_output_bytes > 0) {
compressed_data.insert(
compressed_data.end(),
chunk_res.output,
chunk_res.output + chunk_res.num_output_bytes
);
free(chunk_res.output);
chunk_res.output = nullptr;
}
offset += chunk_size;
chunk_count++;
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Processed compression chunk " << chunk_count
<< ", progress: " << offset << "/" << data.size() << " bytes ("
<< (offset * 100 / data.size()) << "%) - yielded";
BufferedCompressedOutputStream compressed_out(output_stream);
{
cereal::JSONOutputArchive json_archive(compressed_out, cereal::JSONOutputArchive::Options::NoIndent());
save(json_archive);
}
finiCompressionStream(compression_stream);
dbgDebug(D_WAAP_SERIALIZE) << "Yielded after finalizing compression stream. "
<< "Total chunks: " << chunk_count << ", Compression ratio: "
<< (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x";
if (!ok) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to gzip data";
return genError("Failed to compress data");
}
// Create string from compressed data
string compressed_str(reinterpret_cast<const char*>(compressed_data.data()), compressed_data.size());
json = compressed_str;
compressed_out.close();
}
return json;
catch (const exception &e)
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to generate JSON: " << e.what();
return genError("Failed to generate JSON: " + string(e.what()));
}
return output_stream.str();
}
SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, string filePath) :
// Class to handle retrieving the state timestamp file from learning service
class StateTimestampRetriever : public ClientRest
{
public:
StateTimestampRetriever() {}
Maybe<string> getStateTimestamp() const
{
if (timestamp.get().empty()) {
return genError("State timestamp is empty");
}
return timestamp.get();
}
private:
S2C_PARAM(string, timestamp);
};
SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, const string &filePath) :
SerializeToFileBase(filePath),
m_lastSerialization(0),
m_interval(pollingIntervals)
@@ -210,7 +191,7 @@ void SerializeToFilePeriodically::setInterval(ch::seconds newInterval)
}
}
SerializeToFileBase::SerializeToFileBase(string fileName) : m_filePath(fileName)
SerializeToFileBase::SerializeToFileBase(const string &fileName) : m_filePath(fileName)
{
dbgTrace(D_WAAP_SERIALIZE) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath
<< "'";
@@ -243,7 +224,7 @@ void SerializeToFileBase::saveData()
if (maybe_routine.ok()) {
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
}
string data = ss.str();
const string &data = ss.str(); // Use const reference to avoid copying
dbgDebug(D_WAAP_SERIALIZE) << "Serialized data size: " << data.size() << " bytes";
// Get chunk size from profile settings, with default of 16 MiB for compression chunks
@@ -255,13 +236,13 @@ void SerializeToFileBase::saveData()
auto compression_stream = initCompressionStream();
size_t offset = 0;
std::vector<unsigned char> compressed_data;
vector<unsigned char> compressed_data;
bool ok = true;
size_t chunk_count = 0;
// Process data in chunks for compression
while (offset < data.size()) {
size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
size_t chunk_size = min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
bool is_last = (offset + chunk_size >= data.size());
CompressionResult chunk_res = compressData(
compression_stream,
@@ -319,7 +300,7 @@ void SerializeToFileBase::saveData()
size_t write_chunks = 0;
while (offset < data_to_write.size()) {
size_t current_chunk_size = std::min(CHUNK_SIZE, data_to_write.size() - offset);
size_t current_chunk_size = min(CHUNK_SIZE, data_to_write.size() - offset);
filestream.write(data_to_write.c_str() + offset, current_chunk_size);
offset += current_chunk_size;
write_chunks++;
@@ -334,7 +315,7 @@ void SerializeToFileBase::saveData()
<< " (" << data_to_write.size() << " bytes in " << write_chunks << " chunks)";
}
string decompress(string fileContent) {
string decompress(const string &fileContent) {
if (!Waap::Util::isGzipped(fileContent)) {
dbgTrace(D_WAAP_SERIALIZE) << "file note zipped";
return fileContent;
@@ -360,7 +341,7 @@ string decompress(string fileContent) {
return fileContent;
}
void SerializeToFileBase::loadFromFile(string filePath)
void SerializeToFileBase::loadFromFile(const string &filePath)
{
dbgTrace(D_WAAP_SERIALIZE) << "loadFromFile() file: " << filePath;
fstream filestream;
@@ -368,7 +349,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
filestream.open(filePath, fstream::in);
if (filestream.is_open() == false) {
dbgTrace(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " <<
dbgWarning(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " <<
strerror(errno);
if (!Singleton::exists<I_InstanceAwareness>() || errno != ENOENT)
{
@@ -387,34 +368,48 @@ void SerializeToFileBase::loadFromFile(string filePath)
size_t idPosition = filePath.find(idStr);
if (idPosition != string::npos)
{
filePath.erase(idPosition, idStr.length() - 1);
dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << filePath;
loadFromFile(filePath);
string modifiedFilePath = filePath; // Create a mutable copy
modifiedFilePath.erase(idPosition, idStr.length() - 1);
dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << modifiedFilePath;
loadFromFile(modifiedFilePath);
}
return;
}
dbgTrace(D_WAAP_SERIALIZE) << "loading from file: " << filePath;
int length;
filestream.seekg(0, ios::end); // go to the end
length = filestream.tellg(); // report location (this is the length)
try {
dbgTrace(D_WAAP_SERIALIZE) << "Attempting to load file using streaming approach";
BufferedCompressedInputStream decompressed_stream(filestream);
deserialize(decompressed_stream);
filestream.close();
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using streaming approach";
return;
}
catch (const exception &e) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load file using streaming approach: " << e.what()
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
filestream.clear();
filestream.seekg(0, ios::beg);
}
// Legacy approach: manual file reading, decryption, and decompression
filestream.seekg(0, ios::end);
int length = filestream.tellg();
dbgTrace(D_WAAP_SERIALIZE) << "file length: " << length;
assert(length >= 0); // length -1 really happens if filePath is a directory (!)
char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
vector<char> buffer(length); // Use vector instead of raw pointer for safety
filestream.seekg(0, ios::beg); // go back to the beginning
if (!filestream.read(buffer, length)) // read the whole file into the buffer
if (!filestream.read(buffer.data(), length)) // read the whole file into the buffer
{
filestream.close();
delete[] buffer;
dbgWarning(D_WAAP_SERIALIZE) << "Failed to read file, file: " << filePath;
return;
}
filestream.close();
string dataObfuscated(buffer, length);
delete[] buffer;
string dataObfuscated(buffer.begin(), buffer.end());
stringstream ss;
ss << decompress(dataObfuscated);
@@ -422,6 +417,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
try
{
deserialize(ss);
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using legacy approach";
}
catch (runtime_error & e) {
dbgWarning(D_WAAP_SERIALIZE) << "failed to deserialize file: " << m_filePath << ", error: " <<
@@ -441,7 +437,7 @@ RemoteFilesList::RemoteFilesList() : files(), filesPathsList()
// parses xml instead of json
// extracts a file list in <Contents><Key>
bool RemoteFilesList::loadJson(const string& xml)
bool RemoteFilesList::loadJson(const string &xml)
{
xmlDocPtr doc; // the resulting document tree
dbgTrace(D_WAAP_SERIALIZE) << "XML input: " << xml;
@@ -497,8 +493,8 @@ bool RemoteFilesList::loadJson(const string& xml)
}
contents_node = contents_node->next;
}
files.get().push_back(FileMetaData{ file, lastModified });
filesPathsList.push_back(file);
files.get().push_back(FileMetaData{ move(file), move(lastModified) });
filesPathsList.push_back(files.get().back().filename); // Use the moved string to avoid extra copy
}
node = node->next;
}
@@ -511,12 +507,12 @@ bool RemoteFilesList::loadJson(const string& xml)
return true;
}
const vector<string>& RemoteFilesList::getFilesList() const
const vector<string> &RemoteFilesList::getFilesList() const
{
return filesPathsList;
}
const vector<FileMetaData>& RemoteFilesList::getFilesMetadataList() const
const vector<FileMetaData> &RemoteFilesList::getFilesMetadataList() const
{
return files.get();
}
@@ -547,6 +543,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
{
dbgInfo(D_WAAP_SERIALIZE) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId <<
"', owner='" << m_owner << "'";
m_pMainLoop = Singleton::Consume<I_MainLoop>::by<WaapComponent>();
if (Singleton::exists<I_AgentDetails>() &&
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() ==
@@ -586,11 +583,10 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_type = type;
}
}
m_pMainLoop = Singleton::Consume<I_MainLoop>::by<WaapComponent>();
setInterval(interval);
}
bool SerializeToLocalAndRemoteSyncBase::isBase()
bool SerializeToLocalAndRemoteSyncBase::isBase() const
{
return m_remotePath == "";
}
@@ -656,13 +652,12 @@ void SerializeToLocalAndRemoteSyncBase::setRemoteSyncEnabled(bool enabled)
void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
{
dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " <<
newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'";
if (newInterval == m_interval)
{
return;
}
dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " <<
newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'";
m_interval = newInterval;
@@ -730,7 +725,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
m_pMainLoop->yield(remainingTime);
timeBeforeSyncWorker = timer->getWalltime();
m_pMainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::System, [this]() {syncWorker();}, "Sync worker");
syncWorker();
timeAfterSyncWorker = timer->getWalltime();
}
};
@@ -780,32 +775,117 @@ ch::seconds SerializeToLocalAndRemoteSyncBase::getIntervalDuration() const
return m_interval;
}
Maybe<string> SerializeToLocalAndRemoteSyncBase::getStateTimestampByListing()
{
RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
if (remoteFiles.getFilesMetadataList().empty())
{
return genError("No remote processed files available");
}
dbgDebug(D_WAAP_SERIALIZE) << "State timestamp by listing: "
<< remoteFiles.getFilesMetadataList()[0].modified;
return remoteFiles.getFilesMetadataList()[0].modified;
}
bool SerializeToLocalAndRemoteSyncBase::checkAndUpdateStateTimestamp(const string& currentStateTimestamp)
{
// Check if the state has been updated since last check
if (currentStateTimestamp != m_lastProcessedModified)
{
m_lastProcessedModified = currentStateTimestamp;
dbgDebug(D_WAAP_SERIALIZE) << "State timestamp updated: " << m_lastProcessedModified;
return true; // State was updated
}
return false; // State unchanged
}
void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService()
{
bool useFallbackMethod = false;
for (int i = 0; i < remoteSyncMaxPollingAttempts; i++)
{
m_pMainLoop->yield(ch::seconds(60));
RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
if (remoteFiles.getFilesMetadataList().empty())
// Try the dedicated timestamp file first
Maybe<string> timestampResult(genError("Failed to get state timestamp"));
if (!useFallbackMethod) {
timestampResult = getStateTimestamp();
if (!timestampResult.ok()) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp from file: "
<< timestampResult.getErr() << ", trying listing method";
useFallbackMethod = true; // Switch to listing method on first failure
}
}
else
{
dbgWarning(D_WAAP_SERIALIZE) << "no files generated by the remote service were found";
dbgDebug(D_WAAP_SERIALIZE) << "trying listing method";
timestampResult = getStateTimestampByListing();
}
if (!timestampResult.ok())
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp using any method: "
<< timestampResult.getErr();
continue;
}
string lastModified = remoteFiles.getFilesMetadataList().begin()->modified;
if (lastModified != m_lastProcessedModified)
string currentStateTimestamp = timestampResult.unpack();
if (checkAndUpdateStateTimestamp(currentStateTimestamp))
{
m_lastProcessedModified = lastModified;
updateState(remoteFiles.getFilesList());
dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner <<
". updated state generated by remote at " << m_lastProcessedModified;
// Update state directly from the known remote file path
updateStateFromRemoteFile();
dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner
<< ". updated state using " << (useFallbackMethod ? "file listing (fallback)" : "timestamp file")
<< ": " << m_lastProcessedModified;
return;
}
else
{
dbgWarning(D_WAAP_SERIALIZE) << "State timestamp unchanged ("
<< (useFallbackMethod ? "file listing (fallback)" : "timestamp file") << "): "
<< currentStateTimestamp;
}
}
dbgWarning(D_WAAP_SERIALIZE) << "polling for update state timeout. for assetId='"
// All polling attempts failed - fall back to local sync
dbgWarning(D_WAAP_SERIALIZE) << "Polling for update state timeout, falling back to local sync. for assetId='"
<< m_assetId << "', owner='" << m_owner;
localSyncAndProcess();
}
Maybe<void> SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteFile()
{
auto maybeRemoteFilePath = getRemoteStateFilePath();
if (!maybeRemoteFilePath.ok())
{
string error = "Owner: " + m_owner + ", no remote state file path defined: " + maybeRemoteFilePath.getErr();
dbgWarning(D_WAAP_SERIALIZE) << error;
return genError(error);
}
string remoteFilePath = maybeRemoteFilePath.unpack();
vector<string> files = {remoteFilePath};
updateState(files);
dbgDebug(D_WAAP_SERIALIZE) << "updated state from remote file: " << remoteFilePath;
return Maybe<void>();
}
bool SerializeToLocalAndRemoteSyncBase::shouldNotSync() const
{
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
return mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase();
}
bool SerializeToLocalAndRemoteSyncBase::shouldSendSyncNotification() const
{
return getSettingWithDefault<bool>(true, "features", "learningLeader") &&
((m_type == "CentralizedData") ==
(getProfileAgentSettingWithDefault<bool>(false, "agent.learning.centralLogging")));
}
void SerializeToLocalAndRemoteSyncBase::syncWorker()
{
dbgInfo(D_WAAP_SERIALIZE) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
@@ -814,7 +894,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() || !postData()) {
if (shouldNotSync() || !postData()) {
dbgDebug(D_WAAP_SERIALIZE)
<< "Did not synchronize the data. for asset: "
<< m_assetId
@@ -834,12 +914,31 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
if (m_lastProcessedModified == "")
{
dbgTrace(D_WAAP_SERIALIZE) << "check if remote service is operational";
RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
if (!remoteFiles.getFilesMetadataList().empty())
Maybe<string> maybeTimestamp = getStateTimestamp();
if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty())
{
m_lastProcessedModified = remoteFiles.getFilesMetadataList()[0].modified;
m_lastProcessedModified = maybeTimestamp.unpack();
dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service: " << m_lastProcessedModified;
}
else
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp from remote service: "
<< maybeTimestamp.getErr();
maybeTimestamp = getStateTimestampByListing();
if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty())
{
m_lastProcessedModified = maybeTimestamp.unpack();
dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service using listing: " << m_lastProcessedModified;
}
else
{
dbgWarning(D_WAAP_SERIALIZE)
<< "Failed to get state timestamp from remote service by listing: "
<< maybeTimestamp.getErr()
<< " skipping syncWorker for assetId='"
<< m_assetId << "', owner='" << m_owner << "'";
}
}
}
// check if learning service is enabled
@@ -854,6 +953,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
return;
}
// TODO: add should send sync notification function (e.g. do not sync if not leader)
if (mode == OrchestrationMode::HYBRID) {
dbgDebug(D_WAAP_SERIALIZE) << "detected running in standalone mode";
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
@@ -876,7 +977,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
if (!ok) {
dbgWarning(D_WAAP_SERIALIZE) << "failed to send learning notification";
}
} else {
} else if (shouldSendSyncNotification())
{
SyncLearningNotificationObject syncNotification(m_assetId, m_type, getWindowId());
dbgDebug(D_WAAP_SERIALIZE) << "sending sync notification: " << syncNotification;
@@ -941,7 +1043,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
if (!processedFilesList.getFilesList().empty())
{
const vector<FileMetaData>& filesMD = processedFilesList.getFilesMetadataList();
const vector<FileMetaData> &filesMD = processedFilesList.getFilesMetadataList();
if (filesMD.size() > 1) {
dbgWarning(D_WAAP_SERIALIZE) << "got more than 1 expected processed file";
}
@@ -1014,8 +1116,40 @@ void SerializeToLocalAndRemoteSyncBase::mergeProcessedFromRemote()
I_MainLoop::RoutineType::Offline,
[&]()
{
RemoteFilesList processedFiles = getProcessedFilesList();
pullProcessedData(processedFiles.getFilesList());
// Instrumentation breadcrumbs to help diagnose startup crash inside this routine
dbgTrace(D_WAAP_SERIALIZE) << "start routine for assetId='" << m_assetId
<< "', owner='" << m_owner << "'";
try {
auto success = updateStateFromRemoteFile();
if (!success.ok()) {
dbgInfo(D_WAAP_SERIALIZE) << "direct state file unavailable: "
<< success.getErr() << ". Falling back to listing.";
RemoteFilesList remoteFiles = getProcessedFilesList();
if (remoteFiles.getFilesList().empty()) {
dbgWarning(D_WAAP_SERIALIZE) << "no remote processed files";
return;
}
const auto &md_list = remoteFiles.getFilesMetadataList();
if (!md_list.empty()) {
m_lastProcessedModified = md_list[0].modified;
} else {
dbgWarning(D_WAAP_SERIALIZE) << "metadata list empty while files list not empty";
}
updateState(remoteFiles.getFilesList());
dbgInfo(D_WAAP_SERIALIZE) << "updated state from remote files. Last modified: "
<< m_lastProcessedModified;
} else {
dbgTrace(D_WAAP_SERIALIZE) << "updated state via direct remote file";
}
} catch (const JsonError &j) {
dbgError(D_WAAP_SERIALIZE) << "JsonError caught: '" << j.getMsg()
<< "' assetId='" << m_assetId << "' owner='" << m_owner << "'";
throw std::runtime_error(std::string("mergeProcessedFromRemote JsonError: ") + j.getMsg());
} catch (const std::exception &e) {
dbgError(D_WAAP_SERIALIZE) << "std::exception caught: " << e.what()
<< " assetId='" << m_assetId << "' owner='" << m_owner << "'";
throw; // Let mainloop handle termination with detailed message
}
},
"Merge processed data from remote for asset Id: " + m_assetId + ", owner:" + m_owner
);
@@ -1052,3 +1186,32 @@ SerializeToLocalAndRemoteSyncBase::getSharedStorageHost()
}
return defaultSharedStorageHost;
}
string SerializeToLocalAndRemoteSyncBase::getStateTimestampPath()
{
return m_remotePath + "/internal/lastModified.data";
}
Maybe<string> SerializeToLocalAndRemoteSyncBase::getStateTimestamp()
{
string timestampPath = getStateTimestampPath();
if (timestampPath.empty()) {
dbgWarning(D_WAAP_SERIALIZE) << "Cannot get state timestamp - invalid path";
return genError("Invalid timestamp path");
}
StateTimestampRetriever timestampRetriever;
bool isSuccessful = sendObject(
timestampRetriever,
HTTPMethod::GET,
getUri() + "/" + timestampPath);
if (!isSuccessful) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp file from: " << timestampPath;
return genError("Failed to retrieve timestamp file from: " + timestampPath);
}
dbgDebug(D_WAAP_SERIALIZE) << "Retrieved state timestamp: " << timestampRetriever.getStateTimestamp().unpack()
<< " from path: " << timestampPath;
return timestampRetriever.getStateTimestamp().unpack();
}

468
components/security_apps/waap/waap_clib/Signatures.cc
View File

@@ -12,18 +12,24 @@
// limitations under the License.
#include "Signatures.h"
#include "AssertionRegexes.h"
#include "agent_core_utilities.h"
#include "debug.h"
#include "waap.h"
#include <cstdlib> // for getenv
#include <cstring> // for strcmp
#include <fstream>
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN);
USE_DEBUG_FLAG(D_WAAP_HYPERSCAN);
typedef picojson::value::object JsObj;
typedef picojson::value JsVal;
typedef picojson::value::array JsArr;
typedef std::map<std::string, std::vector<std::string>> filtered_parameters_t;
static std::vector<std::string> to_strvec(const picojson::value::array& jsV)
static std::vector<std::string> to_strvec(const picojson::value::array &jsV)
{
std::vector<std::string> r;
@@ -34,7 +40,7 @@ static std::vector<std::string> to_strvec(const picojson::value::array& jsV)
return r;
}
static std::set<std::string> to_strset(const picojson::value::array& jsA)
static std::set<std::string> to_strset(const picojson::value::array &jsA)
{
std::set<std::string> r;
@@ -45,18 +51,18 @@ static std::set<std::string> to_strset(const picojson::value::array& jsA)
return r;
}
static std::map<std::string, Regex*> to_regexmap(const picojson::value::object& jsO, bool& error)
static std::map<std::string, Regex *> to_regexmap(const picojson::value::object &jsO, bool &error)
{
std::map<std::string, Regex*> r;
std::map<std::string, Regex *> r;
for (auto it = jsO.begin(); it != jsO.end(); ++it) {
const std::string& n = it->first;
const std::string &n = it->first;
// convert name to lowercase now (so we don't need to do it at runtime every time).
std::string n_lower;
for (std::string::const_iterator pCh = n.begin(); pCh != n.end(); ++pCh) {
n_lower += std::tolower(*pCh);
}
const picojson::value& v = it->second;
const picojson::value &v = it->second;
if (error) {
// stop loading regexes if there's previous error...
@@ -73,13 +79,12 @@ static std::map<std::string, Regex*> to_regexmap(const picojson::value::object&
return r;
}
static filtered_parameters_t to_filtermap(const picojson::value::object& JsObj)
static filtered_parameters_t to_filtermap(const picojson::value::object &JsObj)
{
filtered_parameters_t result;
for (auto it = JsObj.begin(); it != JsObj.end(); ++it)
{
for (auto it = JsObj.begin(); it != JsObj.end(); ++it) {
const std::string parameter = it->first;
const picojson::value::array& arr = it->second.get<picojson::value::array>();
const picojson::value::array &arr = it->second.get<picojson::value::array>();
result[parameter] = to_strvec(arr);
}
return result;
@@ -215,9 +220,14 @@ Signatures::Signatures(const std::string& filepath) :
to_strset(sigsSource["remove_keywords_always"].get<JsArr>())),
user_agent_prefix_re(sigsSource["user_agent_prefix_re"].get<std::string>()),
binary_data_kw_filter(sigsSource["binary_data_kw_filter"].get<std::string>()),
wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get<std::string>())
wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get<std::string>()),
m_hyperscanInitialized(false)
{
// Only preprocess hyperscan patterns if hyperscan is enabled
bool should_use_hyperscan = Signatures::shouldUseHyperscan();
if (should_use_hyperscan) {
preprocessHyperscanPatterns();
}
}
Signatures::~Signatures()
@@ -229,21 +239,299 @@ bool Signatures::fail()
return error;
}
picojson::value::object Signatures::loadSource(const std::string& waapDataFileName)
// Static helper to process assertion flags for a pattern (for testing and internal use)
std::string
Signatures::processAssertions(const std::string &groupName, const std::string &pattern, AssertionFlags &flags)
{
std::string processed = pattern;
// Use regexes from AssertionRegexes namespace to detect assertions at start/end of the pattern string
using namespace Waap::AssertionRegexes;
boost::smatch match;
// Start assertions - only a single '(' can precede
if (boost::regex_search(processed, match, reStartNonWordBehind) && match.position() >= 0) {
flags.setFlag(AssertionFlag::START_NON_WORD_BEHIND);
processed = boost::regex_replace(processed, reStartNonWordBehind, std::string(""));
}
// Path traversal start assertion
if (boost::regex_search(processed, match, rePathTraversalStart) && match.position() >= 0) {
flags.setFlag(AssertionFlag::PATH_TRAVERSAL_START);
processed = boost::regex_replace(processed, rePathTraversalStart, std::string(""));
}
// End assertions - only a single ')' can follow
if (boost::regex_search(processed, match, reEndNonWordAhead) && match.position() >= 0) {
flags.setFlag(AssertionFlag::END_NON_WORD_AHEAD);
processed = boost::regex_replace(processed, reEndNonWordAhead, std::string(""));
} else if (boost::regex_search(processed, match, reEndNonWordSpecial) && match.position() >= 0) {
flags.setFlag(AssertionFlag::END_NON_WORD_SPECIAL);
processed = boost::regex_replace(processed, reEndNonWordSpecial, std::string(""));
}
// Path traversal end assertion
if (boost::regex_search(processed, match, rePathTraversalEnd) && match.position() >= 0) {
flags.setFlag(AssertionFlag::PATH_TRAVERSAL_END);
processed = boost::regex_replace(processed, rePathTraversalEnd, std::string(""));
}
// wildcard evasion regex group name starts with evasion_wildcard_regex
if (groupName.find("evasion_wildcard_regex") == 0) {
flags.setFlag(AssertionFlag::WILDCARD_EVASION);
}
return processed;
}
// Extracts the group name from a regex pattern string (e.g., (?P<groupName>...))
std::string Signatures::extractGroupName(const std::string &pattern) {
boost::regex namedGroupRegex(R"(\(\?P<([^>]+)>)");
boost::smatch match;
if (boost::regex_search(pattern, match, namedGroupRegex)) {
return match[1].str();
}
return "";
}
void Signatures::preprocessHyperscanPatterns()
{
std::map<std::string, size_t> categoryCount;
// Helper function to check if a pattern is hyperscan compatible
auto isHyperscanCompatible = [&categoryCount](const std::string &pattern) -> bool {
// Hyperscan doesn't support certain regex features that we can't easily convert
static const std::vector<std::string> incompatibleFeatures = {
R"((?!\w)", R"((?<!\w)", R"((?=\w)", R"((?<=\w)", // Lookahead/lookbehind assertions for \w
R"((?!)", R"((?<!)", R"((?=)", R"((?<=)", // Lookahead/lookbehind assertions
R"((?>)", R"((?&)", R"((?|)", R"((?P<)", // Atomic groups, named groups, and branching
R"((?R)" // Recursion
};
for (const auto &feature : incompatibleFeatures) {
if (pattern.find(feature) != std::string::npos) {
dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible feature found: " << feature << " in pattern: " << pattern;
categoryCount[feature]++;
return false;
}
}
boost::regex backrefRegex(R"(\(\\\d+\))");
if (boost::regex_search(pattern, backrefRegex)) {
dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible backreference found: " << pattern;
categoryCount["backreference"]++;
return false;
}
return true;
};
// Helper function to convert regex pattern to hyperscan-compatible format
auto convertToHyperscanPattern = [](const std::string &originalPattern) -> std::string {
std::string converted = originalPattern;
// Remove named group syntax - convert (?P<name>...) to ...
boost::regex namedGroupRegex(R"(\(\?P<[^>]+>)");
if (boost::regex_search(converted, namedGroupRegex)) {
std::string end_str = ")";
if (converted.back() == ')') {
converted.pop_back(); // Remove the trailing ')'
end_str = "";
}
converted = boost::regex_replace(converted, namedGroupRegex, end_str);
}
// Handle atomic groups first (before removing word boundaries)
// Hyperscan doesn't support atomic groups, so we need to convert them
// Convert (?>\b) to nothing (remove word boundary atomic groups)
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\b\))"), std::string(""));
// Convert (?>\B) to nothing (remove non-word boundary atomic groups)
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\B\))"), std::string(""));
// Convert empty atomic groups (?>) to nothing
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\))"), std::string(""));
// // Now remove remaining word boundaries (not supported by Hyperscan)
// // At this point, any \b or \B that was inside atomic groups has been handled above
// converted = boost::regex_replace(converted, boost::regex(R"(\\b)"), std::string(""));
// converted = boost::regex_replace(converted, boost::regex(R"(\\B)"), std::string(""));
return converted;
};
// Helper function to get patterns from sigsSource for each category
auto getCommonPatternsForCategory = [this](const std::string &category,
const std::string &regexSource) -> std::vector<std::string> {
std::vector<std::string> patterns;
// Map regexSource/category to the JSON key in sigsSource
std::string key;
if (regexSource == "specific_acuracy_keywords_regex" || category == "specific_accuracy") {
key = "specific_acuracy_keywords_regex_list";
} else if (regexSource == "words_regex" || category == "keywords") {
key = "words_regex_list";
} else if (regexSource == "pattern_regex" || category == "patterns") {
key = "pattern_regex_list";
} else {
// Fallback: allow passing the exact key name
key = regexSource;
dbgDebug(D_WAAP_HYPERSCAN) << "Unknown category/regexSource: " << category << "/" << regexSource
<< ". Using regexSource as key.";
}
// Fetch patterns directly from sigsSource if available
auto it = sigsSource.find(key);
if (it != sigsSource.end()) {
try {
patterns = to_strvec(it->second.get<JsArr>());
} catch (...) {
// If the type is unexpected, return empty and continue gracefully
patterns.clear();
dbgWarning(D_WAAP_HYPERSCAN) << "Unexpected type for key: " << key;
}
}
return patterns;
};
// Process specific_acuracy_keywords_regex patterns
std::vector<std::string> incompatiblePatterns;
{
auto patterns = getCommonPatternsForCategory("specific_accuracy", "specific_acuracy_keywords_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "specific_accuracy";
hsPattern.regexSource = "specific_acuracy_keywords_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "specific_accuracy_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_keywordHyperscanPatterns.push_back(hsPattern);
m_keywordAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
// Process words_regex patterns
{
auto patterns = getCommonPatternsForCategory("keywords", "words_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "keywords";
hsPattern.regexSource = "words_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "keywords_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_keywordHyperscanPatterns.push_back(hsPattern);
m_keywordAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
// Process pattern_regex patterns
{
auto patterns = getCommonPatternsForCategory("patterns", "pattern_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "patterns";
hsPattern.regexSource = "pattern_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "patterns_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_patternHyperscanPatterns.push_back(hsPattern);
m_patternAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
dbgInfo(D_WAAP_HYPERSCAN) << "Preprocessed Hyperscan patterns: "
<< "keywords=" << m_keywordHyperscanPatterns.size()
<< ", patterns=" << m_patternHyperscanPatterns.size()
<< ", incompatible=" << incompatiblePatterns.size();
for (const auto &it : categoryCount) {
dbgInfo(D_WAAP_HYPERSCAN) << "Feature: " << it.first << ", Count: " << it.second;
}
// Convert incompatible patterns to PmWordSet for traditional regex processing
if (m_regexPreconditions && !incompatiblePatterns.empty()) {
for (const auto &pattern : incompatiblePatterns) {
Waap::RegexPreconditions::WordIndex wordIndex = m_regexPreconditions->getWordByRegex(pattern);
if (wordIndex != Waap::RegexPreconditions::emptyWordIndex) {
m_incompatiblePatternsPmWordSet.insert(wordIndex);
}
}
dbgInfo(D_WAAP_HYPERSCAN) << "Created PmWordSet for " << m_incompatiblePatternsPmWordSet.size()
<< " incompatible patterns (from " << incompatiblePatterns.size() << " total)";
}
}
picojson::value::object Signatures::loadSource(const std::string &waapDataFileName)
{
picojson::value doc;
std::ifstream f(waapDataFileName);
if (f.fail()) {
dbgError(D_WAAP) << "Failed to open json data file '" << waapDataFileName << "'!";
error = true; // flag an error
error = true; // flag an error
return picojson::value::object();
}
int length;
f.seekg(0, std::ios::end); // go to the end
length = f.tellg(); // report location (this is the length)
char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
char *buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
f.seekg(0, std::ios::beg); // go back to the beginning
f.read(buffer, length); // read the whole file into the buffer
f.close();
@@ -258,11 +546,153 @@ picojson::value::object Signatures::loadSource(const std::string& waapDataFileNa
ss >> doc;
if (!picojson::get_last_error().empty()) {
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" <<
picojson::get_last_error() << "').";
error = true; // flag an error
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '"
<< picojson::get_last_error() << "').";
error = true; // flag an error
return picojson::value::object();
}
return doc.get<picojson::value::object>()["waap_signatures"].get<picojson::value::object>();
}
const std::vector<Signatures::HyperscanPattern> &Signatures::getKeywordHyperscanPatterns() const
{
return m_keywordHyperscanPatterns;
}
const std::vector<Signatures::HyperscanPattern> &Signatures::getPatternHyperscanPatterns() const
{
return m_patternHyperscanPatterns;
}
const std::vector<Signatures::AssertionFlags> &Signatures::getKeywordAssertionFlags() const
{
return m_keywordAssertionFlags;
}
const std::vector<Signatures::AssertionFlags> &Signatures::getPatternAssertionFlags() const
{
return m_patternAssertionFlags;
}
const Waap::RegexPreconditions::PmWordSet &Signatures::getIncompatiblePatternsPmWordSet() const
{
return m_incompatiblePatternsPmWordSet;
}
void Signatures::processRegexMatch(const std::string &groupName, const std::string &groupValue, std::string &word,
std::vector<std::string> &keyword_matches,
Waap::Util::map_of_stringlists_t &found_patterns, bool longTextFound,
bool binaryDataFound) const
{
std::string group = groupName;
if (group == "") {
return; // skip unnamed group
}
const std::string &value = groupValue;
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: group name='" << group << "' value='" << value << "', word='" << word
<< "':";
if (group.find("fast_reg") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found '*fast_reg*' in group name";
if (group.find("evasion") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found both 'fast_reg' and 'evasion' in group name.";
word = "encoded_" + repr_uniq(value);
if (word == "encoded_") {
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "checkRegex: empty word after repr_uniq: resetting word to 'character_encoding'"
" and group to 'evasion'.";
word = "character_encoding";
} else if (Waap::Util::str_isalnum(word)) {
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "checkRegex: isalnum word after repr_uniq: resetting group to 'evasion'.";
// If the found match is alphanumeric (we've seen strings like "640x480" match)
// we still should assume evasion but it doesn't need to include "fast_reg",
// which would cause unconditional report to stage2 and hit performance...
// This is why we remove the word "fast_reg" from the group name.
group = "evasion";
}
if (longTextFound) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: longTextFound so resetting group name to 'longtext'";
group = "longtext";
}
} else {
word = group;
}
}
// In sequences detected as "longTextFound" or "longBinaryFound", do not add words in the
// "keyword_matches" list that:
// - starts with "encoded_"
// - or startswith("\")
// - or equal to "character_encoding"
if ((longTextFound || binaryDataFound) &&
(word == "character_encoding" || word.substr(0, 1) == "\\" || word.substr(0, 8) == "encoded_")) {
// For now, do not skip
// TODO - check if skipping improves detection
dbgTrace(D_WAAP_SAMPLE_SCAN) << "longText/binaryData found with character_encoding";
} else if (binaryDataFound && (isShortWord(word) || isShortHtmlTag(word) ||
NGEN::Regex::regexMatch(__FILE__, __LINE__, group, binary_data_kw_filter))) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding group='" << group << "', word='" << word
<< "' - due to binary data";
return;
} else if ((std::find(keyword_matches.begin(), keyword_matches.end(), word) == keyword_matches.end())) {
// python: if (word not in current_matches): current_matches.append(word)
keyword_matches.push_back(word);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "added keyword match for group='" << group << "', value='" << value
<< "', word='" << word << "'";
}
// python:
// if group not in found_patterns:
// found_patterns[group]=[]
if (found_patterns.find(group) == found_patterns.end()) {
found_patterns[group] = std::vector<std::string>();
}
// python:
// if value not in found_patterns[group]:
// found_patterns[group].append(value)
if (std::find(found_patterns[group].begin(), found_patterns[group].end(), value) == found_patterns[group].end()) {
found_patterns[group].push_back(value);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "added pattern match for group='" << group << "', value='" << value
<< "', word='" << word << "'";
}
}
bool Signatures::isHyperscanInitialized() const
{
return m_hyperscanInitialized;
}
void Signatures::setHyperscanInitialized(bool initialized)
{
m_hyperscanInitialized = initialized;
}
bool Signatures::shouldUseHyperscan(bool force)
{
// This can be controlled by environment variable or configuration
static bool useHyperscan = false;
#ifdef USE_HYPERSCAN
static bool checked = false;
if (!checked || force) {
// Check environment variable first
const char *env = getenv("WAAP_USE_HYPERSCAN");
if (env) {
useHyperscan = (strcmp(env, "1") == 0 || strcasecmp(env, "true") == 0);
dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage set by environment: " << useHyperscan;
} else {
// Default to false to maintain backward compatibility - Hyperscan is opt-in
useHyperscan = false;
dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage default (disabled): " << useHyperscan;
}
checked = true;
}
#endif // USE_HYPERSCAN
return useHyperscan;
}

134
components/security_apps/waap/waap_clib/Signatures.h
View File

@@ -16,19 +16,40 @@
#include "Waf2Regex.h"
#include "picojson.h"
#include "flags.h"
#include <boost/regex.hpp>
class Signatures {
private:
// json parsed sources (not really needed once data is loaded)
picojson::value::object sigsSource;
bool error;
public:
// Enum for zero-length assertion flags
enum class AssertionFlag {
START_WORD_BEHIND = 0, // (?<=\w)
START_NON_WORD_BEHIND, // (?<!\w)
END_WORD_AHEAD, // (?=\w)
END_NON_WORD_AHEAD, // (?!\w)
END_NON_WORD_SPECIAL, // (?=[^\w?<>:=]|$)
PATH_TRAVERSAL_START, // (?<![\.,:])
PATH_TRAVERSAL_END, // (?![\.,:])
WILDCARD_EVASION, // (slashes and question mark must be present)
COUNT // Must be last for Flags template
};
// Use the Flags template from utilities/flags.h for assertion flags
using AssertionFlags = Flags<AssertionFlag>;
static std::string extractGroupName(const std::string &pattern);
static std::string processAssertions(const std::string &groupName,
const std::string &pattern,
AssertionFlags &flags);
Signatures(const std::string& filepath);
~Signatures();
bool fail();
picojson::value::object sigsSource;
bool error;
std::shared_ptr<Waap::RegexPreconditions> m_regexPreconditions;
// Regexes loaded from compiled signatures
@@ -81,8 +102,113 @@ public:
const boost::regex binary_data_kw_filter;
const boost::regex wbxml_data_kw_filter;
// Pre-compiled Hyperscan patterns and metadata for performance optimization
struct HyperscanPattern {
std::string originalPattern;
std::string hyperscanPattern;
std::string groupName;
std::string category;
std::string regexSource;
bool isFastReg;
bool isEvasion;
HyperscanPattern() : isFastReg(false), isEvasion(false) {}
};
// Pre-processed hyperscan patterns for each regex category
std::vector<HyperscanPattern> m_keywordHyperscanPatterns;
std::vector<HyperscanPattern> m_patternHyperscanPatterns;
// Assertion flags corresponding to each pattern (same indices as above vectors)
std::vector<AssertionFlags> m_keywordAssertionFlags;
std::vector<AssertionFlags> m_patternAssertionFlags;
// Getter methods for precompiled patterns
const std::vector<HyperscanPattern>& getKeywordHyperscanPatterns() const;
const std::vector<HyperscanPattern>& getPatternHyperscanPatterns() const;
// Getter methods for assertion flags
const std::vector<AssertionFlags>& getKeywordAssertionFlags() const;
const std::vector<AssertionFlags>& getPatternAssertionFlags() const;
// PmWordSet for incompatible patterns that need to use traditional regex scanning
Waap::RegexPreconditions::PmWordSet m_incompatiblePatternsPmWordSet;
// Getter method for incompatible patterns PmWordSet
const Waap::RegexPreconditions::PmWordSet& getIncompatiblePatternsPmWordSet() const;
// Hyperscan initialization state management
bool isHyperscanInitialized() const;
void setHyperscanInitialized(bool initialized);
// Check if Hyperscan should be used (based on configuration)
static bool shouldUseHyperscan(bool force = false);
void processRegexMatch(
const std::string &groupName,
const std::string &groupValue,
std::string &word,
std::vector<std::string> &keyword_matches,
Waap::Util::map_of_stringlists_t &found_patterns,
bool longTextFound,
bool binaryDataFound
) const;
private:
picojson::value::object loadSource(const std::string& waapDataFileName);
void preprocessHyperscanPatterns();
bool m_hyperscanInitialized;
};
inline std::string repr_uniq(const std::string & value) {
std::string result;
char hist[256];
memset(&hist, 0, sizeof(hist));
for (std::string::const_iterator pC = value.begin(); pC != value.end(); ++pC) {
unsigned char ch = (unsigned char)(*pC);
// Only take ASCII characters that are not alphanumeric, and each character only once
if (ch <= 127 && !isalnum(ch) && hist[ch] == 0) {
// Convert low ASCII characters to their C/C++ printable equivalent
// (used for easier viewing. Also, binary data causes issues with ElasticSearch)
switch (ch) {
case 0x07: result += "\\a"; break;
case 0x08: result += "\\b"; break;
case 0x09: result += "\\t"; break;
case 0x0A: result += "\\n"; break;
case 0x0B: result += "\\v"; break;
case 0x0C: result += "\\f"; break;
case 0x0D: result += "\\r"; break;
case 0x5C: result += "\\\\"; break;
case 0x27: result += "\\\'"; break;
case 0x22: result += "\\\""; break;
case 0x3F: result += "\\\?"; break;
default: {
if (ch >= 32) {
result += (char)ch;
}
else {
char buf[16];
sprintf(buf, "\\" "x%02X", ch);
result += buf;
}
}
}
hist[ch] = 1;
}
}
return result;
}
inline bool isShortWord(const std::string& word) {
return word.size() <= 2;
}
inline bool isShortHtmlTag(const std::string& word) {
return !word.empty() && word.size() <= 4 && word[0] == '<' && word[word.size() - 1] == '>';
}
#endif

2
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@@ -225,7 +225,7 @@ void
WaapAttackTypesMetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
if (data.blockType == FORCE_EXCEPTION) {
dbgInfo(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed";
dbgDebug(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed";
return;
}

209
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc
View File

@@ -20,30 +20,60 @@ USE_DEBUG_FLAG(D_WAAP_CONFIDENCE_CALCULATOR);
TrustedSourcesConfidenceCalculator::TrustedSourcesConfidenceCalculator(
std::string path,
const std::string& remotePath,
const std::string& assetId)
const std::string &remotePath,
const std::string &assetId)
:
SerializeToLocalAndRemoteSyncBase(std::chrono::minutes(120),
SYNC_WAIT_TIME,
path,
(remotePath == "") ? remotePath : remotePath + "/Trust",
assetId,
"TrustedSourcesConfidenceCalculator")
"TrustedSourcesConfidenceCalculator"),
m_persistent_state(),
m_incremental_logger(std::make_shared<KeyValSourceLogger>())
{
restore();
}
bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t minSources) const
{
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state first (accumulated data from previous syncs)
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
auto sourceSetItr = sourceCtrItr->second.find(value);
if (sourceSetItr != sourceCtrItr->second.end())
{
size_t persistent_sources = sourceSetItr->second.size();
if (persistent_sources >= minSources) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << persistent_sources << " (persistent only)";
return true;
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr->second.find(src) == sourceSetItr->second.end()) {
incremental_sources++;
}
}
}
}
}
size_t total_sources = persistent_sources + incremental_sources;
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << sourceSetItr->second.size();
return sourceSetItr->second.size() >= minSources;
<< " : " << value << " is " << total_sources << " (persistent: " << persistent_sources
<< ", incremental: " << incremental_sources << ")";
return total_sources >= minSources;
}
else
{
@@ -52,6 +82,18 @@ bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t
}
else
{
// Check if data exists only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << incr_set_itr->second.size() << " (incremental only)";
return incr_set_itr->second.size() >= minSources;
}
}
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return false;
@@ -76,25 +118,36 @@ private:
S2C_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger)
};
class TrsutedSourcesLogger : public RestGetFile
class TrustedSourcesLogger : public RestGetFile
{
public:
TrsutedSourcesLogger(const TrustedSourcesConfidenceCalculator::KeyValSourceLogger& _logger)
: logger(_logger)
TrustedSourcesLogger(std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> _logger_ptr)
: logger_ptr(_logger_ptr)
{
logger = move(*logger_ptr);
}
private:
std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> logger_ptr;
C2S_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger);
};
bool TrustedSourcesConfidenceCalculator::postData()
{
if (m_incremental_logger->empty())
{
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping";
return true; // Nothing to post
}
std::string url = getPostDataUrl();
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url;
mergeIncrementalToPersistent();
TrustedSourcesLogger logger(m_incremental_logger);
// Clear and reset incremental logger for next cycle
m_incremental_logger = std::make_shared<KeyValSourceLogger>();
TrsutedSourcesLogger logger(m_logger);
bool ok = sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
@@ -104,12 +157,12 @@ bool TrustedSourcesConfidenceCalculator::postData()
return ok;
}
void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>& files)
void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string> &files)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the window data for trusted sources";
std::string url = getPostDataUrl();
std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
for (auto file : files)
for (const auto &file : files) // Use const reference to avoid copying
{
if (file == sentFile)
{
@@ -135,19 +188,22 @@ void TrustedSourcesConfidenceCalculator::processData()
{
}
void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::string>& files)
void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::string> &files)
{
m_logger.clear();
pullProcessedData(files);
}
void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string>& files) {
Maybe<std::string> TrustedSourcesConfidenceCalculator::getRemoteStateFilePath() const
{
return m_remotePath + "/remote/data.data";
}
void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string> &files) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources";
bool pull_ok = false;
for (auto file: files) {
for (const auto &file : files) { // Use const reference
GetTrustedFile getTrustFile;
bool res = sendObjectWithRetry(getTrustFile,
bool res = sendObject(getTrustFile,
HTTPMethod::GET,
getUri() + "/" + file);
pull_ok |= res;
@@ -165,43 +221,89 @@ void TrustedSourcesConfidenceCalculator::postProcessedData()
std::string url = getUri() + "/" + m_remotePath + "/processed/data.data";
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the processed data to: " << url;
TrsutedSourcesLogger logger(m_logger);
// Send persistent state as processed data
auto logger_ptr = std::make_shared<TrustedSourcesConfidenceCalculator::KeyValSourceLogger>(m_persistent_state);
TrustedSourcesLogger logger(logger_ptr);
sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
}
TrustedSourcesConfidenceCalculator::ValuesSet TrustedSourcesConfidenceCalculator::getConfidenceValues(
const Key& key,
const Key &key,
size_t minSources) const
{
ValuesSet values;
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
for (auto sourceSetItr : sourceCtrItr->second)
{
if (sourceSetItr.second.size() >= minSources)
size_t persistent_sources = sourceSetItr.second.size();
if (persistent_sources >= minSources)
{
values.insert(sourceSetItr.first);
continue; // No need to check incremental logger if we already have enough sources
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(sourceSetItr.first);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr.second.find(src) == sourceSetItr.second.end()) {
incremental_sources++;
}
}
}
}
}
if (persistent_sources + incremental_sources >= minSources)
{
values.insert(sourceSetItr.first);
}
}
}
else
{
// Also check values that exist only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
for (auto incr_set_itr : incr_ctr_itr->second) {
// Skip if already processed in persistent state
if (sourceCtrItr != m_persistent_state.end() &&
sourceCtrItr->second.find(incr_set_itr.first) != sourceCtrItr->second.end()) {
continue;
}
if (incr_set_itr.second.size() >= minSources) {
values.insert(incr_set_itr.first);
}
}
}
}
if (values.empty()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return values;
}
void TrustedSourcesConfidenceCalculator::serialize(std::ostream& stream)
void TrustedSourcesConfidenceCalculator::serialize(std::ostream &stream)
{
cereal::JSONOutputArchive archive(stream);
archive(cereal::make_nvp("version", 2), cereal::make_nvp("logger", m_logger));
archive(cereal::make_nvp("version", 3), cereal::make_nvp("persistent_state", m_persistent_state));
}
void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
void TrustedSourcesConfidenceCalculator::deserialize(std::istream &stream)
{
cereal::JSONInputArchive archive(stream);
size_t version = 0;
@@ -218,24 +320,31 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
switch (version)
{
case 3:
{
archive(cereal::make_nvp("persistent_state", m_persistent_state));
break;
}
case 2:
{
archive(cereal::make_nvp("logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("logger", m_persistent_state));
break;
}
case 1:
{
KeyValSourceLogger logger;
archive(cereal::make_nvp("logger", logger));
for (auto& log : logger)
for (auto &log : logger)
{
m_logger[normalize_param(log.first)] = log.second;
m_persistent_state[normalize_param(log.first)] = log.second;
}
break;
}
case 0:
{
archive(cereal::make_nvp("m_logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("m_logger", m_persistent_state));
break;
}
default:
@@ -244,17 +353,17 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
}
}
void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger& logs)
void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogger &logs)
{
for (auto& srcCounterItr : logs)
for (auto &srcCounterItr : logs)
{
for (auto& sourcesItr : srcCounterItr.second)
for (auto &sourcesItr : srcCounterItr.second)
{
for (auto& src : sourcesItr.second)
for (auto &src : sourcesItr.second)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Registering the source: " << src
<< " for the value: " << sourcesItr.first << " and the key: " << srcCounterItr.first;
m_logger[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
m_persistent_state[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
}
}
}
@@ -269,10 +378,28 @@ void TrustedSourcesConfidenceCalculator::log(Key key, Val value, Source source)
<< key
<< " from the source: "
<< source;
m_logger[key][value].insert(source);
(*m_incremental_logger)[key][value].insert(source);
}
void TrustedSourcesConfidenceCalculator::reset()
{
m_logger.clear();
m_persistent_state.clear();
m_incremental_logger->clear();
}
void TrustedSourcesConfidenceCalculator::mergeIncrementalToPersistent()
{
// Merge incremental data into persistent state (same logic as in postData but without network operations)
for (const auto &keyEntry : *m_incremental_logger) {
const std::string &key = keyEntry.first;
for (const auto &valueEntry : keyEntry.second) {
const std::string &value = valueEntry.first;
for (const std::string &source : valueEntry.second) {
m_persistent_state[key][value].insert(source);
}
}
}
// Clear incremental logger after merging
m_incremental_logger->clear();
}

30
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h
View File

@@ -19,6 +19,8 @@
USE_DEBUG_FLAG(D_WAAP);
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
// this class is responsible for logging trusted sources indicators matches (without validation)
class TrustedSourcesConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase
{
@@ -31,27 +33,35 @@ public:
typedef std::unordered_map<Val, SourcesSet> SourcesCounter;
typedef std::unordered_map<Key, SourcesCounter> KeyValSourceLogger;
TrustedSourcesConfidenceCalculator(std::string path, const std::string& remotePath,
const std::string& assetId);
TrustedSourcesConfidenceCalculator(std::string path, const std::string &remotePath,
const std::string &assetId);
bool is_confident(Key key, Val value, size_t minSources) const;
virtual bool postData();
virtual void pullData(const std::vector<std::string>& files);
virtual void pullData(const std::vector<std::string> &files);
virtual void processData();
virtual void postProcessedData();
virtual void pullProcessedData(const std::vector<std::string>& files);
virtual void updateState(const std::vector<std::string>& files);
virtual void pullProcessedData(const std::vector<std::string> &files);
virtual void updateState(const std::vector<std::string> &files);
virtual Maybe<std::string> getRemoteStateFilePath() const override;
ValuesSet getConfidenceValues(const Key& key, size_t minSources) const;
ValuesSet getConfidenceValues(const Key &key, size_t minSources) const;
virtual void serialize(std::ostream& stream);
virtual void deserialize(std::istream& stream);
virtual void serialize(std::ostream &stream);
virtual void deserialize(std::istream &stream);
void mergeFromRemote(const KeyValSourceLogger& logs);
void mergeFromRemote(const KeyValSourceLogger &logs);
void log(Key key, Val value, Source source);
void reset();
// Helper method for testing - merges incremental data to persistent state without network operations
void mergeIncrementalToPersistent();
private:
KeyValSourceLogger m_logger;
// Persistent state - accumulated trusted sources data across sync cycles
KeyValSourceLogger m_persistent_state;
// Incremental logger - only new data since last sync (gets moved during postData)
std::shared_ptr<KeyValSourceLogger> m_incremental_logger;
};

19
components/security_apps/waap/waap_clib/TuningDecision.cc
View File

@@ -26,7 +26,7 @@ USE_DEBUG_FLAG(D_WAAP);
TuningDecision::TuningDecision(const string& remotePath)
:
m_remotePath(replaceAllCopy(remotePath + "/tuning", "//", "/")),
m_remotePath(replaceAllCopy(remotePath + "/tuning/decisions.data", "//", "/")),
m_baseUri()
{
if (remotePath == "")
@@ -35,7 +35,7 @@ TuningDecision::TuningDecision(const string& remotePath)
}
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->addRecurringRoutine(
I_MainLoop::RoutineType::System,
chrono::minutes(10),
chrono::minutes(30),
[&]() { updateDecisions(); },
"Get tuning updates"
);
@@ -118,27 +118,16 @@ TuningDecisionType TuningDecision::convertDecisionType(string decisionTypeStr)
void TuningDecision::updateDecisions()
{
TuningEvents tuningEvents;
RemoteFilesList tuningDecisionFiles;
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) {
m_baseUri = "/api/";
} else {
m_baseUri = "/storage/waap/";
}
dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri;
bool isSuccessful = sendObject(tuningDecisionFiles,
HTTPMethod::GET,
m_baseUri + "?list-type=2&prefix=" + m_remotePath);
if (!isSuccessful || tuningDecisionFiles.getFilesList().empty())
{
dbgDebug(D_WAAP) << "Failed to get the list of files";
return;
}
if (!sendObject(tuningEvents,
HTTPMethod::GET,
m_baseUri + tuningDecisionFiles.getFilesList()[0]))
m_baseUri + m_remotePath))
{
return;
}
@@ -181,7 +170,7 @@ TuningDecision::getSharedStorageHost()
char* sharedStorageHost = getenv(SHARED_STORAGE_HOST_ENV_NAME);
if (sharedStorageHost != NULL) {
shared_storage_host = string(sharedStorageHost);
dbgInfo(D_WAAP) << "shared storage host is set to " << shared_storage_host;
dbgDebug(D_WAAP) << "shared storage host is set to " << shared_storage_host;
return shared_storage_host;
}
dbgWarning(D_WAAP) << "shared storage host is not set. using default: " << defaultSharedStorageHost;

35
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc
View File

@@ -29,8 +29,8 @@ USE_DEBUG_FLAG(D_WAAP);
#define TYPES_FILTER_TRUST_PATH(dirPath) dirPath + "/9.data"
TypeIndicatorFilter::TypeIndicatorFilter(I_WaapAssetState* pWaapAssetState,
const std::string& remotePath,
const std::string& assetId,
const std::string &remotePath,
const std::string &assetId,
TuningDecision* tuning,
size_t minSources,
size_t minIntervals,
@@ -78,7 +78,7 @@ bool TypeIndicatorFilter::shouldFilterKeyword(const std::string &key, const std:
return false;
}
void TypeIndicatorFilter::registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
void TypeIndicatorFilter::registerKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords,
IWaf2Transaction* pTransaction)
{
(void)keywords;
@@ -86,15 +86,21 @@ void TypeIndicatorFilter::registerKeywords(const std::string& key, Waap::Keyword
registerKeywords(key, sample, pTransaction);
}
void TypeIndicatorFilter::registerKeywords(const std::string& key, const std::string& sample,
void TypeIndicatorFilter::registerKeywords(const std::string &key, const std::string &sample,
IWaf2Transaction* pTransaction)
{
std::set<std::string> types = m_pWaapAssetState->getSampleType(sample);
std::string source = pTransaction->getSourceIdentifier();
std::string trusted_source = getTrustedSource(pTransaction);
auto trusted_source_maybe = getTrustedSource(pTransaction);
std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : "";
for (const std::string &type : types)
{
if (type == "long_random_text")
{
// Skip long random text types, as they are not useful for filtering
continue;
}
if (type == "local_file_path")
{
std::string location = IndicatorsFiltersManager::getLocationFromKey(key, pTransaction);
@@ -142,7 +148,7 @@ void TypeIndicatorFilter::loadParams(std::shared_ptr<Waap::Parameters::WaapParam
m_confidence_calc.reset(params);
}
std::set<std::string> TypeIndicatorFilter::getParamTypes(const std::string& canonicParam) const
std::set<std::string> TypeIndicatorFilter::getParamTypes(const std::string &canonicParam) const
{
std::set<std::string> types = m_confidence_calc.getConfidenceValues(canonicParam);
if (m_policy != nullptr)
@@ -153,3 +159,20 @@ std::set<std::string> TypeIndicatorFilter::getParamTypes(const std::string& cano
}
return types;
}
bool TypeIndicatorFilter::shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction) {
// Retrieve the sample from the transaction
std::string sample = pTransaction->getLastScanSample();
// Get the types associated with the sample
std::set<std::string> sampleTypes = m_pWaapAssetState->getSampleType(sample);
// Check if any of the sample types should be tracked
for (const auto& type : sampleTypes) {
if (m_confidence_calc.shouldTrackParameter(key, type)) {
return true;
}
}
return false;
}

2
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h
View File

@@ -41,7 +41,7 @@ public:
virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keyword,
IWaf2Transaction* pTransaction);
bool shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction);
void registerKeywords(const std::string& key, const std::string& sample, IWaf2Transaction* pTransaction);
void loadParams(std::shared_ptr<Waap::Parameters::WaapParameters> pParams);

339
components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.cc Normal file
View File

@@ -0,0 +1,339 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "UnifiedIndicatorsContainer.h"
#include <cereal/archives/json.hpp>
#include <cereal/types/string.hpp>
#include <cereal/types/vector.hpp>
#include <algorithm>
using std::string;
using std::unordered_map;
using std::unordered_set;
using std::ostream;
using std::istream;
// -------------------------------
// Interning helpers
// -------------------------------
const std::string*
UnifiedIndicatorsContainer::internValue(const std::string &value)
{
auto it = valuePool.find(value);
if (it == valuePool.end()) it = valuePool.insert(value).first;
return &(*it);
}
const std::string*
UnifiedIndicatorsContainer::internSource(const std::string &source)
{
auto it = sourcesPool.find(source);
if (it == sourcesPool.end()) it = sourcesPool.insert(source).first;
return &(*it);
}
// -------------------------------
// Public API
// -------------------------------
void
UnifiedIndicatorsContainer::addIndicator(
const std::string &key,
const std::string &value,
IndicatorType type,
const std::string &source)
{
auto &filters = filtersDataPerKey[key];
const std::string *valPtr = internValue(value);
const std::string *srcPtr = internSource(source);
FilterData &bucket = (type == IndicatorType::KEYWORD)
? filters.getIndicators()
: filters.getTypes();
auto &srcSet = bucket[const_cast<std::string*>(valPtr)];
srcSet.insert(const_cast<std::string*>(srcPtr));
// Update per-key total sources union
filters.getTotalSources().insert(const_cast<std::string*>(srcPtr));
}
void UnifiedIndicatorsContainer::addEntry(const Entry &entry)
{
const std::string *srcPtr = internSource(entry.sourceId);
if (entry.isTrusted && srcPtr) {
trustedSources.insert(srcPtr);
}
for (const auto &val : entry.indicators) {
addIndicator(entry.key, val, IndicatorType::KEYWORD, entry.sourceId);
}
for (const auto &val : entry.types) {
addIndicator(entry.key, val, IndicatorType::TYPE, entry.sourceId);
}
}
bool
UnifiedIndicatorsContainer::hasIndicator(
const std::string &key,
const std::string &value,
IndicatorType type) const
{
auto keyIt = filtersDataPerKey.find(key);
if (keyIt == filtersDataPerKey.end()) return false;
const Filters &filters = keyIt->second;
const FilterData &bucket = (type == IndicatorType::KEYWORD)
? filters.getIndicators()
: filters.getTypes();
auto valIt = valuePool.find(value);
if (valIt == valuePool.end()) return false;
auto it = bucket.find(const_cast<std::string*>(&(*valIt)));
return it != bucket.end();
}
std::unordered_set<std::string>
UnifiedIndicatorsContainer::getSources(
const std::string &key,
const std::string &value,
IndicatorType type) const
{
std::unordered_set<std::string> out;
auto keyIt = filtersDataPerKey.find(key);
if (keyIt == filtersDataPerKey.end()) return out;
const Filters &filters = keyIt->second;
const FilterData &bucket = (type == IndicatorType::KEYWORD)
? filters.getIndicators()
: filters.getTypes();
auto valIt = valuePool.find(value);
if (valIt == valuePool.end()) return out;
auto it = bucket.find(const_cast<std::string*>(&(*valIt)));
if (it == bucket.end()) return out;
for (auto p : it->second) if (p) out.insert(*p);
return out;
}
size_t
UnifiedIndicatorsContainer::getIndicatorCount() const
{
size_t count = 0;
for (const auto &k : filtersDataPerKey) {
count += k.second.getIndicators().size();
count += k.second.getTypes().size();
}
return count;
}
size_t
UnifiedIndicatorsContainer::getKeyCount() const
{
return filtersDataPerKey.size();
}
size_t
UnifiedIndicatorsContainer::getValuePoolSize() const
{
return valuePool.size();
}
void
UnifiedIndicatorsContainer::clear()
{
filtersDataPerKey.clear();
valuePool.clear();
sourcesPool.clear();
trustedSources.clear();
}
// -------------------------------
// Serialization
// -------------------------------
void
UnifiedIndicatorsContainer::serialize(std::ostream &stream) const
{
cereal::JSONOutputArchive ar(stream);
// Write trustedSources as a named array under the root object (global trusted only)
ar.setNextName("trustedSources");
ar.startNode();
cereal::size_type n_trusted = static_cast<cereal::size_type>(trustedSources.size());
ar(cereal::make_size_tag(n_trusted));
for (auto p : trustedSources) ar(p ? *p : std::string());
ar.finishNode();
// logger: object of keys -> { totalSources: [...], indicators: {...}, types: {...} }
ar.setNextName("logger");
ar.startNode();
for (const auto &k : filtersDataPerKey) {
ar.setNextName(k.first.c_str());
ar.startNode();
// totalSources section (union per key)
ar.setNextName("totalSources");
ar.startNode();
const auto &ts = k.second.getTotalSources();
cereal::size_type ts_sz = static_cast<cereal::size_type>(ts.size());
ar(cereal::make_size_tag(ts_sz));
for (auto p : ts) ar(p ? *p : std::string());
ar.finishNode();
// indicators section
ar.setNextName("indicators");
ar.startNode();
for (const auto &kv : k.second.getIndicators()) {
const std::string *val = kv.first;
ar.setNextName(val ? val->c_str() : "");
ar.startNode();
cereal::size_type sz = static_cast<cereal::size_type>(kv.second.size());
ar(cereal::make_size_tag(sz));
for (auto p : kv.second) ar(p ? *p : std::string());
ar.finishNode(); // end value array
}
ar.finishNode(); // end indicators
// types section
ar.setNextName("types");
ar.startNode();
for (const auto &kv : k.second.getTypes()) {
const std::string *val = kv.first;
ar.setNextName(val ? val->c_str() : "");
ar.startNode();
cereal::size_type sz = static_cast<cereal::size_type>(kv.second.size());
ar(cereal::make_size_tag(sz));
for (auto p : kv.second) ar(p ? *p : std::string());
ar.finishNode(); // end value array
}
ar.finishNode(); // end types
ar.finishNode(); // end key object
}
ar.finishNode(); // end logger
}
void
UnifiedIndicatorsContainer::deserialize(std::istream &stream)
{
cereal::JSONInputArchive ar(stream);
clear();
// trustedSources (optional) as a named array
try {
ar.setNextName("trustedSources");
ar.startNode();
cereal::size_type n = 0;
ar(cereal::make_size_tag(n));
for (cereal::size_type i = 0; i < n; ++i) {
std::string s; ar(s);
const std::string *p = internSource(s);
trustedSources.insert(p);
}
ar.finishNode();
} catch (...) {
// Field may be absent
}
// logger
try {
ar.setNextName("logger");
ar.startNode();
while (true) {
const auto node_name = ar.getNodeName();
if (!node_name) break;
std::string key = node_name;
ar.startNode(); // enter key object
// totalSources (optional)
try {
ar.setNextName("totalSources");
ar.startNode();
cereal::size_type ts_sz = 0;
ar(cereal::make_size_tag(ts_sz));
auto &ts = filtersDataPerKey[key].getTotalSources();
for (cereal::size_type i = 0; i < ts_sz; ++i) {
std::string s; ar(s);
const std::string *p = internSource(s);
ts.insert(const_cast<std::string*>(p));
}
ar.finishNode();
} catch (...) {
// no totalSources
}
// indicators
try {
ar.setNextName("indicators");
ar.startNode();
while (true) {
const auto val_name = ar.getNodeName();
if (!val_name) break;
std::string value = val_name;
ar.startNode();
cereal::size_type sz = 0;
ar(cereal::make_size_tag(sz));
for (cereal::size_type i = 0; i < sz; ++i) {
std::string src; ar(src);
addIndicator(key, value, IndicatorType::KEYWORD, src);
}
ar.finishNode(); // end value array
}
ar.finishNode();
} catch (...) {
// no indicators
}
// types
try {
ar.setNextName("types");
ar.startNode();
while (true) {
const auto val_name = ar.getNodeName();
if (!val_name) break;
std::string value = val_name;
ar.startNode();
cereal::size_type sz = 0;
ar(cereal::make_size_tag(sz));
for (cereal::size_type i = 0; i < sz; ++i) {
std::string src; ar(src);
addIndicator(key, value, IndicatorType::TYPE, src);
}
ar.finishNode(); // end value array
}
ar.finishNode();
} catch (...) {
// no types
}
ar.finishNode(); // finish key object
}
ar.finishNode(); // finish logger
} catch (...) {
// Field may be absent
}
}
bool UnifiedIndicatorsContainer::isTrustedSource(const std::string &source) const {
// Linear check via interning: attempt to find an interned pointer matching source
// We maintain sourcesPool mapping actual std::string storage, so compare by value.
for (const auto &p : trustedSources) {
if (p && *p == source) return true;
}
return false;
}

220
components/security_apps/waap/waap_clib/UnifiedIndicatorsContainer.h Normal file
View File

@@ -0,0 +1,220 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <unordered_map>
#include <unordered_set>
#include <string>
#include <vector>
#include <iostream>
#include <cereal/cereal.hpp>
#include <cereal/archives/json.hpp>
#include "i_serialize.h"
// #include "custom_serialization.h"
// Indicator type enumeration for type safety and compactness
enum class IndicatorType : uint8_t {
KEYWORD = 0,
TYPE = 1
};
typedef std::unordered_set<std::string*> SourcesSet;
typedef std::unordered_map<std::string*, SourcesSet> FilterData;
// Proposed name for `Filters`: KeyLog (represents the per-key section under "logger")
// Keeping class name as Filters to minimize changes; can be renamed in a follow-up.
class Filters {
public:
Filters() = default;
~Filters() = default;
// Const overload for cereal serialization
template<class Archive>
void serialize(Archive& ar) const {
std::vector<std::string> totalSourcesVec;
std::unordered_map<std::string, std::vector<std::string>> indicatorsMap, typesMap;
for (auto p : totalSources) {
if (p) totalSourcesVec.push_back(*p);
}
for (const auto& kv : indicators) {
std::string key = kv.first ? *kv.first : std::string();
std::vector<std::string> sources;
for (auto p : kv.second) {
if (p) sources.push_back(*p);
}
indicatorsMap[key] = sources;
}
for (const auto& kv : types) {
std::string key = kv.first ? *kv.first : std::string();
std::vector<std::string> sources;
for (auto p : kv.second) {
if (p) sources.push_back(*p);
}
typesMap[key] = sources;
}
ar(
cereal::make_nvp("totalSources", totalSourcesVec),
cereal::make_nvp("indicators", indicatorsMap),
cereal::make_nvp("types", typesMap)
);
}
// Accessors for container implementation
FilterData & getIndicators() { return indicators; }
FilterData & getTypes() { return types; }
const FilterData & getIndicators() const { return indicators; }
const FilterData & getTypes() const { return types; }
// Per-key total sources (union of sources from indicators and types)
SourcesSet & getTotalSources() { return totalSources; }
const SourcesSet & getTotalSources() const { return totalSources; }
private:
FilterData indicators;
FilterData types;
SourcesSet totalSources;
};
// Unified indicators container with string interning and memory optimization
class UnifiedIndicatorsContainer {
public:
// Batch entry input
struct Entry {
std::string key;
std::string sourceId;
bool isTrusted = false;
std::vector<std::string> indicators; // values treated as KEYWORD
std::vector<std::string> types; // values treated as TYPE
};
void addEntry(const Entry& entry);
// Check if an indicator exists
bool hasIndicator(const std::string& key, const std::string& value, IndicatorType type) const;
// Get all sources for a specific indicator
std::unordered_set<std::string> getSources(const std::string& key,
const std::string& value,
IndicatorType type) const;
// Statistics and metrics
size_t getIndicatorCount() const;
size_t getKeyCount() const;
size_t getValuePoolSize() const;
// Returns true if the given source string is marked as trusted (appears in the global trustedSources set)
bool isTrustedSource(const std::string &source) const;
// Container management
void clear();
// Serialization for cross-agent compatibility
// void serialize(std::ostream& stream) const;
template<class Archive>
void serialize(Archive& ar) const {
// trustedSources as array
std::vector<std::string> trusted_srcs;
for (auto p : trustedSources) {
if (p) trusted_srcs.push_back(*p);
}
ar.setNextName("trustedSources");
ar.startNode();
cereal::size_type n_trusted = static_cast<cereal::size_type>(trusted_srcs.size());
ar(cereal::make_size_tag(n_trusted));
for (const auto &s : trusted_srcs) ar(s);
ar.finishNode();
// logger: object of keys -> { totalSources: [...], indicators: {...}, types: {...} }
ar.setNextName("logger");
ar.startNode();
for (const auto &k : filtersDataPerKey) {
ar.setNextName(k.first.c_str());
ar.startNode();
// totalSources section (union per key)
ar.setNextName("totalSources");
ar.startNode();
const auto &ts = k.second.getTotalSources();
cereal::size_type ts_sz = static_cast<cereal::size_type>(ts.size());
ar(cereal::make_size_tag(ts_sz));
for (auto p : ts) ar(p ? *p : std::string());
ar.finishNode();
// indicators section
ar.setNextName("indicators");
ar.startNode();
for (const auto &kv : k.second.getIndicators()) {
const std::string *val = kv.first;
ar.setNextName(val ? val->c_str() : "");
ar.startNode();
cereal::size_type sz = static_cast<cereal::size_type>(kv.second.size());
ar(cereal::make_size_tag(sz));
for (auto p : kv.second) ar(p ? *p : std::string());
ar.finishNode(); // end value array
}
ar.finishNode(); // end indicators
// types section
ar.setNextName("types");
ar.startNode();
for (const auto &kv : k.second.getTypes()) {
const std::string *val = kv.first;
ar.setNextName(val ? val->c_str() : "");
ar.startNode();
cereal::size_type sz = static_cast<cereal::size_type>(kv.second.size());
ar(cereal::make_size_tag(sz));
for (auto p : kv.second) ar(p ? *p : std::string());
ar.finishNode(); // end value array
}
ar.finishNode(); // end types
ar.finishNode(); // end key object
}
ar.finishNode(); // end logger
}
void serialize(std::ostream &stream) const;
void deserialize(std::istream& stream);
private:
// Single indicator add
void addIndicator(const std::string& key, const std::string& value,
IndicatorType type, const std::string& source);
// String interning pool for values
std::unordered_set<std::string> valuePool;
// String interning pool for sources
std::unordered_set<std::string> sourcesPool;
// Main storage: key -> Filters
std::unordered_map<std::string, Filters> filtersDataPerKey;
// Global set of trusted sources
std::unordered_set<const std::string*> trustedSources;
// Helper methods
const std::string* internValue(const std::string& value);
const std::string* internSource(const std::string& source);
};
// UnifiedIndicatorsLogPost for REST, compatible with cereal and messaging
class UnifiedIndicatorsLogPost : public RestGetFile {
public:
UnifiedIndicatorsLogPost(std::shared_ptr<UnifiedIndicatorsContainer> container_ptr)
{
unifiedIndicators = std::move(*container_ptr);
}
private:
C2S_PARAM(UnifiedIndicatorsContainer, unifiedIndicators);
};

675
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

File diff suppressed because it is too large Load Diff

22
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -34,6 +34,7 @@
#include "ScanResult.h"
#include "WaapSampleValue.h"
#include "RequestsMonitor.h"
#include "WaapHyperscanEngine.h"
enum space_stage {SPACE_SYNBOL, BR_SYMBOL, BN_SYMBOL, BRN_SEQUENCE, BNR_SEQUENCE, NO_SPACES};
@@ -43,18 +44,34 @@ class WaapAssetState : public boost::noncopyable, public I_WaapAssetState
{
private: //ugly but needed for build
std::shared_ptr<Signatures> m_Signatures;
std::shared_ptr<WaapHyperscanEngine> m_hyperscanEngine;
std::string m_waapDataFileName;
std::map<std::string, std::vector<std::string>> m_filtered_keywords_verbose;
void checkRegex(const SampleValue &sample, const Regex & pattern, std::vector<std::string>& keyword_matches,
Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
// Helper function to perform the common pattern of checkRegex calls (2 or 3 calls based on parameter)
void performStandardRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound, bool includePatternRegex = true) const;
// Helper function to perform response body/header pattern of two checkRegex calls
void performResponseRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound) const;
// Helper function to perform pattern subset match only
void performPatternRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound) const;
void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
std::string nicePrint(Waf2ScanResult &res) const;
public:
// Load and compile signatures from file
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& waapDataFileName,
explicit WaapAssetState(
std::shared_ptr<Signatures> signatures,
const std::string &waapDataFileName,
std::shared_ptr<WaapHyperscanEngine> hyperscanEngine = nullptr,
size_t cleanCacheCapacity = SIGS_APPLY_CLEAN_CACHE_CAPACITY,
size_t suspiciousCacheCapacity = SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
size_t sampleTypeCacheCapacity = SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
@@ -111,7 +128,6 @@ public:
void clearErrorLimitingState();
void clearSecurityHeadersState();
std::shared_ptr<Waap::RateLimiting::State>& getRateLimitingState();
std::shared_ptr<Waap::RateLimiting::State>& getErrorLimitingState();
std::shared_ptr<Waap::SecurityHeaders::State>& getSecurityHeadersState();

22
components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc
View File

@@ -55,6 +55,7 @@ void WaapAssetStatesManager::setAssetDirectoryPath(const std::string &assetDirec
WaapAssetStatesManager::Impl::Impl() :
m_signatures(nullptr),
m_hyperscanEngine(nullptr),
m_basicWaapSigs(nullptr),
m_AssetBasedWaapSigs(),
m_assetDirectoryPath(BACKUP_DIRECTORY_PATH)
@@ -67,18 +68,33 @@ WaapAssetStatesManager::Impl::~Impl()
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapDataFileName)
{
if (m_signatures && !m_signatures->fail() && m_basicWaapSigs)
if (m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs)
{
// already initialized successfully.
return true;
}
try {
m_signatures = std::make_shared<Signatures>(waapDataFileName);
// Initialize Hyperscan engine
m_hyperscanEngine = std::make_shared<WaapHyperscanEngine>();
if (!Signatures::shouldUseHyperscan()) {
dbgTrace(D_WAAP) << "Hyperscan disabled by configuration, will use PCRE2";
} else if (!m_hyperscanEngine->initialize(m_signatures)) {
dbgTrace(D_WAAP) << "Hyperscan initialization failed, will use PCRE2";
} else {
m_signatures->setHyperscanInitialized(true);
dbgTrace(D_WAAP) << "Hyperscan initialized successfully";
}
m_basicWaapSigs = std::make_shared<WaapAssetState>(
m_signatures,
waapDataFileName,
m_hyperscanEngine,
SIGS_APPLY_CLEAN_CACHE_CAPACITY,
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY);
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
"");
}
catch (std::runtime_error & e) {
// TODO:: properly handle component initialization failure
@@ -89,7 +105,7 @@ bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapData
return false;
}
return m_signatures && !m_signatures->fail() && m_basicWaapSigs;
return m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs;
}
std::shared_ptr<WaapAssetState> WaapAssetStatesManager::Impl::getWaapAssetStateGlobal()

2
components/security_apps/waap/waap_clib/WaapAssetStatesManager.h
View File

@@ -15,6 +15,7 @@
#include "singleton.h"
#include "Signatures.h"
#include "WaapHyperscanEngine.h"
#include <string>
#include <memory>
#include <unordered_map>
@@ -65,6 +66,7 @@ private:
const std::string& instanceId);
std::shared_ptr<Signatures> m_signatures;
std::shared_ptr<WaapHyperscanEngine> m_hyperscanEngine;
std::shared_ptr<WaapAssetState> m_basicWaapSigs;
std::unordered_map<std::string, std::shared_ptr<WaapAssetState>> m_AssetBasedWaapSigs;
std::string m_assetDirectoryPath;

2
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@@ -26,7 +26,7 @@ set<string> WaapConfigAPI::assets_ids_aggregation{};
bool
WaapConfigAPI::getWaapAPIConfig(WaapConfigAPI& ngenAPIConfig) {
auto &maybe_ngen_config = getConfiguration<WaapConfigAPI>(
auto &maybe_ngen_config = getConfigurationWithCache<WaapConfigAPI>(
"WAAP",
"WebAPISecurity"
);

1
components/security_apps/waap/waap_clib/WaapConfigApi.h
View File

@@ -34,6 +34,7 @@ public:
static bool getWaapAPIConfig(WaapConfigAPI &ngenAPIConfig);
static void notifyAssetsCount();
static void clearAssetsCount();
virtual WaapConfigType getType() const override { return WaapConfigType::API; }
private:

23
components/security_apps/waap/waap_clib/WaapConfigApplication.cc
View File

@@ -23,16 +23,19 @@ set<string> WaapConfigApplication::assets_ids{};
set<string> WaapConfigApplication::assets_ids_aggregation{};
bool WaapConfigApplication::getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig) {
auto maybe_tenant_id = Singleton::Consume<I_Environment>::by<WaapConfigApplication>()->get<string>(
"ActiveTenantId"
);
auto maybe_profile_id = Singleton::Consume<I_Environment>::by<WaapConfigApplication>()->get<string>(
"ActiveProfileId"
);
string tenant_id = (maybe_tenant_id.ok() ? *maybe_tenant_id : "not found");
string profile_id = (maybe_profile_id.ok() ? *maybe_profile_id : "not found");
dbgTrace(D_WAAP) << "Tenant ID: " << tenant_id << ", Profile ID: " << profile_id;
auto &maybe_ngen_config = getConfiguration<WaapConfigApplication>(
// Only get tenant/profile id if debug is active
if (isDebugRequired(TRACE, D_WAAP)) {
auto maybe_tenant_id = Singleton::Consume<I_Environment>::by<WaapConfigApplication>()->get<string>(
"ActiveTenantId"
);
auto maybe_profile_id = Singleton::Consume<I_Environment>::by<WaapConfigApplication>()->get<string>(
"ActiveProfileId"
);
string tenant_id = (maybe_tenant_id.ok() ? *maybe_tenant_id : "not found");
string profile_id = (maybe_profile_id.ok() ? *maybe_profile_id : "not found");
dbgTrace(D_WAAP) << "Tenant ID: " << tenant_id << ", Profile ID: " << profile_id;
}
auto &maybe_ngen_config = getConfigurationWithCache<WaapConfigApplication>(
"WAAP",
"WebApplicationSecurity"
);

1
components/security_apps/waap/waap_clib/WaapConfigApplication.h
View File

@@ -39,6 +39,7 @@ public:
static bool getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig);
static void notifyAssetsCount();
static void clearAssetsCount();
virtual WaapConfigType getType() const override { return WaapConfigType::Application; }
private:
static const std::string s_PracticeSubType;

767
components/security_apps/waap/waap_clib/WaapHyperscanEngine.cc Normal file
View File

@@ -0,0 +1,767 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "WaapHyperscanEngine.h"
#include "Signatures.h"
#include "ScanResult.h"
#include "WaapSampleValue.h"
#include "Waf2Regex.h"
#include "Waf2Util.h"
#include "debug.h"
#include <algorithm>
#include <cctype>
#include <cstdlib>
#include <cstring>
#include <regex>
#ifdef USE_HYPERSCAN
#include "hs.h"
#endif
USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN);
USE_DEBUG_FLAG(D_WAAP_HYPERSCAN);
#ifdef USE_HYPERSCAN
static const unsigned int HS_STANDARD_FLAGS = HS_FLAG_CASELESS | HS_FLAG_SOM_LEFTMOST;
#endif // USE_HYPERSCAN
static const bool matchOriginalPattern = true;
static const size_t maxRegexValidationMatches = 10;
class WaapHyperscanEngine::Impl {
public:
struct PatternInfo {
std::string originalPattern;
std::string hyperscanPattern;
std::string groupName;
std::string category; // "keywords", "specific_accuracy", "patterns"
bool isFastReg;
bool isEvasion;
std::string regexSource; // "specific_acuracy_keywords_regex", "words_regex", "pattern_regex"
Signatures::AssertionFlags assertionFlags; // Zero-length assertion flags
std::unique_ptr<SingleRegex> originalRegex; // Precompiled original pattern for validation
PatternInfo() : isFastReg(false), isEvasion(false) {}
};
struct MatchContext {
const WaapHyperscanEngine::Impl* engine;
const std::string* sampleText;
std::vector<std::string>* keyword_matches;
std::vector<std::string>* regex_matches;
Waap::Util::map_of_stringlists_t* found_patterns;
bool longTextFound;
bool binaryDataFound;
bool includePatternRegex;
bool includeKeywordRegex;
// Per-signature tracking of last match end (pattern id => last end offset)
std::unordered_map<unsigned int, size_t> lastMatchEndPerSignature;
};
Impl();
~Impl();
bool initialize(const std::shared_ptr<Signatures>& signatures);
void scanSample(const SampleValue& sample,
Waf2ScanResult& res,
bool longTextFound,
bool binaryDataFound,
bool includeKeywordRegex,
bool includePatternRegex) const;
bool isInitialized() const { return m_isInitialized; }
size_t getPatternCount() const { return m_patternInfos.size(); }
size_t getCompiledPatternCount() const { return m_compiledPatternCount; }
size_t getFailedPatternCount() const { return m_failedPatternCount; }
private:
#ifdef USE_HYPERSCAN
hs_database_t* m_keywordDatabase;
hs_database_t* m_patternDatabase;
hs_scratch_t* m_keywordScratch;
hs_scratch_t* m_patternScratch;
#endif
std::shared_ptr<Signatures> m_Signatures;
std::vector<PatternInfo> m_patternInfos;
bool m_isInitialized;
size_t m_compiledPatternCount;
size_t m_failedPatternCount;
// Helper methods
bool compileHyperscanDatabases(const std::shared_ptr<Signatures>& signatures);
void loadPrecompiledPatterns(const std::shared_ptr<Signatures>& signatures);
// use an ordered set to keep PCRE2-validated matches sorted and unique in input order
// LCOV_EXCL_START Reason: Trivial
struct Match {
size_t from;
size_t to;
Match(size_t from, size_t to) : from(from), to(to) {}
bool operator<(const Match& other) const {
return (from < other.from) || (from == other.from && to < other.to);
}
};
// LCOV_EXCL_STOP
// Assertion validation helpers
bool validateAssertions(const std::string& sampleText,
size_t matchStart,
size_t matchEnd,
const PatternInfo& patternInfo,
std::set<Match> &foundMatches,
size_t maxMatches) const;
static bool isWordChar(char c);
static bool isNonWordSpecialChar(char c);
#ifdef USE_HYPERSCAN
// Hyperscan callback function
static int onMatch(unsigned int id,
unsigned long long from,
unsigned long long to,
unsigned int flags,
void* context);
void processMatch(unsigned int id,
unsigned long long from,
unsigned long long to,
MatchContext* context) const;
void identifyFailingPatterns(const std::vector<std::string>& patterns,
const std::vector<PatternInfo>& hsPatterns,
const std::string& logPrefix) {
for (size_t i = 0; i < patterns.size(); ++i) {
const char *single_pattern = patterns[i].c_str();
unsigned int single_flag = HS_STANDARD_FLAGS;
unsigned int single_id = static_cast<unsigned int>(i);
hs_database_t *test_db = nullptr;
hs_compile_error_t *single_err = nullptr;
hs_error_t single_result = hs_compile_multi(&single_pattern,
&single_flag,
&single_id,
1,
HS_MODE_BLOCK,
nullptr,
&test_db,
&single_err);
if (single_result != HS_SUCCESS) {
std::string additional_info = "";
if (i < hsPatterns.size()) {
const auto &hsPattern = hsPatterns[i];
additional_info = " | Category: " + hsPattern.category +
" | Group: " + hsPattern.groupName +
" | Source: " + hsPattern.regexSource;
if (!hsPattern.originalPattern.empty() &&
hsPattern.originalPattern != hsPattern.hyperscanPattern) {
additional_info += " | Original: '" + hsPattern.originalPattern + "'";
}
}
dbgWarning(D_WAAP_HYPERSCAN)
<< logPrefix << " [" << i << "]: '" << patterns[i]
<< "' - Error: " << (single_err ? single_err->message : "unknown") << additional_info;
if (single_err) {
hs_free_compile_error(single_err);
single_err = nullptr;
}
} else {
if (test_db) {
hs_free_database(test_db);
test_db = nullptr;
}
}
if (single_err) {
hs_free_compile_error(single_err);
single_err = nullptr;
}
}
}
#endif // USE_HYPERSCAN
};
WaapHyperscanEngine::Impl::Impl()
:
#ifdef USE_HYPERSCAN
m_keywordDatabase(nullptr), m_patternDatabase(nullptr), m_keywordScratch(nullptr), m_patternScratch(nullptr),
#endif // USE_HYPERSCAN
m_isInitialized(false), m_compiledPatternCount(0), m_failedPatternCount(0)
{
}
WaapHyperscanEngine::Impl::~Impl()
{
#ifdef USE_HYPERSCAN
if (m_keywordScratch) hs_free_scratch(m_keywordScratch);
if (m_patternScratch) hs_free_scratch(m_patternScratch);
if (m_keywordDatabase) hs_free_database(m_keywordDatabase);
if (m_patternDatabase) hs_free_database(m_patternDatabase);
#endif
}
bool WaapHyperscanEngine::Impl::initialize(const std::shared_ptr<Signatures> &signatures)
{
if (!signatures) {
dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::initialize: null signatures";
return false;
}
m_Signatures = signatures;
#ifdef USE_HYPERSCAN
m_isInitialized = compileHyperscanDatabases(signatures);
if (m_isInitialized) {
dbgInfo(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine initialized successfully. "
<< "Compiled: " << m_compiledPatternCount << ", Failed: " << m_failedPatternCount;
} else {
dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine initialization failed";
}
return m_isInitialized;
#else
dbgInfo(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine: Hyperscan not available on this platform";
return false;
#endif
}
bool WaapHyperscanEngine::Impl::compileHyperscanDatabases(const std::shared_ptr<Signatures> &signatures)
{
#ifdef USE_HYPERSCAN
// Load precompiled patterns from signatures instead of extracting at runtime
loadPrecompiledPatterns(signatures);
std::vector<std::string> keywordPatterns;
std::vector<std::string> patternRegexPatterns;
// Collect keyword patterns (from specific_accuracy and keywords categories)
auto keywordAssertionFlags = signatures->getKeywordAssertionFlags();
for (size_t i = 0; i < signatures->getKeywordHyperscanPatterns().size(); ++i) {
const auto &hsPattern = signatures->getKeywordHyperscanPatterns()[i];
keywordPatterns.push_back(hsPattern.hyperscanPattern);
PatternInfo info;
info.originalPattern = hsPattern.originalPattern;
info.hyperscanPattern = hsPattern.hyperscanPattern;
info.category = hsPattern.category;
info.regexSource = hsPattern.regexSource;
info.groupName = hsPattern.groupName;
info.isFastReg = hsPattern.isFastReg;
info.isEvasion = hsPattern.isEvasion;
// Set assertion flags if available
if (i < keywordAssertionFlags.size()) {
info.assertionFlags = keywordAssertionFlags[i];
}
// Compile original regex pattern for validation only when matchOriginal flag is set
if (!info.originalPattern.empty() && matchOriginalPattern) {
bool regexError = false;
info.originalRegex = std::make_unique<SingleRegex>(
info.originalPattern, regexError, "ValidationRegex_" + info.groupName + "_" + std::to_string(i));
if (regexError) {
dbgWarning(D_WAAP_HYPERSCAN)
<< "Failed to compile original regex for pattern: " << info.originalPattern
<< " (group: " << info.groupName << ")";
info.originalRegex.reset(); // Clear failed regex
}
}
m_patternInfos.push_back(std::move(info));
}
// Collect pattern regex patterns (from patterns category)
auto patternAssertionFlags = signatures->getPatternAssertionFlags();
for (size_t i = 0; i < signatures->getPatternHyperscanPatterns().size(); ++i) {
const auto &hsPattern = signatures->getPatternHyperscanPatterns()[i];
patternRegexPatterns.push_back(hsPattern.hyperscanPattern);
PatternInfo info;
info.originalPattern = hsPattern.originalPattern;
info.hyperscanPattern = hsPattern.hyperscanPattern;
info.category = hsPattern.category;
info.regexSource = hsPattern.regexSource;
info.groupName = hsPattern.groupName;
info.isFastReg = hsPattern.isFastReg;
info.isEvasion = hsPattern.isEvasion;
// Set assertion flags if available
if (i < patternAssertionFlags.size()) {
info.assertionFlags = patternAssertionFlags[i];
}
// Compile original regex pattern for validation only when matchOriginal flag is set
if (!info.originalPattern.empty() && matchOriginalPattern) {
bool regexError = false;
size_t patternIndex = keywordPatterns.size() + i; // Offset by keyword patterns count
info.originalRegex = std::make_unique<SingleRegex>(info.originalPattern, regexError,
"ValidationRegex_" + info.groupName + "_" + std::to_string(patternIndex));
if (regexError) {
dbgWarning(D_WAAP_HYPERSCAN)
<< "Failed to compile original regex for pattern: " << info.originalPattern
<< " (group: " << info.groupName << ")";
info.originalRegex.reset(); // Clear failed regex
}
}
m_patternInfos.push_back(std::move(info));
}
dbgInfo(D_WAAP_HYPERSCAN) << "Using precompiled patterns: "
<< "keywords=" << keywordPatterns.size()
<< ", patterns=" << patternRegexPatterns.size();
// Compile keyword database (specific_acuracy_keywords_regex + words_regex)
size_t total_ids = 0;
if (!keywordPatterns.empty()) {
std::vector<const char *> c_patterns;
std::vector<unsigned int> flags;
std::vector<unsigned int> ids;
for (size_t i = 0; i < keywordPatterns.size(); ++i) {
c_patterns.push_back(keywordPatterns[i].c_str());
flags.push_back(HS_STANDARD_FLAGS);
ids.push_back(static_cast<unsigned int>(total_ids++));
}
// Defensive checks before calling hs_compile_multi
if (c_patterns.size() != flags.size() || c_patterns.size() != ids.size()) {
dbgWarning(D_WAAP_HYPERSCAN) << "Pattern, flag, and id arrays are not the same size!";
return false;
}
if (c_patterns.empty()) {
dbgWarning(D_WAAP_HYPERSCAN) << "No patterns to compile!";
return false;
}
dbgInfo(D_WAAP_HYPERSCAN) << "Compiling " << c_patterns.size()
<< " keyword patterns with hs_compile_multi. First pattern: '"
<< keywordPatterns[0] << "'";
hs_compile_error_t *compile_err = nullptr;
hs_error_t result =
hs_compile_multi(c_patterns.data(),
flags.data(),
ids.data(),
static_cast<unsigned int>(c_patterns.size()),
HS_MODE_BLOCK,
nullptr,
&m_keywordDatabase,
&compile_err);
if (result != HS_SUCCESS) {
std::string error_msg = compile_err ? compile_err->message : "unknown error";
dbgWarning(D_WAAP_HYPERSCAN) << "Failed to compile keyword database: " << error_msg;
// Try to identify the specific failing pattern(s)
if (compile_err) {
dbgWarning(D_WAAP_HYPERSCAN) << "Attempting to identify failing keyword pattern(s)...";
auto keywordHsPatterns = signatures->getKeywordHyperscanPatterns();
std::vector<PatternInfo> keywordPatternInfos;
keywordPatternInfos.reserve(keywordHsPatterns.size());
for (const auto& hsPattern : keywordHsPatterns) {
keywordPatternInfos.emplace_back();
PatternInfo& info = keywordPatternInfos.back();
info.originalPattern = hsPattern.originalPattern;
info.hyperscanPattern = hsPattern.hyperscanPattern;
info.category = hsPattern.category;
info.regexSource = hsPattern.regexSource;
info.groupName = hsPattern.groupName;
info.isFastReg = hsPattern.isFastReg;
info.isEvasion = hsPattern.isEvasion;
}
identifyFailingPatterns(keywordPatterns, keywordPatternInfos, "Failing keyword pattern");
}
if (compile_err) {
hs_free_compile_error(compile_err);
compile_err = nullptr;
}
return false;
}
if (hs_alloc_scratch(m_keywordDatabase, &m_keywordScratch) != HS_SUCCESS) {
dbgWarning(D_WAAP_HYPERSCAN) << "Failed to allocate keyword scratch space";
return false;
}
m_compiledPatternCount += keywordPatterns.size();
}
// Compile pattern database (pattern_regex)
if (!patternRegexPatterns.empty()) {
std::vector<const char *> c_patterns;
std::vector<unsigned int> flags;
std::vector<unsigned int> ids;
for (size_t i = 0; i < patternRegexPatterns.size(); ++i) {
c_patterns.push_back(patternRegexPatterns[i].c_str());
flags.push_back(HS_STANDARD_FLAGS);
ids.push_back(static_cast<unsigned int>(total_ids++));
}
// Defensive checks before calling hs_compile_multi
if (c_patterns.size() != flags.size() || c_patterns.size() != ids.size()) {
dbgWarning(D_WAAP_HYPERSCAN)
<< "Pattern, flag, and id arrays are not the same size! (patternRegexPatterns)";
return false;
}
if (c_patterns.empty()) {
dbgWarning(D_WAAP_HYPERSCAN) << "No pattern regex patterns to compile!";
return false;
}
dbgInfo(D_WAAP_HYPERSCAN) << "Compiling " << c_patterns.size()
<< " pattern regex patterns with hs_compile_multi. First pattern: '"
<< patternRegexPatterns[0] << "'";
hs_compile_error_t *compile_err = nullptr;
hs_error_t result =
hs_compile_multi(c_patterns.data(),
flags.data(),
ids.data(),
static_cast<unsigned int>(c_patterns.size()),
HS_MODE_BLOCK,
nullptr,
&m_patternDatabase,
&compile_err);
if (result != HS_SUCCESS) {
std::string error_msg = compile_err ? compile_err->message : "unknown error";
dbgWarning(D_WAAP_HYPERSCAN) << "Failed to compile pattern database: " << error_msg;
// Try to identify the specific failing pattern(s)
if (compile_err) {
dbgWarning(D_WAAP_HYPERSCAN) << "Attempting to identify failing pattern regex pattern(s)...";
auto patternHsPatterns = signatures->getPatternHyperscanPatterns();
std::vector<PatternInfo> patternPatternInfos;
patternPatternInfos.reserve(patternHsPatterns.size());
for (const auto& hsPattern : patternHsPatterns) {
patternPatternInfos.emplace_back();
PatternInfo& info = patternPatternInfos.back();
info.originalPattern = hsPattern.originalPattern;
info.hyperscanPattern = hsPattern.hyperscanPattern;
info.category = hsPattern.category;
info.regexSource = hsPattern.regexSource;
info.groupName = hsPattern.groupName;
info.isFastReg = hsPattern.isFastReg;
info.isEvasion = hsPattern.isEvasion;
}
identifyFailingPatterns(patternRegexPatterns, patternPatternInfos, "Failing pattern regex");
}
if (compile_err) {
hs_free_compile_error(compile_err);
compile_err = nullptr;
}
return false;
}
if (hs_alloc_scratch(m_patternDatabase, &m_patternScratch) != HS_SUCCESS) {
dbgWarning(D_WAAP_HYPERSCAN) << "Failed to allocate pattern scratch space";
return false;
}
m_compiledPatternCount += patternRegexPatterns.size();
}
return true;
#else // USE_HYPERSCAN
return false;
#endif // USE_HYPERSCAN
}
void WaapHyperscanEngine::Impl::loadPrecompiledPatterns(const std::shared_ptr<Signatures> &signatures)
{
// This method is called to initialize any additional pattern processing if needed
// For now, the patterns are directly accessed from the signatures object
dbgTrace(D_WAAP_HYPERSCAN) << "Loading precompiled patterns from Signatures";
m_Signatures = signatures;
}
#ifdef USE_HYPERSCAN
int WaapHyperscanEngine::Impl::onMatch(unsigned int id,
unsigned long long from,
unsigned long long to,
unsigned int flags,
void *context)
{
MatchContext *ctx = static_cast<MatchContext *>(context);
ctx->engine->processMatch(id, from, to, ctx);
return 0; // Continue scanning
}
void WaapHyperscanEngine::Impl::processMatch(unsigned int id,
unsigned long long from,
unsigned long long to,
MatchContext *context) const
{
if (id >= m_patternInfos.size()) {
dbgWarning(D_WAAP_HYPERSCAN) << "Invalid pattern ID: " << id;
return;
}
const PatternInfo &info = m_patternInfos[id];
const std::string &sampleText = *context->sampleText;
size_t start = static_cast<size_t>(from);
size_t end = static_cast<size_t>(to);
if (end > sampleText.length()) end = sampleText.length();
if (start >= end) return;
// skip overlaps for this pattern
size_t &lastEnd = context->lastMatchEndPerSignature[id];
if (start < lastEnd) {
dbgTrace(D_WAAP_HYPERSCAN) << "Skipping overlapping match for pattern id=" << id << " start=" << start
<< " lastEnd=" << lastEnd << ", match: '" << sampleText.substr(start, end - start)
<< "'";
return;
}
std::set<Match> foundMatches;
if (!validateAssertions(sampleText, start, end, info, foundMatches, maxRegexValidationMatches)) return;
for (const auto &match : foundMatches) {
std::string matchedText = sampleText.substr(match.from, match.to - match.from);
std::string word = matchedText;
dbgTrace(D_WAAP_HYPERSCAN) << " match='" << word << "' id='" << id << "' group='" << info.groupName
<< "' category=" << info.category;
if (context->binaryDataFound && word.size() <= 2) {
dbgTrace(D_WAAP_HYPERSCAN)
<< "Will not add a short keyword '" << word << "' because binaryData was found";
continue;
}
if (context->includeKeywordRegex && (info.category == "keywords" || info.category == "specific_accuracy")) {
m_Signatures->processRegexMatch(info.groupName, matchedText, word, *context->keyword_matches,
*context->found_patterns, context->longTextFound,
context->binaryDataFound);
} else if (context->includePatternRegex && info.category == "patterns") {
m_Signatures->processRegexMatch(info.groupName, matchedText, word, *context->regex_matches,
*context->found_patterns, context->longTextFound,
context->binaryDataFound);
}
lastEnd = std::max(lastEnd, match.to);
}
}
#endif // USE_HYPERSCAN
void WaapHyperscanEngine::Impl::scanSample(const SampleValue &sample, Waf2ScanResult &res, bool longTextFound,
bool binaryDataFound, bool includeKeywordRegex, bool includePatternRegex) const
{
#ifdef USE_HYPERSCAN
if (!m_isInitialized) {
dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine: not initialized, skipping scan";
return;
}
const std::string &sampleText = sample.getSampleString();
MatchContext context;
context.engine = this;
context.sampleText = &sampleText;
context.keyword_matches = &res.keyword_matches;
context.regex_matches = &res.regex_matches;
context.found_patterns = &res.found_patterns;
context.longTextFound = longTextFound;
context.binaryDataFound = binaryDataFound;
context.includePatternRegex = includePatternRegex;
context.includeKeywordRegex = includeKeywordRegex;
context.lastMatchEndPerSignature.clear();
dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample: scanning '" << sampleText
<< "' longTextFound=" << longTextFound << " binaryDataFound=" << binaryDataFound
<< " includeKeywordRegex=" << includeKeywordRegex
<< " includePatternRegex=" << includePatternRegex;
if (includeKeywordRegex && m_keywordDatabase && m_keywordScratch) {
hs_error_t result =
hs_scan(m_keywordDatabase, sampleText.c_str(), static_cast<unsigned int>(sampleText.length()), 0,
m_keywordScratch, onMatch, &context);
if (result != HS_SUCCESS) {
dbgWarning(D_WAAP_HYPERSCAN) << "Keyword database scan failed: " << result;
}
}
if (includePatternRegex && m_patternDatabase && m_patternScratch) {
hs_error_t result =
hs_scan(m_patternDatabase, sampleText.c_str(), static_cast<unsigned int>(sampleText.length()), 0,
m_patternScratch, onMatch, &context);
if (result != HS_SUCCESS) {
dbgWarning(D_WAAP_HYPERSCAN) << "Pattern database scan failed: " << result;
}
}
dbgTrace(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample: found " << res.keyword_matches.size()
<< " keyword matches, " << res.regex_matches.size() << " regex matches";
#else
dbgWarning(D_WAAP_HYPERSCAN) << "WaapHyperscanEngine::scanSample called but Hyperscan not available";
#endif
}
bool WaapHyperscanEngine::Impl::validateAssertions(const std::string &sampleText, size_t matchStart, size_t matchEnd,
const PatternInfo &patternInfo, std::set<Match> &foundMatches,
size_t maxMatches) const
{
foundMatches.clear();
// If we don't have an original regex compiled, fall back to the assertion flags validation
if (!patternInfo.originalRegex) {
dbgTrace(D_WAAP_HYPERSCAN) << "No original regex available for validation, "
<< "falling back to assertion flags check";
foundMatches.emplace(matchStart, matchEnd);
// If no assertion flags are set, the match is valid
if (patternInfo.assertionFlags.empty()) {
return true;
}
if (
patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::END_NON_WORD_AHEAD) &&
matchEnd < sampleText.length() &&
isWordChar(sampleText[matchEnd])) {
// (?!\w) - requires NO word character after the match
return false;
}
if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::START_NON_WORD_BEHIND) && matchStart > 0 &&
isWordChar(sampleText[matchStart - 1])) {
// (?<!\w) - requires NO word character before the match
return false;
}
// Check start assertions
if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::START_WORD_BEHIND) &&
(matchStart == 0 || !isWordChar(sampleText[matchStart - 1]))) {
// (?<=\w) - requires a word character before the match
return false;
}
// Check end assertions
if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::END_WORD_AHEAD) &&
(matchEnd >= sampleText.length() || !isWordChar(sampleText[matchEnd]))) {
// (?=\w) - requires a word character after the match
return false;
}
if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::END_NON_WORD_SPECIAL)) {
// (?=[^\w?<>:=]|$) - requires a non-word character (excluding ?<>:=) or end of string after the match
if (matchEnd < sampleText.length()) {
char nextChar = sampleText[matchEnd];
if (isWordChar(nextChar) || nextChar == '?' || nextChar == '<' || nextChar == '>' || nextChar == ':' ||
nextChar == '=') {
return false;
}
}
// If we're at the end of string, this condition is satisfied
}
return true;
}
if (patternInfo.assertionFlags.isSet(Signatures::AssertionFlag::WILDCARD_EVASION)) {
// skip if the match does not contain either type of slash, and not a question mark
bool hasSlash = false;
bool hasQuestionMark = false;
for (size_t i = matchStart; i < matchEnd && !(hasSlash && hasQuestionMark); ++i) {
if (sampleText[i] == '\\' || sampleText[i] == '/') {
hasSlash = true;
}
if (sampleText[i] == '?') {
hasQuestionMark = true;
}
}
dbgTrace(D_WAAP_HYPERSCAN) << "Testing for wildcard evasion: '"
<< " hasSlash=" << hasSlash << " hasQuestionMark=" << hasQuestionMark;
if (!hasSlash || !hasQuestionMark) {
return false;
}
}
// Use the original compiled regex to find matches within the specified range
std::vector<RegexMatchRange> matchRanges;
// look behind to cover possible assertions, look ahead much further to cover lazy hyperscan match end
static const size_t lookbehind_range = 4, lookahead_range = 32;
size_t searchStart = (matchStart > lookbehind_range) ? (matchStart - lookbehind_range) : 0UL;
size_t searchEnd = ((matchEnd + lookahead_range) < matchEnd || (matchEnd + lookahead_range) > sampleText.length())
? sampleText.length() // overflow
: (matchEnd + lookahead_range); // within bounds
std::vector<RegexMatchRange> regex_matches;
patternInfo.originalRegex->findMatchRanges(sampleText, regex_matches, maxMatches, searchStart, searchEnd);
for (const auto &match : regex_matches) {
foundMatches.emplace(match.start, match.end);
if (isDebugRequired(TRACE, D_WAAP_HYPERSCAN)) {
dbgTrace(D_WAAP_HYPERSCAN) << "Match for: '" << patternInfo.originalPattern << "' matched in range ["
<< match.start << "," << match.end << "] "
<< "matched text: '"
<< sampleText.substr(match.start, match.end - match.start)
<< "'";
}
}
if (foundMatches.empty()) {
if (isDebugRequired(TRACE, D_WAAP_HYPERSCAN)) {
dbgTrace(D_WAAP_HYPERSCAN) << "No match for: '" << patternInfo.originalPattern
<< "' did not match in range [" << matchStart << "," << matchEnd << "] "
<< "matched text: '" << sampleText.substr(matchStart, matchEnd - matchStart)
<< "'";
}
return false;
}
return true;
}
// LCOV_EXCL_START Reason: Not in use currently, but kept for future reference
bool WaapHyperscanEngine::Impl::isWordChar(char c)
{
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}
bool WaapHyperscanEngine::Impl::isNonWordSpecialChar(char c)
{
return c == '?' || c == '<' || c == '>' || c == ':' || c == '=';
}
// LCOV_EXCL_STOP
// WaapHyperscanEngine public interface - delegates to Impl
WaapHyperscanEngine::WaapHyperscanEngine() : pimpl(std::make_unique<Impl>())
{
}
WaapHyperscanEngine::~WaapHyperscanEngine() = default;
bool WaapHyperscanEngine::initialize(const std::shared_ptr<Signatures>& signatures)
{
return pimpl->initialize(signatures);
}
void WaapHyperscanEngine::scanSample(const SampleValue& sample, Waf2ScanResult& res, bool longTextFound,
bool binaryDataFound, bool includeKeywordRegex, bool includePatternRegex) const
{
pimpl->scanSample(sample, res, longTextFound, binaryDataFound, includeKeywordRegex, includePatternRegex);
}
bool WaapHyperscanEngine::isInitialized() const
{
return pimpl->isInitialized();
}
size_t WaapHyperscanEngine::getPatternCount() const
{
return pimpl->getPatternCount();
}
size_t WaapHyperscanEngine::getCompiledPatternCount() const
{
return pimpl->getCompiledPatternCount();
}
size_t WaapHyperscanEngine::getFailedPatternCount() const
{
return pimpl->getFailedPatternCount();
}

54
components/security_apps/waap/waap_clib/WaapHyperscanEngine.h Normal file
View File

@@ -0,0 +1,54 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __WAAP_HYPERSCAN_ENGINE_H__
#define __WAAP_HYPERSCAN_ENGINE_H__
#include <string>
#include <vector>
#include <map>
#include <set>
#include <memory>
class Signatures;
class SampleValue;
struct Waf2ScanResult;
class WaapHyperscanEngine {
public:
WaapHyperscanEngine();
~WaapHyperscanEngine();
// Initialize with patterns from Signatures
bool initialize(const std::shared_ptr<Signatures>& signatures);
// Main scanning function - same interface as performStandardRegexChecks
void scanSample(const SampleValue& sample,
Waf2ScanResult& res,
bool longTextFound,
bool binaryDataFound,
bool includeKeywordRegex,
bool includePatternRegex) const;
// Check if the engine is ready to use
bool isInitialized() const;
// Get statistics
size_t getPatternCount() const;
size_t getCompiledPatternCount() const;
size_t getFailedPatternCount() const;
private:
class Impl;
std::unique_ptr<Impl> pimpl;
};
#endif // __WAAP_HYPERSCAN_ENGINE_H__

2
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@@ -103,7 +103,7 @@ const std::vector<std::string>& ExceptionsByPractice::getExceptionsOfPractice(De
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_web_app_ids;
default:
dbgError(D_WAAP) <<
dbgDebug(D_WAAP) <<
"Can't find practice type for exceptions by practice: " <<
practiceType <<
", return web app exceptions";

18
components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc
View File

@@ -26,21 +26,24 @@ namespace Waap {
// Register empty string work under known index
registerWord("");
std::string pm_list_name = "preconditions";
std::string pm_keys_name = "precondition_keys";
// The key should always be there unless data file is corrupted (but there's a unit test that tests exactly
// that!)
if (jsObj.find("preconditions") == jsObj.end()) {
if (jsObj.find(pm_list_name) == jsObj.end()) {
dbgError(D_WAAP_REGEX) << "Error loading regex preconditions (signatures data file corrupt?)...";
error = true;
return;
}
if (jsObj.find("precondition_keys") == jsObj.end()) {
if (jsObj.find(pm_keys_name) == jsObj.end()) {
dbgError(D_WAAP_REGEX) << "Error loading regex precondition sets (signatures data file corrupt?)...";
error = true;
return;
}
auto preconditions = jsObj.at("preconditions").get<picojson::value::object>();
auto preconditions = jsObj.at(pm_list_name).get<picojson::value::object>();
// Loop over pre-conditions (rules) and load them
dbgTrace(D_WAAP_REGEX) << "Loading regex preconditions...";
@@ -142,7 +145,7 @@ namespace Waap {
// Build full list of words to load into aho-corasick pattern matcher
dbgTrace(D_WAAP_REGEX) << "Loading regex precondition_keys into Aho-Corasick pattern matcher...";
auto preconditionKeys = jsObj.at("precondition_keys").get<picojson::value::array>();
auto preconditionKeys = jsObj.at(pm_keys_name).get<picojson::value::array>();
std::set<PMPattern> pmPatterns;
for (const auto &preconditionKey : preconditionKeys) {
@@ -167,18 +170,19 @@ namespace Waap {
// Initialize the aho-corasick pattern matcher with the patterns
Maybe<void> pmHookStatus = m_pmHook.prepare(pmPatterns);
if (!pmHookStatus.ok()) {
dbgError(D_WAAP_REGEX) << "Aho-Corasick engine failed to load!";
error = true;
return;
}
dbgTrace(D_WAAP_REGEX) << "Aho-Corasick engine loaded.";
dbgTrace(D_WAAP_REGEX) << "Aho-corasick pattern matching engine initialized!";
}
RegexPreconditions::~RegexPreconditions() {
// No Hyperscan resource management here; handled by HyperscanHook
}
bool Waap::RegexPreconditions::isNoRegexPattern(const std::string &pattern) const
{
return m_noRegexPatterns.find(pattern) != m_noRegexPatterns.end();

45
components/security_apps/waap/waap_clib/WaapRegexPreconditions.h
View File

@@ -22,9 +22,13 @@
#include <stdint.h>
#include <string>
// Forward declaration for test friend class
class TestableRegexPreconditions;
namespace Waap {
class RegexPreconditions
{
friend class TestableRegexPreconditions; // Allow test access to private methods
public:
typedef size_t WordIndex;
static const WordIndex emptyWordIndex; // special word index used to index the "impossible" empty word
@@ -37,8 +41,29 @@ namespace Waap {
public:
typedef std::unordered_set<WordIndex> PmWordSet;
struct WordInfo {
WordIndex napostNapreWordIndex;
WordIndex napostWordIndex;
WordIndex napreWordIndex;
WordIndex baseWordIndex;
std::string wordStr;
bool noRegex;
WordInfo()
:
napostNapreWordIndex(emptyWordIndex),
napostWordIndex(emptyWordIndex),
napreWordIndex(emptyWordIndex),
baseWordIndex(0),
wordStr(),
noRegex(false)
{
}
};
// The constructor builds internal data from Json object. Once built - the object becomes read-only.
RegexPreconditions(const picojson::value::object &jsObj, bool &error);
~RegexPreconditions();
bool isNoRegexPattern(const std::string &pattern) const;
const std::string &getWordStrByWordIndex(WordIndex wordIndex) const;
Waap::RegexPreconditions::WordIndex getWordByRegex(const std::string &pattern) const;
@@ -61,26 +86,6 @@ namespace Waap {
// Aho-Corasick pattern matcher object
PMHook m_pmHook;
struct WordInfo {
WordIndex napostNapreWordIndex;
WordIndex napostWordIndex;
WordIndex napreWordIndex;
WordIndex baseWordIndex;
std::string wordStr;
bool noRegex;
WordInfo()
:
napostNapreWordIndex(emptyWordIndex),
napostWordIndex(emptyWordIndex),
napreWordIndex(emptyWordIndex),
baseWordIndex(0),
wordStr(),
noRegex(false)
{
}
};
WordIndex registerWord(const std::string &wordStr);
std::vector<WordInfo> m_pmWordInfo;
std::map<std::string, WordIndex> m_wordStrToIndex; // TODO:: remove this into throwaway object, no need to keep

35
components/security_apps/waap/waap_clib/WaapSampleValue.cc
View File

@@ -12,19 +12,31 @@
// limitations under the License.
#include "WaapSampleValue.h"
#include "Signatures.h"
#include "debug.h"
SampleValue::SampleValue(const std::string &sample,
const std::shared_ptr<Waap::RegexPreconditions> &regexPreconditions)
USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN);
SampleValue::SampleValue(const std::string &sample, const Signatures* signatures)
:
m_sample(sample),
m_regexPreconditions(regexPreconditions),
m_signatures(signatures),
m_pmWordSet()
{
if (m_regexPreconditions) {
// Run aho-corasick and related rules once the sample value is known.
// The result pmWordSet is reused later for multiple calls to findMatches on the same sample.
regexPreconditions->pmScan(
Buffer(m_sample.data(), m_sample.size(), Buffer::MemoryType::STATIC), m_pmWordSet);
if (m_signatures) {
if (m_signatures->m_regexPreconditions) {
if (!m_signatures->isHyperscanInitialized()) {
// Run aho-corasick and related rules once the sample value is known.
// The result pmWordSet is reused later for multiple calls to findMatches on the same sample.
m_signatures->m_regexPreconditions->pmScan(
Buffer(m_sample.data(), m_sample.size(), Buffer::MemoryType::STATIC), m_pmWordSet);
} else {
// Add incompatible patterns from Signatures to the PmWordSet so they will be processed
// by traditional regex matching in Regex::findAllMatches
const auto& incompatiblePmWordSet = m_signatures->getIncompatiblePatternsPmWordSet();
m_pmWordSet.insert(incompatiblePmWordSet.begin(), incompatiblePmWordSet.end());
}
}
}
}
@@ -38,5 +50,10 @@ void
SampleValue::findMatches(const Regex &pattern, std::vector<RegexMatch> &matches) const
{
static const size_t maxMatchesPerSignature = 5;
pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr, maxMatchesPerSignature);
pattern.findAllMatches(
m_sample,
matches,
(m_signatures && m_signatures->m_regexPreconditions) ? &m_pmWordSet : nullptr,
maxMatchesPerSignature
);
}

6
components/security_apps/waap/waap_clib/WaapSampleValue.h
View File

@@ -21,16 +21,18 @@
#include "WaapRegexPreconditions.h"
#include "buffer.h"
class Signatures;
class SampleValue
{
public:
SampleValue(const std::string &sample, const std::shared_ptr<Waap::RegexPreconditions> &regexPreconditions);
SampleValue(const std::string &sample, const Signatures* signatures);
const std::string &getSampleString() const;
void findMatches(const Regex &pattern, std::vector<RegexMatch> &matches) const;
private:
std::string m_sample;
const std::shared_ptr<Waap::RegexPreconditions> m_regexPreconditions;
const Signatures* m_signatures;
Waap::RegexPreconditions::PmWordSet m_pmWordSet;
};

7
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -287,8 +287,11 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
m_transaction->getAssetState()->logParamHit(res, m_transaction);
std::set<std::string> paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes(
IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction));
std::set<std::string> paramTypes;
if (m_transaction->getAssetState()->m_filtersMngr != nullptr) {
paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes(
IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction));
}
if (paramTypes.size() == 1 && paramTypes.find("local_file_path") != paramTypes.end())
{

2
components/security_apps/waap/waap_clib/WaapTrigger.cc
View File

@@ -89,7 +89,7 @@ const std::vector<std::string>& TriggersByPractice::getTriggersByPractice(Decisi
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_web_app_ids;
default:
dbgError(D_WAAP) <<
dbgDebug(D_WAAP) <<
"Can't find practice type for triggers by practice: " <<
practiceType <<
", return web app triggers";

4
components/security_apps/waap/waap_clib/WaapTrigger.h
View File

@@ -201,7 +201,7 @@ struct Policy {
}
catch (std::runtime_error &e) {
ar.setNextName(nullptr);
dbgInfo(D_WAAP) << "Failed to load triggers per practice, error: " << e.what();
dbgDebug(D_WAAP) << "Failed to load triggers per practice, error: " << e.what();
triggersByPractice = TriggersByPractice();
}
try {
@@ -211,7 +211,7 @@ struct Policy {
}
catch (std::runtime_error &e) {
ar.setNextName(nullptr);
dbgInfo(D_WAAP) << "Failed to load web user response per practice, error: " << e.what();
dbgDebug(D_WAAP) << "Failed to load web user response per practice, error: " << e.what();
responseByPractice = WebUserResponseByPractice();
}
ar(

66
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc
View File

@@ -246,36 +246,38 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
}
// Detect URLEncode value
isUrlEncoded = checkUrlEncoded(cur_val.data(), cur_val.size());
textual.clear();
textual.append("hasCharSlash = ");
textual +=(hasCharSlash ? "true" : "false");
textual.append("\nhasCharColon = ");
textual +=(hasCharColon ? "true" : "false");
textual.append("\nhasCharAmpersand = ");
textual +=(hasCharAmpersand ? "true" : "false");
textual.append("\nhasCharEqual = ");
textual +=(hasCharEqual ? "true" : "false");
textual.append("\nhasTwoCharsEqual = ");
textual +=(hasTwoCharsEqual ? "true" : "false");
textual.append("\nhasCharSemicolon = ");
textual +=(hasCharSemicolon ? "true" : "false");
textual.append("\nhasCharPipe = ");
textual +=(hasCharPipe ? "true" : "false");
textual.append("\nisUTF16 = ");
textual +=(isUTF16 ? "true" : "false");
textual.append("\ncanSplitSemicolon = ");
textual +=(canSplitSemicolon ? "true" : "false");
textual.append("\ncanSplitPipe = ");
textual +=(canSplitPipe ? "true" : "false");
textual.append("\nhasSpace = ");
textual +=(hasSpace ? "true" : "false");
textual.append("\nisUrlEncoded = ");
textual +=(isUrlEncoded ? "true" : "false");
textual.append("\nhasCharLess = ");
textual +=(hasCharLess ? "true" : "false");
textual.append("\nhasDoubleQuote = ");
textual +=(hasDoubleQuote ? "true" : "false");
textual.append("\nhasPercent = ");
textual +=(hasPercent ? "true" : "false");
if (isDebugRequired(TRACE, D_WAAP))
{
textual.clear();
textual.append("hasCharSlash = ");
textual +=(hasCharSlash ? "true" : "false");
textual.append("\nhasCharColon = ");
textual +=(hasCharColon ? "true" : "false");
textual.append("\nhasCharAmpersand = ");
textual +=(hasCharAmpersand ? "true" : "false");
textual.append("\nhasCharEqual = ");
textual +=(hasCharEqual ? "true" : "false");
textual.append("\nhasTwoCharsEqual = ");
textual +=(hasTwoCharsEqual ? "true" : "false");
textual.append("\nhasCharSemicolon = ");
textual +=(hasCharSemicolon ? "true" : "false");
textual.append("\nhasCharPipe = ");
textual +=(hasCharPipe ? "true" : "false");
textual.append("\nisUTF16 = ");
textual +=(isUTF16 ? "true" : "false");
textual.append("\ncanSplitSemicolon = ");
textual +=(canSplitSemicolon ? "true" : "false");
textual.append("\ncanSplitPipe = ");
textual +=(canSplitPipe ? "true" : "false");
textual.append("\nhasSpace = ");
textual +=(hasSpace ? "true" : "false");
textual.append("\nisUrlEncoded = ");
textual +=(isUrlEncoded ? "true" : "false");
textual.append("\nhasCharLess = ");
textual +=(hasCharLess ? "true" : "false");
textual.append("\nhasDoubleQuote = ");
textual +=(hasDoubleQuote ? "true" : "false");
textual.append("\nhasPercent = ");
textual +=(hasPercent ? "true" : "false");
}
}

75
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -371,6 +371,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
Waf2Transaction::~Waf2Transaction() {
dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser";
delete m_requestBodyParser;
@@ -606,6 +607,7 @@ void Waf2Transaction::set_method(const char* method) {
bool Waf2Transaction::checkIsScanningRequired()
{
bool result = false;
if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
m_siteConfig = &m_ngenAPIConfig;
auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
@@ -639,7 +641,6 @@ bool Waf2Transaction::checkIsScanningRequired()
result = true;
}
}
return result;
}
@@ -652,7 +653,7 @@ bool Waf2Transaction::setCurrentAssetContext()
result |= checkIsScanningRequired();
if (!m_siteConfig) {
dbgWarning(D_WAAP) << "[transaction:" << this << "] "
dbgDebug(D_WAAP) << "[transaction:" << this << "] "
"Failed to set sitePolicy for asset... using the original signatures";
return result;
}
@@ -1572,8 +1573,7 @@ Waf2Transaction::decideFinal(
poolName;
// decision of (either) API or Application module
bool shouldBlock = false;
bool shouldBlock = false;
// TODO:: base class for both, with common inteface
WaapConfigAPI ngenAPIConfig;
WaapConfigApplication ngenSiteConfig;
@@ -1586,7 +1586,7 @@ Waf2Transaction::decideFinal(
m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy);
// User limits
shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
shouldBlock = (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP);
}
else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) {
dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S";
@@ -1608,7 +1608,7 @@ Waf2Transaction::decideFinal(
shouldBlock |= m_csrfState.decide(m_methodStr, m_waapDecision, csrfPolicy);
}
// User limits
shouldBlock |= (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
shouldBlock |= (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP);
}
if (mode == 2) {
@@ -1641,7 +1641,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
waapLog << LogField("sourceIP", m_remote_addr);
waapLog << LogField("httpSourceId", m_source_identifier);
waapLog << LogField("sourcePort", m_remote_port);
if (getProfileAgentSettingWithDefault<bool>(false, "agent.saasProfile.ignoreSourceIP")){
dbgTrace(D_WAAP) << "ignoring remote port in nexus log";
} else {
waapLog << LogField("sourcePort", m_remote_port);
}
waapLog << LogField("httpHostName", m_hostStr);
waapLog << LogField("httpMethod", m_methodStr);
if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
@@ -1757,7 +1761,9 @@ Waf2Transaction::sendLog()
dbgTrace(D_WAAP) << "send log got decision type: " << decision_type;
auto final_decision = m_waapDecision.getDecision(decision_type);
if (!final_decision) {
dbgTrace(D_WAAP) << "send log no decision found, using AUTONOMOUS_SECURITY_DECISION";
final_decision = m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION);
decision_type = AUTONOMOUS_SECURITY_DECISION;
}
std::string attackTypes = buildAttackTypes();
@@ -1853,6 +1859,14 @@ Waf2Transaction::sendLog()
return;
}
auto triggerLog = maybeTriggerLog.unpack();
if(final_decision->shouldForceLog() &&
triggerLog.shouldIgnoreExceptionLog(LogTriggerConf::SecurityType::ThreatPrevention)) {
// If we should ignore exception log, we need to handle it
dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: ignoring exception log";
return;
}
bool send_extended_log = shouldSendExtendedLog(triggerLog, decision_type);
shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision();
// Do not send Detect log if trigger disallows it
@@ -1898,6 +1912,16 @@ Waf2Transaction::sendLog()
return;
}
ReportIS::Severity severity =
decision_type == USER_LIMITS_DECISION ? ReportIS::Severity::HIGH : ReportIS::Severity::CRITICAL;
if (final_decision->shouldForceLog()) {
if (logOverride == OVERRIDE_DROP) {
severity = ReportIS::Severity::MEDIUM;
} else if (logOverride == OVERRIDE_ACCEPT) {
severity = ReportIS::Severity::INFO;
}
}
switch (decision_type)
{
case USER_LIMITS_DECISION: {
@@ -1923,7 +1947,7 @@ Waf2Transaction::sendLog()
"Web Request",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::HIGH,
severity,
Priority::HIGH,
shouldBlock);
@@ -1942,7 +1966,7 @@ Waf2Transaction::sendLog()
"API Request",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::CRITICAL,
severity,
Priority::HIGH,
shouldBlock);
@@ -1972,7 +1996,7 @@ Waf2Transaction::sendLog()
"CSRF Protection",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::CRITICAL,
severity,
Priority::HIGH,
shouldBlock);
@@ -2277,18 +2301,27 @@ bool Waf2Transaction::decideResponse()
}
auto triggerLog = maybeTriggerLog.unpack();
auto env = Singleton::Consume<I_Environment>::by<Waf2Transaction>();
auto http_chunk_type = env->get<ngx_http_chunk_type_e>("HTTP Chunk type");
auto http_chunk_type = env->get<AttachmentDataType>("HTTP Chunk type");
bool should_send_extended_log = shouldSendExtendedLog(triggerLog) && http_chunk_type.ok();
if (should_send_extended_log &&
*http_chunk_type == ngx_http_chunk_type_e::RESPONSE_CODE &&
*http_chunk_type == AttachmentDataType::RESPONSE_CODE &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
) {
dbgTrace(D_WAAP) << "response body is not active. Disabling extended logging";
should_send_extended_log = false;
} else if (should_send_extended_log &&
*http_chunk_type == ngx_http_chunk_type_e::REQUEST_END &&
*http_chunk_type == AttachmentDataType::REQUEST_END &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode) &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
) {
dbgTrace(D_WAAP) << "response code is not active. Disabling extended logging";
should_send_extended_log = false;
} else if (should_send_extended_log &&
*http_chunk_type == AttachmentDataType::RESPONSE_BODY &&
triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) &&
m_response_body.length() >= MAX_RESPONSE_BODY_SIZE)
{
dbgTrace(D_WAAP) << "response body collected (" << m_response_body.length() << " bytes). Disabling extended logging";
should_send_extended_log = false;
}
@@ -2347,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// collect sourceip, sourceIdentifier, url
exceptions_dict["sourceIP"].insert(m_remote_addr);
exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
exceptions_dict["url"].insert(getUriStr());
exceptions_dict["url"].insert(m_uriPath);
exceptions_dict["hostName"].insert(m_hostStr);
exceptions_dict["method"].insert(m_methodStr);
@@ -2359,15 +2392,21 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
}
bool isConfigExist = false;
if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap api config found";
m_siteConfig = &m_ngenAPIConfig;
if (m_siteConfig &&
(m_siteConfig->getType() == WaapConfigType::API || m_siteConfig->getType() == WaapConfigType::Application)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap config already exists";
isConfigExist = true;
} else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap web application config found";
m_siteConfig = &m_ngenSiteConfig;
isConfigExist = true;
}
else if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap api config found";
m_siteConfig = &m_ngenAPIConfig;
isConfigExist = true;
}
std::vector<std::string> site_exceptions;
if (isConfigExist) {
dbgTrace(D_WAAP_OVERRIDE) << "config exists, get override policy";
@@ -2387,7 +2426,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
behaviors.insert(params.begin(), params.end());
}
} else {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
auto exceptions = getConfigurationWithCache<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
return false;

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -63,6 +63,8 @@ typedef void(*subtransaction_cb_t)(Waf2Transaction* subTransaction, void *ctx);
#define OVERRIDE_ACCEPT "Accept"
#define OVERRIDE_DROP "Drop"
#define NEXUS_PORT 99999
class Waf2Transaction :
public IWaf2Transaction,
public TableOpaqueSerialize<Waf2Transaction>,
@@ -134,7 +136,7 @@ public:
virtual std::shared_ptr<WaapAssetState> getAssetState();
virtual const std::string getLocation() const;
ngx_http_cp_verdict_e getUserLimitVerdict();
ServiceVerdict getUserLimitVerdict();
const std::string getUserLimitVerdictStr() const;
const std::string getViolatedUserLimitTypeStr() const;
const std::string getCurrentWebUserResponse();

140
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -22,6 +22,8 @@
#include <memory>
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
#define LOW_REPUTATION_THRESHOLD 4
#define NORMAL_REPUTATION_THRESHOLD 6
@@ -363,7 +365,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
const ReportIS::Priority priority =
Waap::Util::computePriorityFromThreatLevel(autonomousSecurityDecision->getThreatLevel());
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
auto maybeLogTriggerConf = getConfigurationWithCache<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"Web Request",
@@ -445,12 +447,12 @@ void Waf2Transaction::createUserLimitsState()
}
}
ngx_http_cp_verdict_e
ServiceVerdict
Waf2Transaction::getUserLimitVerdict()
{
if (!isUserLimitReached()) {
// Either limit not reached or attack mitigation mode is DISABLED
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
std::string msg;
@@ -460,7 +462,7 @@ Waf2Transaction::getUserLimitVerdict()
std::string reason;
reason = " reason: " + getViolatedUserLimitTypeStr();
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
const AttackMitigationMode mode = WaapConfigBase::get_WebAttackMitigationMode(*m_siteConfig);
auto decision = m_waapDecision.getDecision(USER_LIMITS_DECISION);
if (mode == AttackMitigationMode::LEARNING) {
@@ -469,19 +471,19 @@ Waf2Transaction::getUserLimitVerdict()
// detect mode and no active drop exception
decision->setBlock(false);
if (isIllegalMethodViolation()) {
dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
dbgDebug(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
else {
dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
dbgDebug(D_WAAP_ULIMITS) << msg << "PASS" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
} else {
// detect mode and active drop exception
decision->setBlock(true);
decision->setForceBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Block" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
dbgDebug(D_WAAP_ULIMITS) << msg << "Override Block" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
}
else if (mode == AttackMitigationMode::PREVENT) {
@@ -489,14 +491,14 @@ Waf2Transaction::getUserLimitVerdict()
if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) {
// prevent mode and no active accept exception
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
dbgDebug(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
} else {
// prevent mode and active accept exception
decision->setBlock(false);
decision->setForceAllow(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
dbgDebug(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
}
@@ -662,25 +664,79 @@ Waf2Transaction::getBehaviors(
std::set<ParameterBehavior> all_params;
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<Waf2Transaction>();
for (const auto &id : exceptions) {
dbgTrace(D_WAAP) << "get parameter exception for: " << id;
auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict);
dbgTrace(D_WAAP_OVERRIDE) << "get parameter exception for: " << id;
if (checkResponse && !getResponseBody().empty()) {
std::unordered_map<std::string, std::set<std::string>> response_dict = {
{"responseBody", {getResponseBody()}}
};
auto params = i_rulebase->getParameterException(id).getBehavior(response_dict);
const auto &paramException = i_rulebase->getParameterException(id);
auto params = paramException.getBehavior(exceptions_dict);
bool hasKVPair = paramException.isContainingKVPair();
// If isContainingKVPair() is false, continue as usual
if (!hasKVPair) {
dbgTrace(D_WAAP_OVERRIDE) << "Using traditional matching for exception " << id << " (no KV pairs)";
if (checkResponse && !getResponseBody().empty()) {
std::unordered_map<std::string, std::set<std::string>> response_dict = {
{"responseBody", {getResponseBody()}}
};
auto responseParams = paramException.getBehavior(response_dict);
if (responseParams.size() > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "got responseBody behavior, setApplyOverride(true)";
m_responseInspectReasons.setApplyOverride(true);
all_params.insert(responseParams.begin(), responseParams.end());
// once found, no need to check again
checkResponse = false;
}
}
dbgTrace(D_WAAP_OVERRIDE) << "got "<< params.size() << " behaviors (non-KV pair)";
all_params.insert(params.begin(), params.end());
} else {
if (params.size() > 0) {
dbgTrace(D_WAAP) << "got responseBody behavior, setApplyOverride(true)";
m_responseInspectReasons.setApplyOverride(true);
all_params.insert(params.begin(), params.end());
// once found, no need to check again
checkResponse = false;
bool anyKVMatched = false;
std::set<ParameterBehavior> kvParams;
dbgTrace(D_WAAP_OVERRIDE) << "Using KV pair matching for exception " << id;
// Check parameter name/value pairs
for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) {
std::unordered_map<std::string, std::set<std::string>> kv_dict = {
{"paramName", {keywordInfo.getName()}},
{"paramValue", {keywordInfo.getValue()}}
};
auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true);
if (kvBehaviors.size() > 0) {
anyKVMatched = true;
kvParams.insert(kvBehaviors.begin(), kvBehaviors.end());
}
}
// Check header name/value pairs
for (const auto& hdr_pair : getHdrPairs()) {
std::string name = hdr_pair.first;
std::transform(name.begin(), name.end(), name.begin(), ::tolower);
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
std::unordered_map<std::string, std::set<std::string>> kv_dict = {
{"headerName", {name}},
{"headerValue", {value}}
};
auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true);
if (kvBehaviors.size() > 0) {
anyKVMatched = true;
kvParams.insert(kvBehaviors.begin(), kvBehaviors.end());
}
}
// Consider matches only if at least one of the key/value pairs have been successfully matched
if (anyKVMatched) {
dbgTrace(D_WAAP_OVERRIDE) << "got "<< kvParams.size() << " behaviors (KV pair matched)";
all_params.insert(kvParams.begin(), kvParams.end());
} else {
dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no specific pairs matched";
}
} else {
dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no initial match";
}
}
dbgTrace(D_WAAP) << "got "<< params.size() << " behaviors";
all_params.insert(params.begin(), params.end());
}
return all_params;
}
@@ -696,26 +752,26 @@ bool Waf2Transaction::shouldEnforceByPracticeExceptions(DecisionType practiceTyp
auto exceptions = overridePolicy->getExceptionsByPractice().getExceptionsOfPractice(practiceType);
if (!exceptions.empty()) {
dbgTrace(D_WAAP) << "get behaviors for practice: " << practiceType;
dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for practice: " << practiceType;
auto behaviors = getBehaviors(getExceptionsDict(practiceType), exceptions);
if (behaviors.size() > 0)
{
auto it = m_overrideStateByPractice.find(practiceType);
if (it == m_overrideStateByPractice.end()) {
dbgWarning(D_WAAP) << "no override state for practice: " << practiceType;
dbgDebug(D_WAAP_OVERRIDE) << "no override state for practice: " << practiceType;
return false;
}
setOverrideState(behaviors, it->second);
if (it->second.bForceBlock) {
dbgTrace(D_WAAP)
dbgTrace(D_WAAP_OVERRIDE)
<< "should block by exceptions for practice: " << practiceType;
decision->setBlock(true);
decision->setForceBlock(true);
shouldEnforce = true;
}
if (it->second.bForceException) {
dbgTrace(D_WAAP)
dbgTrace(D_WAAP_OVERRIDE)
<< "should not block by exceptions for practice: " << practiceType;
decision->setBlock(false);
decision->setForceAllow(true);
@@ -748,16 +804,16 @@ void Waf2Transaction::setOverrideState(const std::set<ParameterBehavior>& behavi
m_matchedOverrideIds.insert(behavior.getId());
if (behavior.getKey() == BehaviorKey::ACTION) {
if (behavior.getValue() == BehaviorValue::ACCEPT) {
dbgTrace(D_WAAP) << "setting bForceException due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bForceException due to override behavior: " << behavior.getId();
state.bForceException = true;
state.forceExceptionIds.insert(behavior.getId());
} else if (behavior.getValue() == BehaviorValue::REJECT) {
dbgTrace(D_WAAP) << "setting bForceBlock due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bForceBlock due to override behavior: " << behavior.getId();
state.bForceBlock = true;
state.forceBlockIds.insert(behavior.getId());
}
} else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) {
dbgTrace(D_WAAP) << "setting bSupressLog due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bSupressLog due to override behavior: " << behavior.getId();
state.bSupressLog = true;
}
}
@@ -771,11 +827,11 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
auto exceptions = overridePolicy->getExceptionsByPractice().
getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION);
if (!exceptions.empty()) {
dbgTrace(D_WAAP) << "get behaviors for override state";
dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for override state";
m_responseInspectReasons.setApplyOverride(false);
auto behaviors = getBehaviors(getExceptionsDict(AUTONOMOUS_SECURITY_DECISION), exceptions, true);
if (behaviors.size() > 0) {
dbgTrace(D_WAAP) << "set override state by practice found behaviors";
dbgTrace(D_WAAP_OVERRIDE) << "set override state by practice found behaviors";
setOverrideState(behaviors, m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]);
}
m_isHeaderOverrideScanRequired = false;
@@ -819,10 +875,10 @@ const Maybe<LogTriggerConf, Config::Errors> Waf2Transaction::getTriggerLog(const
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto trigger_config = getConfiguration<LogTriggerConf>("rulebase", "log");
auto trigger_config = getConfigurationWithCache<LogTriggerConf>("rulebase", "log");
if (!trigger_config.ok()) {
dbgError(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr();
dbgDebug(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr();
}
return trigger_config;
}
@@ -851,9 +907,9 @@ bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr<
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto trigger_config = getConfiguration<ReportTriggerConf>("rulebase", "report");
auto trigger_config = getConfigurationWithCache<ReportTriggerConf>("rulebase", "report");
if (!trigger_config.ok()) {
dbgWarning(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr();
dbgDebug(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr();
m_triggerReport = false;
return false;
}

31
components/security_apps/waap/waap_clib/Waf2Regex.cc
View File

@@ -117,11 +117,14 @@ SingleRegex::~SingleRegex() {
}
}
bool SingleRegex::hasMatch(const std::string& s) const {
bool SingleRegex::hasMatch(const std::string& s, size_t start, size_t end) const {
PCRE2_SIZE startOffset = static_cast<PCRE2_SIZE>(start);
size_t data_size = std::min(s.size(), end);
int rc = pcre2_match(
m_re, // code
reinterpret_cast<PCRE2_SPTR>(s.data()), s.size(), // subject/subject length
0, // start offset
reinterpret_cast<PCRE2_SPTR>(s.data()), data_size, // subject/subject length
startOffset, // start offset
0, // options
m_matchData,
NULL // match_context
@@ -244,13 +247,23 @@ const std::string &SingleRegex::getName() const
return m_regexName;
}
size_t SingleRegex::findMatchRanges(const std::string& s, std::vector<RegexMatchRange>& matchRanges) const {
PCRE2_SIZE startOffset = 0;
size_t
SingleRegex::findMatchRanges(
const std::string &s,
std::vector<RegexMatchRange> &matchRanges,
size_t maxMatches,
size_t start,
size_t end
) const
{
PCRE2_SIZE startOffset = static_cast<PCRE2_SIZE>(start);
size_t matchCount = 0;
size_t data_size = std::min(s.size(), end);
do {
int rc = pcre2_match(
m_re, // code
reinterpret_cast<PCRE2_SPTR>(s.data()), s.size(), // subject/subject length
reinterpret_cast<PCRE2_SPTR>(s.data()), data_size, // subject/subject length
startOffset, // start offset
0, // options
m_matchData,
@@ -277,6 +290,12 @@ size_t SingleRegex::findMatchRanges(const std::string& s, std::vector<RegexMatch
startOffset = ov[1];
matchRanges.push_back(RegexMatchRange(ov[0], ov[1]));
matchCount++;
// Check if we've reached the maximum number of matches (if maxMatches is not 0)
if (maxMatches > 0 && matchCount >= maxMatches) {
break;
}
} while (true);
return matchRanges.size();

10
components/security_apps/waap/waap_clib/Waf2Regex.h
View File

@@ -54,10 +54,16 @@ public:
SingleRegex(const std::string &pattern, bool &error, const std::string &regexName, bool bNoRegex=false,
const std::string &regexMatchName="", const std::string &regexMatchValue="");
~SingleRegex();
bool hasMatch(const std::string &s) const;
bool hasMatch(const std::string &s, size_t start = 0, size_t end = SIZE_MAX) const;
size_t findAllMatches(const std::string &s, std::vector<RegexMatch> &matches,
size_t max_matches = std::string::npos) const;
size_t findMatchRanges(const std::string &s, std::vector<RegexMatchRange> &matchRanges) const;
size_t findMatchRanges(
const std::string &s,
std::vector<RegexMatchRange> &matchRanges,
size_t maxMatches = 0,
size_t start = 0,
size_t end = SIZE_MAX
) const;
const std::string &getName() const;
private:
pcre2_code *m_re;

123
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -48,6 +48,49 @@ USE_DEBUG_FLAG(D_WAAP_JSON);
#define MIN_HEX_LENGTH 6
#define charToDigit(c) (c - '0')
// Base64 decoding lookup table constants
#define BASE64_INVALID -1
#define BASE64_PADDING -2
// Base64 lookup table for optimized decoding
static int base64_table[256];
static bool base64_table_initialized = false;
// Initialize the base64 lookup table for optimized decoding
static void initialize_base64_table()
{
if (base64_table_initialized) {
return;
}
// Initialize all entries to invalid
for (int i = 0; i < 256; i++) {
base64_table[i] = BASE64_INVALID;
}
// Set valid base64 characters (A-Z = 0-25)
for (int i = 0; i < 26; i++) {
base64_table['A' + i] = i;
}
// Set valid base64 characters (a-z = 26-51)
for (int i = 0; i < 26; i++) {
base64_table['a' + i] = 26 + i;
}
// Set valid base64 characters (0-9 = 52-61)
for (int i = 0; i < 10; i++) {
base64_table['0' + i] = 52 + i;
}
// Set special base64 characters
base64_table['+'] = 62;
base64_table['/'] = 63;
base64_table['='] = BASE64_PADDING;
base64_table_initialized = true;
}
// See https://dev.w3.org/html5/html-author/charref
const struct HtmlEntity g_htmlEntities[] =
{
@@ -1086,6 +1129,19 @@ base64_decode_status decideStatusBase64Decoded(
return B64_DECODE_INVALID;
}
//
// Base64 decoding with direct array lookup
// Performance optimization notes:
// - Replaced unordered_map lookups with direct array indexing
// - Improved cache locality with a simple 256-element array
// - Eliminated hash computation and collision handling overhead
// - One-time initialization of the lookup table
//
// The table maps each ASCII character to its base64 value:
// - For valid base64 characters (A-Z, a-z, 0-9, +, /): returns 0-63 value
// - For padding character ('='): returns BASE64_PADDING (-2)
// - For invalid characters: returns BASE64_INVALID (-1)
//
// Attempts to validate and decode base64-encoded chunk.
// Value is the full value inside which potential base64-encoded chunk was found,
@@ -1107,6 +1163,9 @@ base64_decode_status decodeBase64Chunk(
bool clear_on_error,
bool called_with_prefix)
{
// Initialize the base64 lookup table on first call
initialize_base64_table();
decoded.clear();
uint32_t acc = 0;
int acc_bits = 0; // how many bits are filled in acc
@@ -1135,8 +1194,9 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID;
}
std::unordered_map<char, double> original_occurences_counter;
std::unordered_map<char, double> decoded_occurences_counter;
// Fixed arrays for character counting - Proposal 1 optimization
uint32_t original_counts[256] = {0}; // Count for each ASCII character in original
uint32_t decoded_counts[256] = {0}; // Count for each byte value in decoded
while (it != end) {
unsigned char c = *it;
@@ -1159,35 +1219,21 @@ base64_decode_status decodeBase64Chunk(
// allow for more terminator characters
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
continue;
}
unsigned char val = 0;
// Use lookup table for faster decoding
int lookup_value = base64_table[static_cast<unsigned char>(c)];
if (c >= 'A' && c <= 'Z') {
val = c - 'A';
}
else if (c >= 'a' && c <= 'z') {
val = c - 'a' + 26;
}
else if (isdigit(c)) {
val = c - '0' + 52;
}
else if (c == '+') {
val = 62;
}
else if (c == '/') {
val = 63;
}
else if (c == '=') {
// Start tracking terminator characters
if (lookup_value == BASE64_PADDING) {
// Start tracking terminator characters ('=')
terminatorCharsSeen++;
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
continue;
}
else {
else if (lookup_value == BASE64_INVALID) {
dbgTrace(D_WAAP_BASE64)
<< "(leave as-is) because of non-base64 character ('"
<< c
@@ -1199,6 +1245,8 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID; // non-base64 character
}
unsigned char val = static_cast<unsigned char>(lookup_value);
acc = (acc << 6) | val;
acc_bits += 6;
@@ -1218,11 +1266,11 @@ base64_decode_status decodeBase64Chunk(
}
decoded += (char)code;
decoded_occurences_counter[(char)code]++;
decoded_counts[static_cast<unsigned char>(code)]++;
}
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
}
// end of encoded sequence decoded.
@@ -1242,13 +1290,21 @@ base64_decode_status decodeBase64Chunk(
double entropy = 0;
double p = 0;
double decoded_entropy = 0;
for (const auto& pair : original_occurences_counter) {
p = pair.second / length;
entropy -= p * std::log2(p);
// Calculate entropy from original character counts
for (int i = 0; i < 256; i++) {
if (original_counts[i] > 0) {
p = static_cast<double>(original_counts[i]) / length;
entropy -= p * std::log2(p);
}
}
for (const auto &pair : decoded_occurences_counter) {
p = pair.second / decoded.size();
decoded_entropy -= p * std::log2(p);
// Calculate entropy from decoded character counts
for (int i = 0; i < 256; i++) {
if (decoded_counts[i] > 0) {
p = static_cast<double>(decoded_counts[i]) / decoded.size();
decoded_entropy -= p * std::log2(p);
}
}
dbgTrace(D_WAAP_BASE64)
<< "Base entropy = "
@@ -1917,9 +1973,6 @@ isGzipped(const string &stream)
{
if (stream.size() < 2) return false;
auto unsinged_stream = reinterpret_cast<const u_char *>(stream.data());
dbgTrace(D_WAAP) << "isGzipped: first two bytes: "
<< std::hex << static_cast<int>(unsinged_stream[0]) << " "
<< std::hex << static_cast<int>(unsinged_stream[1]);
return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b;
}
@@ -2310,7 +2363,7 @@ string extractForwardedIp(const string &x_forwarded_hdr_val)
vector<string> trusted_ips;
string forward_ip;
auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
auto identify_config = getConfigurationWithCache<UsersAllIdentifiersConfig>(
"rulebase",
"usersIdentifiers"
);

481
components/security_apps/waap/waap_clib/buffered_compressed_stream.cc Normal file
View File

@@ -0,0 +1,481 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "buffered_compressed_stream.h"
#include "waap.h"
#include "compression_utils.h"
#include <sstream>
USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
using namespace std;
void yieldIfPossible(const string &func, int line)
{
// check mainloop exists and current routine is not the main routine
if (Singleton::exists<I_MainLoop>() &&
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->getCurrentRoutineId().ok())
{
dbgDebug(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line;
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
}
}
// Static member definitions
const size_t BufferedCompressedOutputStream::CompressedBuffer::BUFFER_SIZE;
const size_t BufferedCompressedInputStream::DecompressedBuffer::OUTPUT_BUFFER_SIZE;
const size_t BufferedCompressedInputStream::DecompressedBuffer::CHUNK_SIZE;
BufferedCompressedOutputStream::BufferedCompressedOutputStream(ostream &underlying_stream)
:
ostream(nullptr),
m_buffer(make_unique<CompressedBuffer>(underlying_stream))
{
rdbuf(m_buffer.get());
}
BufferedCompressedOutputStream::~BufferedCompressedOutputStream()
{
try {
close();
} catch (exception &e) {
// Destructor should not throw
dbgWarning(D_WAAP_SERIALIZE) << "Exception in BufferedCompressedOutputStream destructor: " << e.what();
}
}
void BufferedCompressedOutputStream::flush()
{
if (m_buffer) {
dbgTrace(D_WAAP_SERIALIZE) << "Flushing internal buffer...";
m_buffer->flushBuffer(); // This will compress and encrypt the current buffer with is_last=false
// and flush the underlying stream.
}
// Do NOT call ostream::flush() here, as it would call sync() on our m_buffer,
// which calls compressAndEncryptBuffer(true) and finalizes the GZIP stream prematurely.
// The m_underlying_stream within m_buffer is flushed by compressAndEncryptBuffer itself.
}
void BufferedCompressedOutputStream::close()
{
if (m_buffer) {
dbgTrace(D_WAAP_SERIALIZE) << "Closing stream and flushing buffer...";
m_buffer->flushAndClose();
}
}
BufferedCompressedOutputStream::CompressedBuffer::CompressedBuffer(ostream &underlying_stream)
:
m_underlying_stream(underlying_stream),
m_buffer(),
m_compression_stream(nullptr),
m_closed(false)
{
m_buffer.reserve(BUFFER_SIZE);
m_compression_stream = initCompressionStream();
}
BufferedCompressedOutputStream::CompressedBuffer::~CompressedBuffer()
{
try {
if (!m_closed) {
sync();
}
if (m_compression_stream) {
finiCompressionStream(m_compression_stream);
m_compression_stream = nullptr;
}
} catch (exception &e) {
// Destructor should not throw
dbgWarning(D_WAAP_SERIALIZE) << "Exception in CompressedBuffer destructor: " << e.what();
}
}
void BufferedCompressedOutputStream::CompressedBuffer::flushAndClose()
{
sync();
}
int BufferedCompressedOutputStream::CompressedBuffer::overflow(int c)
{
if (m_closed) {
dbgTrace(D_WAAP_SERIALIZE) << "Stream is closed, returning EOF";
return traits_type::eof();
}
if (c != traits_type::eof()) {
m_buffer.push_back(static_cast<char>(c));
dbgTrace(D_WAAP_SERIALIZE) << "Added char, buffer size now: " << m_buffer.size();
}
if (m_buffer.size() >= BUFFER_SIZE) {
dbgTrace(D_WAAP_SERIALIZE) << "Buffer full, flushing...";
compressAndEncryptBuffer(false);
}
return c;
}
streamsize BufferedCompressedOutputStream::CompressedBuffer::xsputn(const char* s, streamsize n)
{
if (m_closed) {
dbgDebug(D_WAAP_SERIALIZE) << "Stream is closed, returning 0";
return 0;
}
dbgTrace(D_WAAP_SERIALIZE) << "Writing " << n << " bytes";
streamsize written = 0;
while (written < n) {
size_t space_available = BUFFER_SIZE - m_buffer.size();
size_t to_write = min(static_cast<size_t>(n - written), space_available);
m_buffer.insert(m_buffer.end(), s + written, s + written + to_write);
written += to_write;
dbgTrace(D_WAAP_SERIALIZE) << "Wrote " << to_write << " bytes, total written: " << written
<< ", buffer size: " << m_buffer.size();
if (m_buffer.size() >= BUFFER_SIZE) {
dbgTrace(D_WAAP_SERIALIZE) << "Buffer full, flushing...";
compressAndEncryptBuffer(false);
}
}
dbgTrace(D_WAAP_SERIALIZE) << "Completed, total written: " << written;
return written;
}
int BufferedCompressedOutputStream::CompressedBuffer::sync()
{
dbgTrace(D_WAAP_SERIALIZE) << "Called, closed=" << m_closed << ", buffer size=" << m_buffer.size();
if (!m_closed) {
bool success = compressAndEncryptBuffer(true); // Attempt final compression/encryption
// Mark as closed REGARDLESS of the success of the attempt to ensure finalization logic
// for this context isn't re-attempted if this call failed.
m_closed = true;
if (!success) {
dbgWarning(D_WAAP_SERIALIZE) << "Final compression/encryption failed";
return -1;
}
dbgTrace(D_WAAP_SERIALIZE) << "Stream closed successfully";
} else {
dbgDebug(D_WAAP_SERIALIZE) << "Stream already closed, skipping";
}
return 0;
}
void BufferedCompressedOutputStream::CompressedBuffer::flushBuffer()
{
if (m_buffer.empty() || m_closed) {
return;
}
dbgTrace(D_WAAP_SERIALIZE) << "Flushing buffer with " << m_buffer.size() << " bytes";
compressAndEncryptBuffer(false);
}
bool BufferedCompressedOutputStream::CompressedBuffer::compressAndEncryptBuffer(bool is_last)
{
// If the stream is already marked as closed at this buffer's level,
// it means sync() has run, and everything, including encryption, has been finalized.
if (m_closed) {
dbgTrace(D_WAAP_SERIALIZE) << "Stream is already closed, skipping.";
return true;
}
// Skip if there's nothing to compress and this is not the final flush
if (m_buffer.empty() && !is_last) {
dbgTrace(D_WAAP_SERIALIZE) << "Buffer empty and not last call, skipping.";
return true;
}
dbgTrace(D_WAAP_SERIALIZE) << "Compressing and encrypting " << m_buffer.size() << " bytes, is_last: " << is_last;
// Compress the buffer
CompressionResult result = compressData(
m_compression_stream,
CompressionType::GZIP,
static_cast<uint32_t>(m_buffer.size()),
reinterpret_cast<const unsigned char*>(m_buffer.data()),
is_last ? 1 : 0
);
if (!result.ok) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to compress data";
return false;
}
string compressed_data;
if (result.output && result.num_output_bytes > 0) {
compressed_data = string(reinterpret_cast<const char*>(result.output), result.num_output_bytes);
free(result.output);
}
dbgDebug(D_WAAP_SERIALIZE) << "Compression complete: " << m_buffer.size()
<< " bytes -> " << compressed_data.size() << " bytes";
// Yield after compression to allow other routines to run
YIELD_IF_POSSIBLE();
string final_data = compressed_data;
// Write to underlying stream only if we have data to write
if (!final_data.empty()) {
m_underlying_stream.write(final_data.c_str(), final_data.size());
m_underlying_stream.flush();
}
m_buffer.clear();
// Yield after writing chunk to allow other routines to run
YIELD_IF_POSSIBLE();
return true;
}
BufferedCompressedInputStream::BufferedCompressedInputStream(istream &underlying_stream)
:
istream(nullptr),
m_buffer(make_unique<DecompressedBuffer>(underlying_stream))
{
rdbuf(m_buffer.get());
}
BufferedCompressedInputStream::~BufferedCompressedInputStream()
{
// DecompressedBuffer destructor will handle cleanup
}
BufferedCompressedInputStream::DecompressedBuffer::DecompressedBuffer(istream &underlying_stream)
:
m_underlying_stream(underlying_stream),
m_buffer(),
m_encrypted_buffer(),
m_compressed_buffer(),
m_decompressed_buffer(),
m_decompressed_pos(0),
m_compression_stream(nullptr),
m_eof_reached(false),
m_stream_finished(false)
{
m_buffer.resize(OUTPUT_BUFFER_SIZE);
m_encrypted_buffer.reserve(CHUNK_SIZE);
m_compressed_buffer.reserve(CHUNK_SIZE);
m_decompressed_buffer.reserve(OUTPUT_BUFFER_SIZE);
m_compression_stream = initCompressionStream();
// Set buffer pointers to indicate empty buffer
setg(m_buffer.data(), m_buffer.data(), m_buffer.data());
}
BufferedCompressedInputStream::DecompressedBuffer::~DecompressedBuffer()
{
try {
if (m_compression_stream) {
finiCompressionStream(m_compression_stream);
m_compression_stream = nullptr;
}
} catch (exception &e) {
// Destructor should not throw
dbgWarning(D_WAAP_SERIALIZE) << "Exception in DecompressedBuffer destructor: " << e.what();
}
}
int BufferedCompressedInputStream::DecompressedBuffer::underflow()
{
if (gptr() < egptr()) {
return traits_type::to_int_type(*gptr());
}
if (m_eof_reached) {
return traits_type::eof();
}
if (!fillBuffer()) {
m_eof_reached = true;
return traits_type::eof();
}
return traits_type::to_int_type(*gptr());
}
streamsize BufferedCompressedInputStream::DecompressedBuffer::xsgetn(char* s, streamsize n)
{
streamsize total_read = 0;
while (total_read < n) {
if (gptr() >= egptr()) {
if (!fillBuffer()) {
m_eof_reached = true;
break;
}
}
streamsize available = egptr() - gptr();
streamsize to_copy = min(n - total_read, available);
memcpy(s + total_read, gptr(), to_copy);
gbump(static_cast<int>(to_copy));
total_read += to_copy;
}
return total_read;
}
bool BufferedCompressedInputStream::DecompressedBuffer::fillBuffer()
{
if (m_eof_reached) {
return false;
}
// If we have remaining data in the decompressed buffer, use it first
if (m_decompressed_pos < m_decompressed_buffer.size()) {
size_t remaining = m_decompressed_buffer.size() - m_decompressed_pos;
size_t to_copy = min(remaining, OUTPUT_BUFFER_SIZE);
memcpy(m_buffer.data(), m_decompressed_buffer.data() + m_decompressed_pos, to_copy);
m_decompressed_pos += to_copy;
// Set up the buffer pointers for streambuf:
// eback() = m_buffer.data() (start of buffer)
// gptr() = m_buffer.data() (current position)
// egptr() = m_buffer.data() + to_copy (end of valid data)
setg(m_buffer.data(), m_buffer.data(), m_buffer.data() + to_copy);
dbgTrace(D_WAAP_SERIALIZE) << "Serving " << to_copy << " bytes from existing decompressed buffer";
// Yield after serving data from buffer to allow other routines to run
YIELD_IF_POSSIBLE();
return true;
}
// Need to process the next chunk
if (!processNextChunk()) {
m_eof_reached = true;
return false;
}
// Now try again with the new data
return fillBuffer();
}
bool BufferedCompressedInputStream::DecompressedBuffer::processNextChunk()
{
while (true) {
if (m_stream_finished) {
return false;
}
// Read a chunk of encrypted data from the underlying stream
if (m_encrypted_buffer.size() < CHUNK_SIZE) {
m_encrypted_buffer.resize(CHUNK_SIZE);
}
m_underlying_stream.read(m_encrypted_buffer.data(), CHUNK_SIZE);
streamsize bytes_read = m_underlying_stream.gcount();
if (bytes_read <= 0) {
m_stream_finished = true;
// End of stream - no more data to process
dbgTrace(D_WAAP_SERIALIZE) << "Reached end of input stream";
return false;
}
m_encrypted_buffer.resize(bytes_read);
dbgTrace(D_WAAP_SERIALIZE) << "Read " << bytes_read << " encrypted bytes from stream";
// Decrypt the chunk
std::vector<char> decrypted_chunk;
if (!decryptChunk(m_encrypted_buffer, decrypted_chunk)) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to decrypt chunk";
break;
}
// Decompress the chunk
std::vector<char> decompressed_chunk;
if (!decompressChunk(decrypted_chunk, decompressed_chunk)) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to decompress chunk";
break;
}
if (decompressed_chunk.empty()) {
dbgTrace(D_WAAP_SERIALIZE) << "Decompressed chunk is empty, skipping";
continue; // Nothing to add to the buffer
}
// Replace the decompressed buffer with new data using swap to avoid unnecessary allocations
m_decompressed_buffer.swap(decompressed_chunk);
m_decompressed_pos = 0;
dbgTrace(D_WAAP_SERIALIZE) << "Processed chunk: " << bytes_read
<< " encrypted -> " << decrypted_chunk.size()
<< " compressed -> " << m_decompressed_buffer.size() << " decompressed";
// Yield after processing chunk to allow other routines to run
YIELD_IF_POSSIBLE();
return true;
}
return false;
}
bool BufferedCompressedInputStream::DecompressedBuffer::decryptChunk(
const std::vector<char> &encrypted_chunk,
std::vector<char> &decrypted_chunk)
{
// No encryption - just copy the data
decrypted_chunk = encrypted_chunk;
return true;
}
bool BufferedCompressedInputStream::DecompressedBuffer::decompressChunk(
const std::vector<char> &compressed_chunk,
std::vector<char> &decompressed_chunk)
{
if (compressed_chunk.empty()) {
return true; // Nothing to decompress
}
// Use the streaming decompression
DecompressionResult result = decompressData(
m_compression_stream,
compressed_chunk.size(),
reinterpret_cast<const unsigned char*>(compressed_chunk.data())
);
if (!result.ok) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to decompress chunk";
return false;
}
if (result.output && result.num_output_bytes > 0) {
decompressed_chunk.assign(
reinterpret_cast<const char*>(result.output),
reinterpret_cast<const char*>(result.output) + result.num_output_bytes
);
free(result.output);
dbgTrace(D_WAAP_SERIALIZE) << "Decompressed chunk: " << compressed_chunk.size()
<< " -> " << decompressed_chunk.size() << " bytes";
// Yield after decompression to allow other routines to run
YIELD_IF_POSSIBLE();
return true;
}
// No output data yet (might need more input for compression algorithm)
decompressed_chunk.clear();
return true;
}

10
components/security_apps/waap/waap_component.cc
View File

@@ -50,10 +50,16 @@ void
WaapComponent::preload()
{
// TODO:: call stuff like registerExpectedCofiguration here..
registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
// registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
// registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
registerExpectedConfigurationWithCache<WaapConfigApplication>(
"assetId", "WAAP", "WebApplicationSecurity");
registerExpectedConfigurationWithCache<WaapConfigAPI>("assetId", "WAAP", "WebAPISecurity");
registerExpectedConfiguration<std::string>("WAAP", "Sigs file path");
registerExpectedConfigFile("waap", Config::ConfigFileType::Policy);
registerExpectedSetting<bool>("features", "learningLeader");
registerConfigLoadCb(
[this]()
{

83
components/security_apps/waap/waap_component_impl.cc
View File

@@ -24,6 +24,7 @@
#include <libxml/parser.h>
#include "debug.h"
#include "user_identifiers_config.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "waap_clib/Waf2Engine.h"
#include "waap_clib/WaapConfigApi.h"
@@ -45,10 +46,10 @@ USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
USE_DEBUG_FLAG(D_NGINX_EVENTS);
WaapComponent::Impl::Impl() :
pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT),
accept_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT),
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
limit_response_headers(ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS),
pending_response(ServiceVerdict::TRAFFIC_VERDICT_INSPECT),
accept_response(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT),
drop_response(ServiceVerdict::TRAFFIC_VERDICT_DROP),
limit_response_headers(ServiceVerdict::LIMIT_RESPONSE_HEADERS),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer()
@@ -221,17 +222,47 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event)
IWaf2Transaction& waf2Transaction = waapStateTable->getState<Waf2Transaction>();
// Tell waf2 API that another request header arrived
waf2Transaction.add_request_hdr(
reinterpret_cast<const char *>(header_name.data()), //const char * name //
header_name.size(), //int name_len //
reinterpret_cast<const char *>(header_value.data()), //const char * value //
header_value.size() //int value_len //
);
if (event.shouldLog()) {
waf2Transaction.add_request_hdr(
reinterpret_cast<const char *>(header_name.data()), //const char * name //
header_name.size(), //int name_len //
reinterpret_cast<const char *>(header_value.data()), //const char * value //
header_value.size() //int value_len //
);
} else {
dbgTrace(D_WAAP)
<< "Header '"
<< std::dumpHex(header_name)
<< "' marked as should not log, skipping adding to WAF2";
}
EventVerdict verdict = pending_response;
// Last header handled
if (event.isLastHeader()) {
// NEXUS env should take real client ip from X-Forwarded-For header
if (getProfileAgentSettingWithDefault<bool>(false, "agent.saasProfile.ignoreSourceIP")) {
auto env = Singleton::Consume<I_Environment>::by<WaapComponent>();
auto maybe_xff = env->get<std::string>(HttpTransactionData::xff_vals_ctx);
if (!maybe_xff.ok()) {
dbgTrace(D_WAAP) << "failed to get xff vals from env";
} else {
// Extract the last IP from XFF header (actual client IP)
std::string xff_header = maybe_xff.unpack();
size_t last_comma_pos = xff_header.find_last_of(',');
std::string sourceIpStr = (last_comma_pos != std::string::npos) ?
xff_header.substr(last_comma_pos + 1) : xff_header;
// Trim whitespace
sourceIpStr.erase(0, sourceIpStr.find_first_not_of(" \t"));
sourceIpStr.erase(sourceIpStr.find_last_not_of(" \t") + 1);
dbgTrace(D_WAAP) << "taking source IP for nexus from last xff header: " << sourceIpStr;
waf2Transaction.set_transaction_remote(sourceIpStr.c_str(), NEXUS_PORT);
}
}
waf2Transaction.end_request_hdrs();
verdict = waf2Transaction.getUserLimitVerdict();
@@ -283,9 +314,9 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event)
waf2Transaction.add_request_body_chunk(dataBuf, dataBufLen);
ngx_http_cp_verdict_e verdict = waf2Transaction.getUserLimitVerdict();
ServiceVerdict verdict = waf2Transaction.getUserLimitVerdict();
EventVerdict eventVedict(verdict);
if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) {
if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT) {
finishTransaction(waf2Transaction, eventVedict);
}
@@ -324,8 +355,8 @@ WaapComponent::Impl::respond(const EndRequestEvent &)
EventVerdict verdict = waapDecision(waf2Transaction);
// Delete state before returning any verdict which is not pending
if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
verdict.getVerdict() != ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS &&
if (verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INSPECT &&
verdict.getVerdict() != ServiceVerdict::LIMIT_RESPONSE_HEADERS &&
waapStateTable->hasState<Waf2Transaction>()
) {
finishTransaction(waf2Transaction, verdict);
@@ -373,8 +404,8 @@ WaapComponent::Impl::respond(const ResponseCodeEvent &event)
}
// Delete state before returning any verdict which is not pending
if (verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
verdict.getVerdict() != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
if (verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INSPECT &&
verdict.getVerdict() != ServiceVerdict::TRAFFIC_VERDICT_INJECT &&
waapStateTable->hasState<Waf2Transaction>()
) {
finishTransaction(waf2Transaction, verdict);
@@ -413,7 +444,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
header_value.size()
);
ngx_http_cp_verdict_e verdict = pending_response.getVerdict();
ServiceVerdict verdict = pending_response.getVerdict();
HttpHeaderModification modifications;
std::string webUserResponseByPractice;
bool isSecurityHeadersInjected = false;
@@ -441,7 +472,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
}
}
isSecurityHeadersInjected = true;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT;
}
}
@@ -461,7 +492,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
<< ". Error: "
<< result.getErr();
}
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT;
}
}
@@ -476,15 +507,15 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
}
if (waf2Transaction.shouldInjectSecurityHeaders() && isSecurityHeadersInjected &&
verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT
verdict == ServiceVerdict::TRAFFIC_VERDICT_INJECT
) {
// disable should inject security headers after injection to avoid response body scanning when it's unnecessary
waf2Transaction.disableShouldInjectSecurityHeaders();
}
EventVerdict eventVedict(move(modifications.getModificationList()), verdict);
// Delete state before returning any verdict which is not pending
if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT &&
verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT &&
waapStateTable->hasState<Waf2Transaction>()
) {
finishTransaction(waf2Transaction, eventVedict);
@@ -530,7 +561,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
waf2Transaction.add_response_body_chunk(dataBuf, dataBufLen);
ngx_http_cp_verdict_e verdict = pending_response.getVerdict();
ServiceVerdict verdict = pending_response.getVerdict();
HttpBodyModification modifications;
std::string webUserResponseByPractice;
@@ -571,7 +602,7 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
if(!result.ok()) {
dbgWarning(D_WAAP) << "HttpBodyResponse(): Scripts injection failed!";
}
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT;
} else {
// This response body is not considered "HTML" - disable injection
dbgTrace(D_WAAP) << "HttpBodyResponse(): the response body is not HTML - disabling injection";
@@ -589,8 +620,8 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
}
EventVerdict eventVedict(modifications.getModificationList(), verdict);
// Delete state before returning any verdict which is not pending or inject
if (verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT &&
verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT &&
if (verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT &&
verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT &&
waapStateTable->hasState<Waf2Transaction>()
) {
finishTransaction(waf2Transaction, eventVedict);