github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2026-01-17 16:00:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2026-01-03 18:59:01 +00:00
parent c1058db57d
commit 2105628f05
188 changed files with 8272 additions and 2723 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc
View File

@@ -28,10 +28,10 @@ initAttachmentConfig(c_str conf_file)
return conf_data.init(conf_file);
}
ngx_http_inspection_mode_e
NanoHttpInspectionMode
getInspectionMode()
{
return static_cast<ngx_http_inspection_mode_e>(conf_data.getNumericalValue("nginx_inspection_mode"));
return static_cast<NanoHttpInspectionMode>(conf_data.getNumericalValue("nginx_inspection_mode"));
}
unsigned int
@@ -191,6 +191,24 @@ getRemoveResServerHeader()
return conf_data.getNumericalValue("remove_server_header");
}
unsigned int
getDecompressionPoolSize()
{
return conf_data.getNumericalValue("decompression_pool_size");
}
unsigned int
getRecompressionPoolSize()
{
return conf_data.getNumericalValue("recompression_pool_size");
}
unsigned int
getIsBrotliInspectionEnabled()
{
return conf_data.getNumericalValue("is_brotli_inspection_enabled");
}
int
isIPAddress(c_str ip_str)
{
@@ -285,3 +303,15 @@ isSkipSource(c_str ip_str)
return 0;
}
unsigned int
isPairedAffinityEnabled()
{
return conf_data.getNumericalValue("is_paired_affinity_enabled") != 0;
}
unsigned int
isAsyncModeEnabled()
{
return conf_data.getNumericalValue("is_async_mode_enabled") != 0;
}

15
attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc
View File

@@ -69,7 +69,12 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
"\"hold_verdict_retries\": 3,\n"
"\"hold_verdict_polling_time\": 1,\n"
"\"body_size_trigger\": 777,\n"
"\"remove_server_header\": 1\n"
"\"remove_server_header\": 1,\n"
"\"decompression_pool_size\": 524288,\n"
"\"recompression_pool_size\": 32768,\n"
"\"is_paired_affinity_enabled\": 0,\n"
"\"is_async_mode_enabled\": 0,\n"
"\"is_brotli_inspection_enabled\": 1\n"
"}\n";
ofstream valid_configuration_file(attachment_configuration_file_name);
valid_configuration_file << valid_configuration;
@@ -97,10 +102,13 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
EXPECT_EQ(getMaxRetriesForVerdict(), 3u);
EXPECT_EQ(getReqBodySizeTrigger(), 777u);
EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75u);
EXPECT_EQ(getInspectionMode(), ngx_http_inspection_mode::BLOCKING_THREAD);
EXPECT_EQ(getInspectionMode(), NanoHttpInspectionMode::BLOCKING_THREAD);
EXPECT_EQ(getRemoveResServerHeader(), 1u);
EXPECT_EQ(getDecompressionPoolSize(), 524288u);
EXPECT_EQ(getRecompressionPoolSize(), 32768u);
EXPECT_EQ(getHoldVerdictRetries(), 3u);
EXPECT_EQ(getHoldVerdictPollingTime(), 1u);
EXPECT_EQ(getIsBrotliInspectionEnabled(), 1u);
EXPECT_EQ(isDebugContext("1.2.3.4", "5.6.7.8", 80, "GET", "test", "/abc"), 1);
EXPECT_EQ(isDebugContext("1.2.3.9", "5.6.7.8", 80, "GET", "test", "/abc"), 0);
@@ -125,6 +133,9 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:4"), 1);
EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:5"), 1);
EXPECT_EQ(isSkipSource("0:0:0:0:0:0:0:6"), 0);
EXPECT_EQ(isPairedAffinityEnabled(), 0u);
EXPECT_EQ(isAsyncModeEnabled(), 0u);
}
TEST_F(HttpAttachmentUtilTest, CheckIPAddrValidity)

2
components/attachment-intakers/attachment_registrator/attachment_registrator.cc
View File

@@ -33,7 +33,7 @@
#include "i_mainloop.h"
#include "buffer.h"
#include "enum_array.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
USE_DEBUG_FLAG(D_ATTACHMENT_REGISTRATION);

620
components/attachment-intakers/nginx_attachment/nginx_attachment.cc
View File

@@ -15,6 +15,8 @@
#include <pwd.h>
#include <grp.h>
#include <sched.h>
#include <errno.h>
#include <iostream>
#include <map>
#include <queue>
@@ -29,6 +31,7 @@
#include <unistd.h>
#include <utility>
#include <stdarg.h>
#include <cstring>
#include <boost/range/iterator_range.hpp>
#include <boost/algorithm/string.hpp>
@@ -48,7 +51,7 @@
#include "shmem_ipc.h"
#include "i_http_manager.h"
#include "http_transaction_common.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "hash_combine.h"
#include "cpu/failopen_mode_status.h"
#include "attachment_registrator.h"
@@ -75,7 +78,7 @@ USE_DEBUG_FLAG(D_METRICS_NGINX_ATTACHMENT);
using namespace std;
using ChunkType = ngx_http_chunk_type_e;
using ChunkType = AttachmentDataType;
static const uint32_t corrupted_session_id = CORRUPTED_SESSION_ID;
static const AlertInfo alert(AlertTeam::CORE, "nginx attachment");
@@ -130,14 +133,15 @@ class NginxAttachment::Impl
:
Singleton::Provide<I_StaticResourcesHandler>::From<NginxAttachment>
{
static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
static constexpr auto LIMIT_RESPONSE_HEADERS = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto INJECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
static constexpr auto IRRELEVANT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
static constexpr auto RECONF = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_RECONF;
static constexpr auto WAIT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
static constexpr auto LIMIT_RESPONSE_HEADERS = ServiceVerdict::LIMIT_RESPONSE_HEADERS;
static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP;
static constexpr auto INJECT = ServiceVerdict::TRAFFIC_VERDICT_INJECT;
static constexpr auto IRRELEVANT = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
static constexpr auto RECONF = ServiceVerdict::TRAFFIC_VERDICT_RECONF;
static constexpr auto WAIT = ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
static constexpr auto CUSTOM_RESPONSE = ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE;
public:
Impl()
@@ -197,6 +201,8 @@ public:
NUM_OF_NGINX_IPC_ELEMENTS, "nginxAttachment.numOfNginxIpcElements"
);
dbgInfo(D_NGINX_ATTACHMENT) << "Async mode configuration: enabled=" << attachment_config.isAsyncModeEnabled();
nginx_attachment_metric.init(
"Nginx Attachment data",
ReportIS::AudienceTeam::AGENT_CORE,
@@ -366,10 +372,13 @@ public:
}
bool
registerStaticResource(const string &resource_name, const string &resource_path)
registerStaticResource(
const string &resource_name,
const string &resource_path,
bool overwrite_if_exists)
{
string dest_path = static_resources_path + "/" + resource_name;
if (NGEN::Filesystem::exists(dest_path)) {
if (!overwrite_if_exists && NGEN::Filesystem::exists(dest_path)) {
dbgDebug(D_NGINX_ATTACHMENT) << "Static resource already exist. path: " << dest_path;
return true;
}
@@ -377,7 +386,7 @@ public:
if (!NGEN::Filesystem::copyFile(
resource_path,
dest_path,
false,
overwrite_if_exists,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH
)) {
dbgWarning(D_NGINX_ATTACHMENT)
@@ -397,6 +406,36 @@ public:
return true;
}
bool
registerStaticResourceByContent(
const string &resource_name,
const string &file_content
)
{
string dest_path = static_resources_path + "/" + resource_name;
if (!NGEN::Filesystem::createFileWithContent(
dest_path,
file_content,
true,
S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH
)) {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Failed to create static resource file. Resource name: "
<< resource_name
<< ", destination path: " << dest_path
<< ", content size: " << file_content.size();
return false;
}
dbgTrace(D_NGINX_ATTACHMENT)
<< "Successfully created static resource. Resource name: "
<< resource_name
<< ", destination path: " << dest_path
<< ", content size: " << file_content.size();
return true;
}
void
printMetrics()
{
@@ -443,6 +482,11 @@ private:
attachment_routine_id = 0;
}
if (async_attachment_routine_id > 0 && mainloop->doesRoutineExist(async_attachment_routine_id)) {
mainloop->stop(async_attachment_routine_id);
async_attachment_routine_id = 0;
}
string curr_instance_unique_id = inst_awareness->getUniqueID().unpack();
if (attachment_ipc != nullptr) {
if (nginx_worker_user_id != nginx_user_id || nginx_worker_group_id != nginx_group_id) {
@@ -548,8 +592,12 @@ private:
);
while (isSignalPending()) {
if (attachment_config.isAsyncModeEnabled()) {
if (!handleInspectionAsync()) break;
} else {
if (!handleInspection()) break;
}
}
},
"Nginx Attachment inspection handler",
true
@@ -791,7 +839,7 @@ private:
}
void
sendMetricToKibana(const ngx_http_cp_metric_data_t *received_metric_data)
sendMetricToKibana(const NanoHttpMetricData *received_metric_data)
{
nginx_intaker_event.addPluginMetricCounter(received_metric_data);
nginx_intaker_event.notify();
@@ -822,7 +870,7 @@ private:
return "Request End";
case ChunkType::METRIC_DATA_FROM_PLUGIN:
return "Metrics";
case ChunkType::HOLD_DATA:
case ChunkType::REQUEST_DELAYED_VERDICT:
return "HOLD_DATA";
case ChunkType::COUNT:
dbgAssert(false) << alert << "Invalid 'COUNT' ChunkType";
@@ -842,7 +890,7 @@ private:
return default_verdict;
}
auto rule_by_ctx = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
auto rule_by_ctx = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
if (rule_by_ctx.ok()) {
BasicRuleConfig rule = rule_by_ctx.unpack();
opaque.setSavedData("assetId", rule.getAssetId(), EnvKeyAttr::LogSection::SOURCEANDDATA);
@@ -917,6 +965,7 @@ private:
FilterVerdict cur_verdict = http_manager->inspect(chunk, is_request);
if (cur_verdict.getVerdict() == ACCEPT ||
cur_verdict.getVerdict() == DROP ||
cur_verdict.getVerdict() == CUSTOM_RESPONSE ||
cur_verdict.getVerdict() == WAIT) {
return cur_verdict;
}
@@ -1030,16 +1079,16 @@ private:
handleChunkedData(ChunkType chunk_type, const Buffer &data, NginxAttachmentOpaque &opaque)
{
ScopedContext event_type;
event_type.registerValue<ngx_http_chunk_type_e>("HTTP Chunk type", chunk_type);
event_type.registerValue<AttachmentDataType>("HTTP Chunk type", chunk_type);
if (chunk_type > ChunkType::REQUEST_HEADER && opaque.getApplicationState() == ApplicationState::UNKOWN) {
auto rule_by_ctx = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
auto rule_by_ctx = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
ApplicationState state = rule_by_ctx.ok() ? ApplicationState::DEFINED : ApplicationState::UNDEFINED;
opaque.setApplicationState(state);
}
if (opaque.getApplicationState() == ApplicationState::UNDEFINED) {
ngx_http_cp_verdict_e verdict_action =
ServiceVerdict verdict_action =
getSettingWithDefault<bool>(false, "allowOnlyDefinedApplications") ? DROP : ACCEPT;
dbgDebug(D_NGINX_ATTACHMENT)
@@ -1079,8 +1128,8 @@ private:
case ChunkType::RESPONSE_END:
return FilterVerdict(http_manager->inspectEndTransaction());
case ChunkType::METRIC_DATA_FROM_PLUGIN:
return FilterVerdict(ngx_http_cp_verdict::TRAFFIC_VERDICT_IRRELEVANT);
case ChunkType::HOLD_DATA:
return FilterVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT);
case ChunkType::REQUEST_DELAYED_VERDICT:
return FilterVerdict(http_manager->inspectDelayedVerdict());
case ChunkType::COUNT:
break;
@@ -1106,10 +1155,10 @@ private:
<< "Handling Injection of HTTP session modification data. Modifications amount: "
<< modifications_amount;
vector<ngx_http_cp_inject_data> injection_data_persistency(modifications_amount);
vector<HttpInjectData> injection_data_persistency(modifications_amount);
for (const EventModifications &modifications : modifications_lists) {
for (const ModificationBuffer &modification_buffer_list : modifications.second) {
ngx_http_cp_inject_data injection_data;
HttpInjectData injection_data;
injection_data.orig_buff_index = modifications.first;
injection_data.injection_pos = std::get<0>(modification_buffer_list);
injection_data.mod_type = std::get<1>(modification_buffer_list);
@@ -1149,22 +1198,9 @@ private:
SharedMemoryIPC *ipc,
vector<const char *> &verdict_data,
vector<uint16_t> &verdict_data_sizes,
string web_user_response_id)
const WebTriggerConf &web_trigger_conf)
{
ngx_http_cp_web_response_data_t web_response_data;
ScopedContext ctx;
if (web_user_response_id != "") {
dbgTrace(D_NGINX_ATTACHMENT)
<< "web user response ID registered in contex: "
<< web_user_response_id;
set<string> triggers_set{web_user_response_id};
ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
}
WebTriggerConf web_trigger_conf = getConfigurationWithDefault<WebTriggerConf>(
WebTriggerConf::default_trigger_conf,
"rulebase",
"webUserResponse"
);
HttpWebResponseData web_response_data;
bool remove_event_id_param =
getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
@@ -1176,8 +1212,16 @@ private:
uuid = opaque.getSessionUUID();
}
web_response_data.uuid_size = uuid.size();
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
string details_level = web_trigger_conf.getDetailsLevel();
if (details_level == "Custom Block Page") {
auto body = web_trigger_conf.getResponseBody();
auto content_type = web_trigger_conf.getContentType();
auto response_code = web_trigger_conf.getResponseCode();
string processed_body = process_html_content(body, uuid);
auto custom_response = CustomResponse(processed_body, response_code, content_type);
handleCustomResponse(ipc, verdict_data, verdict_data_sizes, custom_response);
return;
} else if (details_level == "Redirect") {
web_response_data.response_data.redirect_data.redirect_location_size =
web_trigger_conf.getRedirectURL().size();
bool add_event = web_trigger_conf.getAddEventId();
@@ -1186,19 +1230,19 @@ private:
strlen("?event_id=") + uuid.size();
}
web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
web_response_data.web_response_type = static_cast<uint8_t>(NanoWebResponseType::REDIRECT_WEB_RESPONSE);
} else {
web_response_data.response_data.custom_response_data.title_size =
web_trigger_conf.getResponseTitle().size();
web_response_data.response_data.custom_response_data.body_size = web_trigger_conf.getResponseBody().size();
web_response_data.response_data.custom_response_data.response_code = web_trigger_conf.getResponseCode();
web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::CUSTOM_WEB_RESPONSE);
web_response_data.web_response_type = static_cast<uint8_t>(NanoWebResponseType::CUSTOM_WEB_RESPONSE);
}
verdict_data.push_back(reinterpret_cast<const char *>(&web_response_data));
verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));
verdict_data_sizes.push_back(sizeof(HttpWebResponseData));
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
if (details_level == "Redirect") {
redirectUrl = web_trigger_conf.getRedirectURL();
if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
redirectUrl += "?event-id=" + uuid;
@@ -1249,16 +1293,81 @@ private:
<< uuid
<< " (UUID size: "
<< static_cast<uint>(web_response_data.uuid_size)
<< "), Response Type: "
<< static_cast<uint>(web_response_data.web_response_type)
<< ")";
}
sendChunkedData(ipc, verdict_data_sizes.data(), verdict_data.data(), verdict_data.size());
}
string
process_html_content(const string &body, const string &uuid)
{
const string uuid_placeholder = "<!-- CHECK_POINT_USERCHECK_UUID_PLACEHOLDER-->";
size_t placeholder_pos = body.find(uuid_placeholder);
if (placeholder_pos == string::npos) {
dbgTrace(D_NGINX_ATTACHMENT) << "No UUID placeholder found in body content";
return body;
}
dbgTrace(D_NGINX_ATTACHMENT) << "Found UUID placeholder at position: " << placeholder_pos;
string incident_id_text = "Incident Id: " + uuid;
string processed_body = body;
processed_body.replace(placeholder_pos, uuid_placeholder.length(), incident_id_text);
dbgTrace(D_NGINX_ATTACHMENT)
<< "UUID replacement: original_len=" << body.length()
<< ", processed_len=" << processed_body.length()
<< ", uuid=" << uuid;
return processed_body;
}
void
handleCustomResponse(
SharedMemoryIPC *ipc,
vector<const char *> &verdict_data,
vector<uint16_t> &verdict_data_sizes,
const CustomResponse &custom_response_data)
{
HttpJsonResponseData json_response_data;
json_response_data.response_code = custom_response_data.getStatusCode();
string response_body = custom_response_data.getBody();
string content_type = custom_response_data.getContentType();
json_response_data.body_size = response_body.size();
if (content_type == "application/json") {
json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_APPLICATION_JSON;
} else if (content_type == "text/html") {
json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_TEXT_HTML;
} else {
json_response_data.content_type = AttachmentContentType::CONTENT_TYPE_OTHER;
}
verdict_data.push_back(reinterpret_cast<const char *>(&json_response_data));
verdict_data_sizes.push_back(sizeof(HttpJsonResponseData));
verdict_data.push_back(reinterpret_cast<const char *>(response_body.data()));
verdict_data_sizes.push_back(response_body.size());
dbgTrace(D_NGINX_ATTACHMENT)
<< "Added Custom JSON Response to current session."
<< " Response code: "
<< static_cast<uint>(json_response_data.response_code)
<< ", Body size: "
<< static_cast<uint>(json_response_data.body_size);
sendChunkedData(ipc, verdict_data_sizes.data(), verdict_data.data(), verdict_data.size());
}
void
handleVerdictResponse(const FilterVerdict &verdict, SharedMemoryIPC *ipc, SessionID session_id, bool is_header)
{
ngx_http_cp_reply_from_service_t verdict_to_send;
HttpReplyFromService verdict_to_send;
verdict_to_send.verdict = static_cast<uint16_t>(verdict.getVerdict());
verdict_to_send.session_id = session_id;
@@ -1281,7 +1390,31 @@ private:
if (verdict.getVerdict() == DROP) {
nginx_attachment_event.addTrafficVerdictCounter(nginxAttachmentEvent::trafficVerdict::DROP);
verdict_to_send.modification_count = 1;
return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, verdict.getWebUserResponseID());
ScopedContext ctx;
set<string> triggers_set{verdict.getWebUserResponseID()};
ctx.registerValue<set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto web_trigger_config = getConfiguration<WebTriggerConf>("rulebase", "webUserResponse");
WebTriggerConf web_trigger_conf = web_trigger_config.ok() ?
web_trigger_config.unpack() : WebTriggerConf::default_trigger_conf;
if (web_trigger_conf.getDetailsLevel() == "Custom Block Page") {
dbgTrace(D_NGINX_ATTACHMENT) << "Changing verdict from DROP to CUSTOM_BLOCK_PAGE";
verdict_to_send.verdict = static_cast<uint16_t>(CUSTOM_RESPONSE);
}
return handleCustomWebResponse(ipc, verdict_fragments, fragments_sizes, web_trigger_conf);
}
if (verdict.getVerdict() == CUSTOM_RESPONSE) {
verdict_to_send.modification_count = 1;
if (!verdict.getCustomResponse().ok()) {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Failed to get custom web response data. Returning default verdict: "
<< ", Error: "
<< verdict.getCustomResponse().getErr();
return handleCustomResponse(ipc, verdict_fragments, fragments_sizes,
CustomResponse("{\"error\": \"Internal Server Error\"}", 500));
}
return handleCustomResponse(ipc, verdict_fragments, fragments_sizes, verdict.getCustomResponse().unpack());
}
if (verdict.getVerdict() == ACCEPT) {
@@ -1329,7 +1462,7 @@ private:
break;
}
auto transaction_data = reinterpret_cast<const ngx_http_cp_request_data_t *>(incoming_data);
auto transaction_data = reinterpret_cast<const NanoHttpRequestData *>(incoming_data);
if (transaction_data->session_id != cur_session_id) break;
popData(attachment_ipc);
@@ -1371,15 +1504,15 @@ private:
}
if (SHOULD_FAIL(
incoming_data_size >= sizeof(ngx_http_cp_request_data_t),
incoming_data_size >= sizeof(NanoHttpRequestData),
IntentionalFailureHandler::FailureType::GetDataFromAttchment,
&did_fail_on_purpose
)) {
dbgError(D_NGINX_ATTACHMENT)
<< "Corrupted transaction raw data received from NGINX attachment, size received: "
<< incoming_data_size
<< " is lower than ngx_http_cp_request_data_t size="
<< sizeof(ngx_http_cp_request_data_t)
<< " is lower than NanoHttpRequestData size="
<< sizeof(NanoHttpRequestData)
<< ". Resetting IPC"
<< dumpIpcWrapper(attachment_ipc)
<< (did_fail_on_purpose ? "[Intentional Failure]" : "");
@@ -1409,8 +1542,8 @@ private:
return make_pair(corrupted_session_id, false);
}
const ngx_http_cp_request_data_t *transaction_data =
reinterpret_cast<const ngx_http_cp_request_data_t *>(incoming_data);
const NanoHttpRequestData *transaction_data =
reinterpret_cast<const NanoHttpRequestData *>(incoming_data);
Maybe<ChunkType> chunked_data_type = convertToEnum<ChunkType>(transaction_data->data_type);
if (!chunked_data_type.ok()) {
@@ -1426,8 +1559,8 @@ private:
}
if (chunked_data_type.unpack() == ChunkType::METRIC_DATA_FROM_PLUGIN) {
const ngx_http_cp_metric_data_t *recieved_metric_data =
reinterpret_cast<const ngx_http_cp_metric_data_t *>(incoming_data);
const NanoHttpMetricData *recieved_metric_data =
reinterpret_cast<const NanoHttpMetricData *>(incoming_data);
sendMetricToKibana(recieved_metric_data);
popData(attachment_ipc);
return pair<uint32_t, bool>(0, false);
@@ -1478,7 +1611,7 @@ private:
const Buffer inspection_data(
transaction_data->data,
incoming_data_size - sizeof(ngx_http_cp_request_data_t),
incoming_data_size - sizeof(NanoHttpRequestData),
Buffer::MemoryType::VOLATILE
);
@@ -1522,6 +1655,7 @@ private:
bool is_final_verdict = verdict.getVerdict() == ACCEPT ||
verdict.getVerdict() == DROP ||
verdict.getVerdict() == CUSTOM_RESPONSE ||
verdict.getVerdict() == IRRELEVANT;
dbgTrace(D_NGINX_ATTACHMENT)
@@ -1638,6 +1772,8 @@ private:
return "RECONF";
case WAIT:
return "WAIT";
case CUSTOM_RESPONSE:
return "CUSTOM_RESPONSE";
}
dbgAssert(false) << alert << "Invalid EventVerdict enum: " << static_cast<int>(verdict.getVerdict());
return string();
@@ -1716,7 +1852,16 @@ private:
Maybe<string> uid = getUidFromSocket(new_attachment_socket);
Maybe<uint32_t> nginx_user_id = readIdFromSocket(new_attachment_socket);
Maybe<uint32_t> nginx_group_id = readIdFromSocket(new_attachment_socket);
Maybe<int32_t> target_core = readSignedIdFromSocket(new_attachment_socket);
DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::RegisterAttchment);
// Target core failure should not block registration - maintain backward compatibility
bool target_core_available = target_core.ok();
if (!target_core_available) {
dbgInfo(D_NGINX_ATTACHMENT) << "Failed to read target core (backward compatibility mode): "
<< target_core.getErr() << ". Proceeding without affinity information.";
}
if (SHOULD_FAIL(
nginx_user_id.ok() && nginx_group_id.ok() && uid.ok(),
IntentionalFailureHandler::FailureType::RegisterAttchment,
@@ -1743,6 +1888,7 @@ private:
return;
}
int32_t target_core_value = target_core_available ? *target_core : -1;
if (!registerAttachmentProcess(*nginx_user_id, *nginx_group_id, new_attachment_socket)) {
i_socket->closeSocket(new_attachment_socket);
new_attachment_socket = -1;
@@ -1753,6 +1899,48 @@ private:
nginx_attachment_event.notify();
nginx_attachment_event.resetAllCounters();
dbgWarning(D_NGINX_ATTACHMENT) << "Failed to register attachment";
} else {
// Set affinity to core based on received target core from NGINX
if (attachment_config.getIsPairedAffinityEnabled()
&& target_core_available && target_core_value >= 0) {
auto uid_integer = getUidInteger(uid.unpack());
if (!uid_integer.ok()) {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Failed to convert UID to integer. Error: " << uid_integer.getErr();
} else {
dbgDebug(D_NGINX_ATTACHMENT)
<< "Setting CPU affinity for NGINX attachment using target core from NGINX"
<< " UID: " << uid_integer.unpack()
<< ", unique_id: " << uid.unpack()
<< ", received_target_core: " << target_core_value;
// Use common affinity function
int result = set_affinity_to_core(target_core_value);
if (result == 0) {
dbgDebug(D_NGINX_ATTACHMENT)
<< "Successfully set CPU affinity for NGINX attachment to core "
<< target_core_value;
} else {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Failed to set CPU affinity for NGINX attachment to core "
<< target_core_value
<< ". Error code: "
<< result;
}
}
} else if (!target_core_available) {
dbgTrace(D_NGINX_ATTACHMENT)
<< "Target core not available from NGINX"
" (backward compatibility mode), skipping CPU affinity setting";
} else if (target_core_value < 0) {
dbgTrace(D_NGINX_ATTACHMENT)
<< "Paired affinity disabled by NGINX (target_core=" << target_core_value
<< "), skipping CPU affinity setting for NGINX attachment";
} else {
dbgTrace(D_NGINX_ATTACHMENT)
<< "Paired affinity is not enabled in service config, "
"skipping setting CPU affinity for NGINX attachment";
}
}
};
mainloop->addFileRoutine(
@@ -1811,6 +1999,35 @@ private:
return uid;
}
Maybe<uint32_t>
getUidInteger(string uid)
{
if (uid.empty()) {
return genError("UID is empty");
}
uint32_t uid_integer = 0;
try {
// Handle both container format "{container_id}_{worker_id}" and simple format "{worker_id}"
size_t underscore_pos = uid.find_last_of('_');
string worker_id_str;
if (underscore_pos != string::npos) {
// Container format: extract worker_id after the last underscore
worker_id_str = uid.substr(underscore_pos + 1);
} else {
// Simple format: entire string is the worker_id
worker_id_str = uid;
}
uid_integer = stoul(worker_id_str);
} catch (const std::exception &e) {
return genError(string("Failed to convert UID to integer: ") + e.what());
}
return uid_integer;
}
Maybe<uint32_t>
readIdFromSocket(I_Socket::socketFd new_attachment_socket)
{
@@ -1833,6 +2050,36 @@ private:
return attachment_id;
}
Maybe<int32_t>
readSignedIdFromSocket(I_Socket::socketFd new_attachment_socket)
{
bool did_fail_on_purpose = false;
DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket);
// Try to read target core - if this fails, it's likely backward compatibility
if (!i_socket->isDataAvailable(new_attachment_socket)) {
dbgTrace(D_NGINX_ATTACHMENT)
<< "No target core data available on socket - likely backward compatibility mode";
return genError("No target core data available on socket");
}
Maybe<vector<char>> id = i_socket->receiveData(new_attachment_socket, sizeof(int32_t));
if (SHOULD_FAIL(
id.ok(),
IntentionalFailureHandler::FailureType::ReceiveDataFromSocket,
&did_fail_on_purpose
)) {
return genError(
string("Failed to read the signed attachment ID (likely backward compatibility issue): ") +
(did_fail_on_purpose ? "[Intentional Failure]" : id.getErr())
);
}
int32_t attachment_id = *reinterpret_cast<const int32_t *>(id.unpack().data());
dbgTrace(D_NGINX_ATTACHMENT) << "Signed Attachment ID: " << static_cast<int>(attachment_id);
return attachment_id;
}
string static_resources_path;
FilterVerdict default_verdict;
FailopenModeListener fail_open_mode_listener;
@@ -1852,6 +2099,7 @@ private:
SharedMemoryIPC *attachment_ipc = nullptr;
HttpAttachmentConfig attachment_config;
I_MainLoop::RoutineID attachment_routine_id = 0;
I_MainLoop::RoutineID async_attachment_routine_id = 0;
bool traffic_indicator = false;
unordered_set<string> ignored_headers;
@@ -1882,7 +2130,267 @@ private:
nginxIntakerMetric nginx_intaker_metric;
TransactionTableEvent transaction_table_event;
TransactionTableMetric transaction_table_metric;
///
/// @brief Async version of handleInspection - simplified for fastest response time
/// @return true on success, false on error
///
// LCOV_EXCL_START Reason: Temporary INXT-49318
bool
handleInspectionAsync()
{
Maybe<vector<char>> comm_trigger = genError("comm trigger uninitialized");
static map<I_Socket::socketFd, bool> comm_status;
if (comm_status.find(attachment_sock) == comm_status.end()) {
comm_status[attachment_sock] = true;
}
DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket);
// Read session ID from socket (used as doorbell only)
uint32_t signaled_session_id = 0;
for (int retry = 0; retry < 3; retry++) {
comm_trigger = i_socket->receiveData(attachment_sock, sizeof(signaled_session_id));
if (comm_trigger.ok()) break;
}
bool did_fail_on_purpose = false;
if (SHOULD_FAIL(
comm_trigger.ok(),
IntentionalFailureHandler::FailureType::ReceiveDataFromSocket,
&did_fail_on_purpose
)) {
if (comm_status[attachment_sock] == true) {
dbgDebug(D_NGINX_ATTACHMENT)
<< "Failed to get signal from attachment socket (async mode)"
<< ", Socket: " << attachment_sock
<< ", Error: " << (did_fail_on_purpose ? "Intentional Failure" : comm_trigger.getErr());
comm_status[attachment_sock] = false;
}
return false;
}
signaled_session_id = *reinterpret_cast<const uint32_t *>(comm_trigger.unpack().data());
comm_status.erase(attachment_sock);
traffic_indicator = true;
while (isDataAvailable(attachment_ipc)) {
traffic_indicator = true;
uint32_t handled_session_id = handleRequestFromQueueAsync(attachment_ipc);
if (handled_session_id == 0 || handled_session_id == corrupted_session_id) {
continue;
}
// Always signal back to nginx - never leave it waiting
dbgTrace(D_NGINX_ATTACHMENT) << "Signaling attachment to read verdict (async mode)";
bool res = false;
vector<char> session_id_data(
reinterpret_cast<char *>(&handled_session_id),
reinterpret_cast<char *>(&handled_session_id) + sizeof(handled_session_id)
);
DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::WriteDataToSocket);
if (!SHOULD_FAIL(
true,
IntentionalFailureHandler::FailureType::WriteDataToSocket,
&did_fail_on_purpose
)) {
for (int retry = 0; retry < 3; retry++) {
if (i_socket->writeDataAsync(attachment_sock, session_id_data)) {
dbgTrace(D_NGINX_ATTACHMENT)
<< "Successfully sent signal to attachment (async mode).";
res = true;
break;
}
dbgDebug(D_NGINX_ATTACHMENT)
<< "Failed to send ACK to attachment (async mode, try " << retry << ")";
mainloop->yield(true);
}
}
if (!res) {
dbgWarning(D_NGINX_ATTACHMENT) << "Failed to send ACK to attachment (async mode)"
<< (did_fail_on_purpose ? "[Intentional Failure]" : "");
if (!did_fail_on_purpose) {
dbgWarning(D_NGINX_ATTACHMENT) << "Resetting IPC and socket";
resetIpc(attachment_ipc, num_of_nginx_ipc_elements);
}
return false;
}
}
return true;
}
// LCOV_EXCL_STOP
///
/// @brief Async version of handleRequestFromQueue - always signals back to nginx
/// @param[in] attachment_ipc IPC channel
/// @param[in] signaled_session_id Session ID that triggered the signal (not used, doorbell only)
/// @return session_id of processed request
///
// LCOV_EXCL_START Reason: Temporary INXT-49318
uint32_t
handleRequestFromQueueAsync(SharedMemoryIPC *attachment_ipc)
{
Maybe<pair<uint16_t, const char *>> read_data = readData(attachment_ipc);
if (!read_data.ok()) {
dbgWarning(D_NGINX_ATTACHMENT) << "Failed to read data. Error: " << read_data.getErr();
return corrupted_session_id;
}
uint16_t incoming_data_size = read_data.unpack().first;
const char *incoming_data = read_data.unpack().second;
if (incoming_data_size == 0 || incoming_data == nullptr) {
dbgWarning(D_NGINX_ATTACHMENT) << "No data received from NGINX attachment";
return corrupted_session_id;
}
const NanoHttpRequestData *transaction_data =
reinterpret_cast<const NanoHttpRequestData *>(incoming_data);
Maybe<ChunkType> chunked_data_type = convertToEnum<ChunkType>(transaction_data->data_type);
if (!chunked_data_type.ok()) {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Could not convert "
<< static_cast<int>(transaction_data->data_type)
<< " to ChunkType enum. Resetting IPC"
<< dumpIpcWrapper(attachment_ipc);
popData(attachment_ipc);
resetIpc(attachment_ipc, num_of_nginx_ipc_elements);
nginx_attachment_event.addNetworkingCounter(nginxAttachmentEvent::networkVerdict::CONNECTION_FAIL);
return corrupted_session_id;
}
if (chunked_data_type.unpack() == ChunkType::METRIC_DATA_FROM_PLUGIN) {
const NanoHttpMetricData *recieved_metric_data =
reinterpret_cast<const NanoHttpMetricData *>(incoming_data);
sendMetricToKibana(recieved_metric_data);
popData(attachment_ipc);
return 0; // No signaling needed for metrics
}
dbgTrace(D_NGINX_ATTACHMENT)
<< "Reading "
<< incoming_data_size
<<" bytes "
<< convertChunkTypeToString(*chunked_data_type)
<< "(type = "
<< static_cast<int>(*chunked_data_type)
<< ") of data from NGINX attachment for session ID: "
<< transaction_data->session_id;
const uint32_t cur_session_id = transaction_data->session_id;
if (
isFailOpenTriggered()
&& (
chunked_data_type.unpack() == ChunkType::REQUEST_START
|| chunked_data_type.unpack() == ChunkType::REQUEST_HEADER
)
) {
dbgTrace(D_NGINX_ATTACHMENT)
<< "Agent is set to Fail Open Mode. Passing inspection and returning Accept."
<< " Session ID: "
<< cur_session_id
<< ", Chunked data type: "
<< static_cast<int>(*chunked_data_type);
if (i_transaction_table->hasEntry(cur_session_id)) {
i_transaction_table->deleteEntry(cur_session_id);
}
popData(attachment_ipc);
handleVerdictResponse(
FilterVerdict(ACCEPT),
attachment_ipc,
cur_session_id,
false
);
return cur_session_id;
}
if (!setActiveTransactionEntry(cur_session_id, chunked_data_type.unpack())) {
popData(attachment_ipc);
return cur_session_id;
}
const Buffer inspection_data(
transaction_data->data,
incoming_data_size - sizeof(NanoHttpRequestData),
Buffer::MemoryType::VOLATILE
);
if (*chunked_data_type == ChunkType::REQUEST_START && !createTransactionState(inspection_data)) {
dbgWarning(D_NGINX_ATTACHMENT)
<< "Failed to handle new request. Returning default verdict: "
<< verdictToString(default_verdict.getVerdict());
handleVerdictResponse(
default_verdict,
attachment_ipc,
cur_session_id,
false
);
popData(attachment_ipc);
removeTransactionEntry(cur_session_id);
return cur_session_id;
}
if (i_transaction_table != nullptr) {
transaction_table_event.setTransactionTableSize(i_transaction_table->count());
transaction_table_event.notify();
}
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
opaque.activateContext();
FilterVerdict verdict = handleChunkedData(*chunked_data_type, inspection_data, opaque);
bool is_header =
*chunked_data_type == ChunkType::REQUEST_HEADER ||
*chunked_data_type == ChunkType::RESPONSE_HEADER ||
*chunked_data_type == ChunkType::CONTENT_LENGTH;
if (verdict.getVerdict() == LIMIT_RESPONSE_HEADERS) {
handleVerdictResponse(verdict, attachment_ipc, transaction_data->session_id, is_header);
popData(attachment_ipc);
verdict = FilterVerdict(INSPECT);
}
handleVerdictResponse(verdict, attachment_ipc, cur_session_id, is_header);
bool is_final_verdict = verdict.getVerdict() == ACCEPT ||
verdict.getVerdict() == DROP ||
verdict.getVerdict() == CUSTOM_RESPONSE ||
verdict.getVerdict() == IRRELEVANT;
dbgTrace(D_NGINX_ATTACHMENT)
<< "Request handled successfully - for"
<< " NGINX attachment session ID: "
<< transaction_data->session_id
<< " verdict: "
<< verdictToString(verdict.getVerdict())
<< " verdict_data_code="
<< static_cast<int>(verdict.getVerdict());
popData(attachment_ipc);
opaque.deactivateContext();
if (is_final_verdict) {
removeTransactionEntry(cur_session_id);
} else {
i_transaction_table->unsetActiveKey();
}
// Always signal back in async mode - never leave nginx waiting
return cur_session_id;
}
};
// LCOV_EXCL_STOP
NginxAttachment::NginxAttachment() : Component("NginxAttachment"), pimpl(make_unique<Impl>()) {}
@@ -1921,7 +2429,7 @@ NginxAttachment::preload()
registerExpectedConfiguration<uint>("Nginx Attachment", "metric reporting interval");
registerExpectedSetting<bool>("allowOnlyDefinedApplications");
registerExpectedConfigFile("activeContextConfig", Config::ConfigFileType::Policy);
registerExpectedConfiguration<UsersAllIdentifiersConfig>("rulebase", "usersIdentifiers");
registerExpectedConfigurationWithCache<UsersAllIdentifiersConfig>("assetId", "rulebase", "usersIdentifiers");
BasicRuleConfig::preload();
WebTriggerConf::preload();
}

74
components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc
View File

@@ -14,6 +14,7 @@
#include "nginx_attachment_config.h"
#include <stdlib.h>
#include <algorithm>
#include "nginx_attachment.h"
#include "config.h"
@@ -25,7 +26,7 @@ USE_DEBUG_FLAG(D_NGINX_ATTACHMENT);
using namespace std;
using DebugLevel = ngx_http_cp_debug_level_e;
using DebugLevel = nano_http_cp_debug_level_e;
void
HttpAttachmentConfig::init()
@@ -43,6 +44,8 @@ HttpAttachmentConfig::init()
setDebugByContextValues();
setKeepAliveIntervalMsec();
setRetriesForVerdict();
setPairedAffinityEnabled();
setAsyncMode();
}
bool
@@ -210,15 +213,36 @@ HttpAttachmentConfig::setFailOpenTimeout()
"Response server header removal"
));
conf_data.setNumericalValue("decompression_pool_size", getAttachmentConf<uint>(
262144,
"agent.decompressionPoolSize.nginxModule",
"HTTP manager",
"Decompression pool size in bytes"
));
conf_data.setNumericalValue("recompression_pool_size", getAttachmentConf<uint>(
16384,
"agent.recompressionPoolSize.nginxModule",
"HTTP manager",
"Recompression pool size in bytes"
));
conf_data.setNumericalValue("is_brotli_inspection_enabled", getAttachmentConf<uint>(
0,
"agent.isBrotliInspectionEnabled.nginxModule",
"HTTP manager",
"Brotli inspection enabled"
));
uint inspection_mode = getAttachmentConf<uint>(
static_cast<uint>(ngx_http_inspection_mode_e::NON_BLOCKING_THREAD),
static_cast<uint>(NanoHttpInspectionMode::NON_BLOCKING_THREAD),
"agent.inspectionMode.nginxModule",
"HTTP manager",
"NGINX inspection mode"
);
if (inspection_mode >= ngx_http_inspection_mode_e::INSPECTION_MODE_COUNT) {
inspection_mode = ngx_http_inspection_mode_e::NON_BLOCKING_THREAD;
if (inspection_mode >= static_cast<uint>(NanoHttpInspectionMode::INSPECTION_MODE_COUNT)) {
inspection_mode = static_cast<uint>(NanoHttpInspectionMode::NON_BLOCKING_THREAD);
}
conf_data.setNumericalValue("nginx_inspection_mode", inspection_mode);
}
@@ -377,3 +401,45 @@ HttpAttachmentConfig::setDebugByContextValues()
<< ", listening_port: "
<< new_ctx_cfg.port;
}
void
HttpAttachmentConfig::setPairedAffinityEnabled() {
bool is_paired_affinity_enabled = getAttachmentConf<bool>(
false,
"agent.pairedAffinity.nginxModule",
"HTTP manager",
"Paired Affinity state"
);
// Check environment variable to allow runtime override
const char* env_enable_affinity = getenv("ENABLE_AFFINITY_PAIRING");
if (env_enable_affinity != nullptr) {
string env_value(env_enable_affinity);
// Convert to lowercase for case insensitive comparison
transform(env_value.begin(), env_value.end(), env_value.begin(), ::tolower);
is_paired_affinity_enabled = (env_value == "true");
dbgTrace(D_NGINX_ATTACHMENT)
<< "Paired affinity overridden by environment variable ENABLE_AFFINITY_PAIRING="
<< env_enable_affinity;
}
dbgTrace(D_NGINX_ATTACHMENT)
<< "Attachment paired affinity is: "
<< (is_paired_affinity_enabled ? "Enabled" : "Disabled");
conf_data.setNumericalValue("is_paired_affinity_enabled", is_paired_affinity_enabled);
}
void
HttpAttachmentConfig::setAsyncMode() {
bool is_async_mode_enabled = getAttachmentConf<bool>(
false,
"agent.asyncMode.nginxModule",
"HTTP manager",
"Async Mode state"
);
dbgTrace(D_NGINX_ATTACHMENT)
<< "Attachment async mode is: "
<< (is_async_mode_enabled ? "Enabled" : "Disabled");
conf_data.setNumericalValue("is_async_mode_enabled", is_async_mode_enabled);
}

8
components/attachment-intakers/nginx_attachment/nginx_attachment_config.h
View File

@@ -44,6 +44,10 @@ public:
unsigned int getMaxSessionsPerMinute() const { return conf_data.getNumericalValue("max_sessions_per_minute"); }
unsigned int getNumOfNginxElements() const { return conf_data.getNumericalValue("num_of_nginx_ipc_elements"); }
unsigned int getKeepAliveIntervalMsec() const { return conf_data.getNumericalValue("keep_alive_interval_msec"); }
bool getIsPairedAffinityEnabled() const { return conf_data.getNumericalValue("is_paired_affinity_enabled"); }
// LCOV_EXCL_START Reason: Temporary INXT-49318
bool isAsyncModeEnabled() const { return conf_data.getNumericalValue("is_async_mode_enabled"); }
// LCOV_EXCL_STOP
private:
void setGradualDeploymentIPs();
@@ -72,6 +76,10 @@ private:
void setRetriesForVerdict();
void setPairedAffinityEnabled();
void setAsyncMode();
WebTriggerConf web_trigger_conf;
HttpAttachmentConfiguration conf_data;
};

10
components/attachment-intakers/nginx_attachment/nginx_attachment_opaque.cc
View File

@@ -51,18 +51,19 @@ NginxAttachmentOpaque::NginxAttachmentOpaque(HttpTransactionData _transaction_da
std::stringstream client_ip_str;
client_ip_str << client_ip;
setSourceIdentifier("sourceip", client_ip_str.str());
ctx.registerValue("eventReferenceId", uuid, EnvKeyAttr::LogSection::DATA);
ctx.registerValue<string>(HttpTransactionData::http_proto_ctx, transaction_data.getHttpProtocol());
ctx.registerValue<string>(HttpTransactionData::method_ctx, transaction_data.getHttpMethod());
ctx.registerValue<string>(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost());
ctx.registerValue<uint16_t>(HttpTransactionData::listening_port_ctx, transaction_data.getListeningPort());
ctx.registerQuickAccessValue<string>(HttpTransactionData::host_name_ctx, transaction_data.getParsedHost());
ctx.registerQuickAccessValue<uint16_t>(
HttpTransactionData::listening_port_ctx,
transaction_data.getListeningPort());
ctx.registerValue<IPAddr>(HttpTransactionData::listening_ip_ctx, transaction_data.getListeningIP());
ctx.registerValue<IPAddr>(HttpTransactionData::client_ip_ctx, transaction_data.getSourceIP());
ctx.registerValue<uint16_t>(HttpTransactionData::client_port_ctx, transaction_data.getSourcePort());
ctx.registerFunc<string>(HttpTransactionData::source_identifier, [this](){ return source_identifier; });
ctx.registerValue<string>(HttpTransactionData::uri_ctx, transaction_data.getParsedURI());
ctx.registerQuickAccessValue<string>(HttpTransactionData::uri_ctx, transaction_data.getParsedURI());
auto decoder = makeVirtualContainer<HexDecoder<'%'>>(transaction_data.getURI());
string decoded_url(decoder.begin(), decoder.end());
auto question_mark_location = decoded_url.find('?');
@@ -143,6 +144,7 @@ NginxAttachmentOpaque::setKeepAliveCtx(const string &hdr_key, const string &hdr_
ctx.registerValue("keep_alive_request_ctx", true);
return true;
}
dbgTrace(D_HTTP_MANAGER) << "Not a keep alive header";
return false;
}

298
components/attachment-intakers/nginx_attachment/nginx_intaker_metric.cc
View File

@@ -54,204 +54,204 @@ nginxIntakerEvent::resetAllCounters()
cpu_event.setCPU(0);
}
ngx_http_plugin_metric_type_e
AttachmentMetricType
nginxIntakerEvent::EnumOfIndex(int i)
{
return static_cast<ngx_http_plugin_metric_type_e>(i);
return static_cast<AttachmentMetricType>(i);
}
void
nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recieved_metric_data)
nginxIntakerEvent::addPluginMetricCounter(const NanoHttpMetricData *recieved_metric_data)
{
for (int i = 0; i < static_cast<int>(ngx_http_plugin_metric_type_e::METRIC_TYPES_COUNT); i++) {
ngx_http_plugin_metric_type_e metric_type = EnumOfIndex(i);
for (int i = 0; i < static_cast<int>(AttachmentMetricType::METRIC_TYPES_COUNT); i++) {
AttachmentMetricType metric_type = EnumOfIndex(i);
uint64_t amount = recieved_metric_data->data[i];
switch (metric_type) {
case ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT: {
case AttachmentMetricType::INSPECTION_SUCCESSES_COUNT: {
successfull_inspection_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT: {
case AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT: {
open_failure_inspection_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT: {
case AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT: {
close_failure_inspection_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT: {
case AttachmentMetricType::TRANSPARENTS_COUNT: {
transparent_mode_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME: {
case AttachmentMetricType::TOTAL_TRANSPARENTS_TIME: {
total_transparent_time += amount;
break;
}
case ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT: {
case AttachmentMetricType::INSPECT_VERDICTS_COUNT: {
inspect_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT: {
case AttachmentMetricType::ACCEPT_VERDICTS_COUNT: {
accept_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT: {
case AttachmentMetricType::DROP_VERDICTS_COUNT: {
drop_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT: {
case AttachmentMetricType::INJECT_VERDICTS_COUNT: {
inject_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT: {
case AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT: {
irrelevant_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT: {
case AttachmentMetricType::RECONF_VERDICTS_COUNT: {
reconf_verdict_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) average_overall_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) max_overall_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) min_overall_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) average_req_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) max_req_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) min_req_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) average_res_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) max_res_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
case AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT: {
if (amount > 0) min_res_processing_time_until_verdict = amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT: {
case AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT: {
req_failed_compression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT: {
case AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT: {
res_failed_compression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT: {
case AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT: {
req_failed_decompression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT: {
case AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT: {
res_failed_decompression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT: {
case AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT: {
req_successful_compression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT: {
case AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT: {
res_successful_compression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: {
case AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT: {
req_successful_decompression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT: {
case AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT: {
res_successful_decompression_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: {
case AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT: {
corrupted_zip_skipped_session_counter += amount;
break;
}
case ngx_http_plugin_metric_type_e::THREAD_TIMEOUT: {
case AttachmentMetricType::THREAD_TIMEOUT: {
thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT: {
case AttachmentMetricType::REG_THREAD_TIMEOUT: {
reg_thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT: {
case AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT: {
req_header_thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT: {
case AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT: {
req_body_thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) average_req_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) max_req_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) min_req_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT: {
case AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT: {
res_header_thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT: {
case AttachmentMetricType::RES_BODY_THREAD_TIMEOUT: {
res_body_thread_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) average_res_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) max_res_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT: {
case AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT: {
if (amount > 0) min_res_body_size_upon_timeout = amount;
break;
}
case ngx_http_plugin_metric_type_e::THREAD_FAILURE: {
case AttachmentMetricType::THREAD_FAILURE: {
thread_failure += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT: {
case AttachmentMetricType::REQ_PROCCESSING_TIMEOUT: {
req_proccessing_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT: {
case AttachmentMetricType::RES_PROCCESSING_TIMEOUT: {
res_proccessing_timeout += amount;
break;
}
case ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM: {
case AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM: {
req_failed_to_reach_upstream += amount;
break;
}
case ngx_http_plugin_metric_type_e::CPU_USAGE: {
case AttachmentMetricType::CPU_USAGE: {
cpu_event.setCPU(amount);
break;
}
case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT: {
case AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT: {
req_overall_size += amount;
static const uint64_t max_expected_res_size = 100ULL * 1024 * 1024 * 1024;
if (amount > max_expected_res_size) {
@@ -259,7 +259,7 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie
}
break;
}
case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT: {
case AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT: {
res_overall_size += amount;
break;
}
@@ -272,104 +272,104 @@ nginxIntakerEvent::addPluginMetricCounter(const ngx_http_cp_metric_data_t *recie
}
uint64_t
nginxIntakerEvent::getPluginMetricCounter(ngx_http_plugin_metric_type_e metric_type) const
nginxIntakerEvent::getPluginMetricCounter(AttachmentMetricType metric_type) const
{
switch (metric_type) {
case ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT:
case AttachmentMetricType::INSPECTION_SUCCESSES_COUNT:
return successfull_inspection_counter;
case ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT:
case AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT:
return open_failure_inspection_counter;
case ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT:
case AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT:
return close_failure_inspection_counter;
case ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT:
case AttachmentMetricType::TRANSPARENTS_COUNT:
return transparent_mode_counter;
case ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME:
case AttachmentMetricType::TOTAL_TRANSPARENTS_TIME:
return total_transparent_time;
case ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT:
case AttachmentMetricType::INSPECT_VERDICTS_COUNT:
return inspect_verdict_counter;
case ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT:
case AttachmentMetricType::ACCEPT_VERDICTS_COUNT:
return accept_verdict_counter;
case ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT:
case AttachmentMetricType::DROP_VERDICTS_COUNT:
return drop_verdict_counter;
case ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT:
case AttachmentMetricType::INJECT_VERDICTS_COUNT:
return inject_verdict_counter;
case ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT:
case AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT:
return irrelevant_verdict_counter;
case ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT:
case AttachmentMetricType::RECONF_VERDICTS_COUNT:
return reconf_verdict_counter;
case ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
return average_overall_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
return max_overall_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT:
return min_overall_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
return average_req_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
return max_req_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT:
return min_req_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT:
return average_res_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT:
return max_res_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT:
case AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT:
return min_res_processing_time_until_verdict;
case ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT:
case AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT:
return req_failed_compression_counter;
case ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT:
case AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT:
return res_failed_compression_counter;
case ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT:
case AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT:
return req_failed_decompression_counter;
case ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT:
case AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT:
return res_failed_decompression_counter;
case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT:
case AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT:
return req_successful_compression_counter;
case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT:
case AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT:
return res_successful_compression_counter;
case ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT:
case AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT:
return req_successful_decompression_counter;
case ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT:
case AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT:
return res_successful_decompression_counter;
case ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT:
case AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT:
return corrupted_zip_skipped_session_counter;
case ngx_http_plugin_metric_type_e::THREAD_TIMEOUT:
case AttachmentMetricType::THREAD_TIMEOUT:
return thread_timeout;
case ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT:
case AttachmentMetricType::REG_THREAD_TIMEOUT:
return reg_thread_timeout;
case ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT:
case AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT:
return req_header_thread_timeout;
case ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT:
case AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT:
return req_body_thread_timeout;
case ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT:
return average_req_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT:
return max_req_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT:
return min_req_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT:
case AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT:
return res_header_thread_timeout;
case ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT:
case AttachmentMetricType::RES_BODY_THREAD_TIMEOUT:
return res_body_thread_timeout;
case ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT:
return average_res_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT:
return max_res_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT:
case AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT:
return min_res_body_size_upon_timeout;
case ngx_http_plugin_metric_type_e::THREAD_FAILURE:
case AttachmentMetricType::THREAD_FAILURE:
return thread_failure;
case ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT:
case AttachmentMetricType::REQ_PROCCESSING_TIMEOUT:
return req_proccessing_timeout;
case ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT:
case AttachmentMetricType::RES_PROCCESSING_TIMEOUT:
return res_proccessing_timeout;
case ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM:
case AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM:
return req_failed_to_reach_upstream;
case ngx_http_plugin_metric_type_e::CPU_USAGE:
case AttachmentMetricType::CPU_USAGE:
return static_cast<uint64_t>(cpu_event.getCPU());
case ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT:
case AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT:
return req_overall_size;
case ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT:
case AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT:
return res_overall_size;
default:
dbgWarning(D_METRICS_NGINX_ATTACHMENT)
@@ -382,145 +382,145 @@ void
nginxIntakerMetric::upon(const nginxIntakerEvent &event)
{
successfull_inspection_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_SUCCESSES_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_SUCCESSES_COUNT)
);
transparent_mode_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::TRANSPARENTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::TRANSPARENTS_COUNT)
);
total_transparent_time.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::TOTAL_TRANSPARENTS_TIME)
event.getPluginMetricCounter(AttachmentMetricType::TOTAL_TRANSPARENTS_TIME)
);
open_failure_inspection_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_OPEN_FAILURES_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_OPEN_FAILURES_COUNT)
);
close_failure_inspection_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECTION_CLOSE_FAILURES_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::INSPECTION_CLOSE_FAILURES_COUNT)
);
inject_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INJECT_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::INJECT_VERDICTS_COUNT)
);
inspect_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::INSPECT_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::INSPECT_VERDICTS_COUNT)
);
accept_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::ACCEPT_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::ACCEPT_VERDICTS_COUNT)
);
drop_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::DROP_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::DROP_VERDICTS_COUNT)
);
irrelevant_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::IRRELEVANT_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::IRRELEVANT_VERDICTS_COUNT)
);
reconf_verdict_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RECONF_VERDICTS_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RECONF_VERDICTS_COUNT)
);
average_overall_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
);
max_overall_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MAX_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
);
min_overall_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MIN_OVERALL_PPROCESSING_TIME_UNTIL_VERDICT)
);
average_req_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
);
max_req_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MAX_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
);
min_req_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MIN_REQ_PPROCESSING_TIME_UNTIL_VERDICT)
);
average_res_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_RES_PPROCESSING_TIME_UNTIL_VERDICT)
);
max_res_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MAX_RES_PPROCESSING_TIME_UNTIL_VERDICT)
);
min_res_processing_time_until_verdict.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT)
event.getPluginMetricCounter(AttachmentMetricType::MIN_RES_PPROCESSING_TIME_UNTIL_VERDICT)
);
req_failed_compression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_COMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_COMPRESSION_COUNT)
);
res_failed_compression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_FAILED_COMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RES_FAILED_COMPRESSION_COUNT)
);
req_failed_decompression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_DECOMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_DECOMPRESSION_COUNT)
);
res_failed_decompression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_FAILED_DECOMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RES_FAILED_DECOMPRESSION_COUNT)
);
req_successful_compression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_COMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_SUCCESSFUL_COMPRESSION_COUNT)
);
res_successful_compression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_COMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RES_SUCCESSFUL_COMPRESSION_COUNT)
);
req_successful_decompression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_SUCCESSFUL_DECOMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_SUCCESSFUL_DECOMPRESSION_COUNT)
);
res_successful_decompression_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_SUCCESSFUL_DECOMPRESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RES_SUCCESSFUL_DECOMPRESSION_COUNT)
);
corrupted_zip_skipped_session_counter.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::CORRUPTED_ZIP_SKIPPED_SESSION_COUNT)
);
thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::THREAD_TIMEOUT)
);
reg_thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REG_THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::REG_THREAD_TIMEOUT)
);
req_header_thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_HEADER_THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_HEADER_THREAD_TIMEOUT)
);
req_body_thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_BODY_THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_BODY_THREAD_TIMEOUT)
);
average_req_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_REQ_BODY_SIZE_UPON_TIMEOUT)
);
max_req_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_REQ_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::MAX_REQ_BODY_SIZE_UPON_TIMEOUT)
);
min_req_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_REQ_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::MIN_REQ_BODY_SIZE_UPON_TIMEOUT)
);
res_header_thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_HEADER_THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::RES_HEADER_THREAD_TIMEOUT)
);
res_body_thread_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_BODY_THREAD_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::RES_BODY_THREAD_TIMEOUT)
);
average_res_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::AVERAGE_RES_BODY_SIZE_UPON_TIMEOUT)
);
max_res_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MAX_RES_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::MAX_RES_BODY_SIZE_UPON_TIMEOUT)
);
min_res_body_size_upon_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::MIN_RES_BODY_SIZE_UPON_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::MIN_RES_BODY_SIZE_UPON_TIMEOUT)
);
thread_failure.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::THREAD_FAILURE)
event.getPluginMetricCounter(AttachmentMetricType::THREAD_FAILURE)
);
req_proccessing_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_PROCCESSING_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::REQ_PROCCESSING_TIMEOUT)
);
res_proccessing_timeout.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RES_PROCCESSING_TIMEOUT)
event.getPluginMetricCounter(AttachmentMetricType::RES_PROCCESSING_TIMEOUT)
);
req_failed_to_reach_upstream.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQ_FAILED_TO_REACH_UPSTREAM)
event.getPluginMetricCounter(AttachmentMetricType::REQ_FAILED_TO_REACH_UPSTREAM)
);
req_overall_size.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::REQUEST_OVERALL_SIZE_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::REQUEST_OVERALL_SIZE_COUNT)
);
res_overall_size.report(
event.getPluginMetricCounter(ngx_http_plugin_metric_type_e::RESPONSE_OVERALL_SIZE_COUNT)
event.getPluginMetricCounter(AttachmentMetricType::RESPONSE_OVERALL_SIZE_COUNT)
);
event.notifyCPU();
}

115
components/attachment-intakers/nginx_attachment/nginx_parser.cc
View File

@@ -34,7 +34,8 @@ bool is_keep_alive_ctx = getenv("SAAS_KEEP_ALIVE_HDR_NAME") != nullptr;
map<Buffer, CompressionType> NginxParser::content_encodings = {
{Buffer("identity"), CompressionType::NO_COMPRESSION},
{Buffer("gzip"), CompressionType::GZIP},
{Buffer("deflate"), CompressionType::ZLIB}
{Buffer("deflate"), CompressionType::ZLIB},
{Buffer("br"), CompressionType::BROTLI}
};
Maybe<HttpTransactionData>
@@ -180,6 +181,7 @@ getActivetenantAndProfile(const string &str, const string &deli = ",")
Maybe<vector<HttpHeader>>
NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set<string> &ignored_headers)
{
dbgFlow(D_NGINX_ATTACHMENT_PARSER) << "Parsing request headers";
auto maybe_parsed_headers = genHeaders(data);
if (!maybe_parsed_headers.ok()) return maybe_parsed_headers.passErr();
@@ -188,44 +190,72 @@ NginxParser::parseRequestHeaders(const Buffer &data, const unordered_set<string>
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
if (is_keep_alive_ctx || !ignored_headers.empty()) {
bool is_last_header_removed = false;
parsed_headers.erase(
remove_if(
parsed_headers.begin(),
parsed_headers.end(),
[&opaque, &is_last_header_removed, &ignored_headers](const HttpHeader &header)
{
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "is_keep_alive_ctx: " << is_keep_alive_ctx
<< ", ignored_headers size: " << ignored_headers.size()
<< ", headers count before removal: " << parsed_headers.size();
// First, determine which headers would be removed
std::vector<size_t> remove_indices;
for (size_t i = 0; i < parsed_headers.size(); ++i) {
const auto &header = parsed_headers[i];
string hdr_key = static_cast<string>(header.getKey());
string hdr_val = static_cast<string>(header.getValue());
if (
opaque.setKeepAliveCtx(hdr_key, hdr_val)
|| ignored_headers.find(hdr_key) != ignored_headers.end()
) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Header was removed from headers list: " << hdr_key;
if (header.isLastHeader()) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Last header was removed from headers list";
bool should_remove =
opaque.setKeepAliveCtx(hdr_key, hdr_val) ||
ignored_headers.find(hdr_key) != ignored_headers.end();
if (should_remove) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "Header should be removed from headers list: "
<< dumpHex(header.getKey());
remove_indices.push_back(i);
}
}
if (remove_indices.size() == parsed_headers.size() && !parsed_headers.empty()) {
// All headers would be removed: keep the last, mark as shouldNotLog
parsed_headers.back().setShouldNotLog();
// Remove all except the last
parsed_headers.erase(parsed_headers.begin(), parsed_headers.end() - 1);
// Ensure the last header is marked as last
parsed_headers.back().setIsLastHeader();
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "All headers were marked for removal. Keeping last header and marking it as shouldNotLog: "
<< dumpHex(parsed_headers.back().getKey());
} else {
// Remove the marked headers as usual
// (remove from end to start to keep indices valid)
bool is_last_header_removed = false;
for (auto it = remove_indices.rbegin(); it != remove_indices.rend(); ++it) {
if (parsed_headers[*it].isLastHeader()) {
is_last_header_removed = true;
}
return true;
parsed_headers.erase(parsed_headers.begin() + *it);
}
return false;
}
),
parsed_headers.end()
);
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "Headers removal completed. Remaining headers count: "
<< parsed_headers.size();
if (is_last_header_removed) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Adjusting last header flag";
if (!parsed_headers.empty()) parsed_headers.back().setIsLastHeader();
if (!parsed_headers.empty()) {
parsed_headers.back().setIsLastHeader();
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "New last header after removal: "
<< dumpHex(parsed_headers.back().getKey());
}
}
}
}
static auto default_identifiers = UsersAllIdentifiersConfig();
auto maybe_source_identifiers =
getConfigurationWithCache<UsersAllIdentifiersConfig>("rulebase", "usersIdentifiers");
auto source_identifiers = maybe_source_identifiers.ok() ? maybe_source_identifiers.unpack() : default_identifiers;
for (const HttpHeader &header : parsed_headers) {
auto source_identifiers = getConfigurationWithDefault<UsersAllIdentifiersConfig>(
UsersAllIdentifiersConfig(),
"rulebase",
"usersIdentifiers"
);
source_identifiers.parseRequestHeaders(header);
if (!header.shouldLog()) {
continue;
}
opaque.addToSavedData(
HttpTransactionData::req_headers,
static_cast<string>(header.getKey()) + ": " + static_cast<string>(header.getValue()) + "\r\n"
@@ -419,11 +449,38 @@ NginxParser::convertToContentEncoding(const Buffer &content_encoding_header_valu
return genError("Multiple content encodings for a specific HTTP request/response body are not supported");
}
if (content_encodings.find(content_encoding_header_value) == content_encodings.end()) {
auto it = find_if(
content_encodings.begin(),
content_encodings.end(),
[&content_encoding_header_value](const pair<Buffer, CompressionType> &encoding_pair) {
bool is_equal = content_encoding_header_value.isEqualLowerCase(encoding_pair.first);
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "Comparing '"
<< static_cast<string>(content_encoding_header_value)
<< "' with '"
<< static_cast<string>(encoding_pair.first)
<< ". Comparison result: "
<< (is_equal ? "MATCH" : "NO MATCH");
return is_equal;
}
);
if (it == content_encodings.end()) {
dbgDebug(D_NGINX_ATTACHMENT_PARSER)
<< "No matching compression type found for: '"
<< static_cast<string>(content_encoding_header_value)
<< "'";
return genError(
"Unsupported or undefined \"Content-Encoding\" value: " +
static_cast<string>(content_encoding_header_value)
);
}
return content_encodings[content_encoding_header_value];
dbgDebug(D_NGINX_ATTACHMENT_PARSER)
<< "Successfully matched '"
<< static_cast<string>(content_encoding_header_value)
<< "' to compression type";
return it->second;
}

2
components/attachment-intakers/nginx_attachment/nginx_parser.h
View File

@@ -17,7 +17,7 @@
#include <vector>
#include "compression_utils.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "http_transaction_common.h"
#include "http_inspection_events.h"
#include "i_encryptor.h"

91
components/http_manager/http_manager.cc
View File

@@ -36,14 +36,15 @@ static ostream &
operator<<(ostream &os, const EventVerdict &event)
{
switch (event.getVerdict()) {
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT: return os << "Inspect";
case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT: return os << "Accept";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP: return os << "Drop";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT: return os << "Inject";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT: return os << "Irrelevant";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_RECONF: return os << "Reconf";
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: return os << "Wait";
case ServiceVerdict::TRAFFIC_VERDICT_INSPECT: return os << "Inspect";
case ServiceVerdict::LIMIT_RESPONSE_HEADERS: return os << "Limit Response Headers";
case ServiceVerdict::TRAFFIC_VERDICT_ACCEPT: return os << "Accept";
case ServiceVerdict::TRAFFIC_VERDICT_DROP: return os << "Drop";
case ServiceVerdict::TRAFFIC_VERDICT_INJECT: return os << "Inject";
case ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT: return os << "Irrelevant";
case ServiceVerdict::TRAFFIC_VERDICT_RECONF: return os << "Reconf";
case ServiceVerdict::TRAFFIC_VERDICT_DELAYED: return os << "Wait";
case ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE: return os << "Force 200";
}
dbgAssert(false)
@@ -66,6 +67,13 @@ public:
i_transaction_table = Singleton::Consume<I_Table>::by<HttpManager>();
Singleton::Consume<I_Logging>::by<HttpManager>()->addGeneralModifier(compressAppSecLogs);
custom_header = getProfileAgentSettingWithDefault<string>("", "agent.customHeaderValueLogging");
registerConfigLoadCb(
[this]() {
custom_header = getProfileAgentSettingWithDefault<string>("", "agent.customHeaderValueLogging");
}
);
}
FilterVerdict
@@ -95,8 +103,6 @@ public:
HttpManagerOpaque &state = i_transaction_table->getState<HttpManagerOpaque>();
const auto &custom_header = getProfileAgentSettingWithDefault<string>("", "agent.customHeaderValueLogging");
if (event.getKey().isEqualLowerCase(custom_header)) {
string event_value = static_cast<string>(event.getValue());
dbgTrace(D_HTTP_MANAGER)
@@ -117,7 +123,7 @@ public:
HttpRequestHeaderEvent(event).performNamedQuery() :
HttpResponseHeaderEvent(event).performNamedQuery();
FilterVerdict verdict = handleEvent(event_responds);
if (verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) {
if (verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) {
applyInjectionModifications(verdict, event_responds, event.getHeaderIndex());
}
return verdict;
@@ -131,8 +137,8 @@ public:
return FilterVerdict(default_verdict);
}
ngx_http_cp_verdict_e body_size_limit_verdict = handleBodySizeLimit(is_request, event);
if (body_size_limit_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT) {
ServiceVerdict body_size_limit_verdict = handleBodySizeLimit(is_request, event);
if (body_size_limit_verdict != ServiceVerdict::TRAFFIC_VERDICT_INSPECT) {
return FilterVerdict(body_size_limit_verdict);
}
@@ -144,7 +150,7 @@ public:
ctx.registerValue("UserDefined", state.getUserDefinedValue().unpack(), EnvKeyAttr::LogSection::DATA);
}
FilterVerdict verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
FilterVerdict verdict(ServiceVerdict::TRAFFIC_VERDICT_INSPECT);
if (!is_request && event.getData().size() == 0 && !event.isLastChunk()) {
dbgDebug(D_HTTP_MANAGER) << "Skipping inspection of first empty chunk for respond body";
return verdict;
@@ -156,7 +162,7 @@ public:
HttpResponseBodyEvent(event, state.getPreviousDataCache()).performNamedQuery();
verdict = handleEvent(event_responds);
state.saveCurrentDataToCache(event.getData());
if (verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) {
if (verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) {
applyInjectionModifications(verdict, event_responds, event.getBodyChunkIndex());
}
return verdict;
@@ -252,13 +258,13 @@ public:
}
private:
ngx_http_cp_verdict_e
ServiceVerdict
handleBodySizeLimit(bool is_request_body_type, const HttpBody &event)
{
HttpManagerOpaque &state = i_transaction_table->getState<HttpManagerOpaque>();
state.updatePayloadSize(event.getData().size());
auto size_limit = getConfiguration<uint>(
auto size_limit = getConfigurationWithCache<uint>(
"HTTP manager",
is_request_body_type ? "Max Request Body Size" : "Max Response Body Size"
);
@@ -270,12 +276,12 @@ private:
);
if (!size_limit.ok() || state.getAggeregatedPayloadSize() < size_limit.unpack()) {
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
ngx_http_cp_verdict_e verdict = size_limit_verdict == "Drop" ?
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP :
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
ServiceVerdict verdict = size_limit_verdict == "Drop" ?
ServiceVerdict::TRAFFIC_VERDICT_DROP :
ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
dbgDebug(D_HTTP_MANAGER)
<< "Transaction body size is over the limit. Max body size: "
@@ -295,7 +301,7 @@ private:
ModifiedChunkIndex event_idx)
{
for (const pair<string, EventVerdict> &respond : event_responds) {
if (respond.second.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) {
if (respond.second.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_INJECT) {
dbgTrace(D_HTTP_MANAGER)
<< "Applying inject verdict modifications for security App: "
<< respond.first;
@@ -310,7 +316,7 @@ private:
HttpManagerOpaque &state = i_transaction_table->getState<HttpManagerOpaque>();
for (const pair<string, EventVerdict> &respond : event_responds) {
if (state.getApplicationsVerdict(respond.first) == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) {
if (state.getApplicationsVerdict(respond.first) == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT) {
dbgTrace(D_HTTP_MANAGER)
<< "Skipping event verdict for app that already accepted traffic. App: "
<< respond.first;
@@ -325,13 +331,33 @@ private:
state.setApplicationVerdict(respond.first, respond.second.getVerdict());
state.setApplicationWebResponse(respond.first, respond.second.getWebUserResponseByPractice());
if (respond.second.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE) {
if (!respond.second.getCustomResponse().ok()) {
dbgWarning(D_HTTP_MANAGER)
<< "Security app: "
<< respond.first
<< ", returned verdict CUSTOM_RESPONSE, but no custom response was found.";
continue;
}
FilterVerdict aggregated_verdict(state.getCurrVerdict(), state.getCurrWebUserResponse());
if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
state.setCustomResponse(respond.first, respond.second.getCustomResponse().unpack());
}
}
auto ver = state.getCurrVerdict();
dbgTrace(D_HTTP_MANAGER) << "Aggregated verdict is: " << ver;
if (ver == ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE) {
if (!state.getCurrentCustomResponse().ok()) {
dbgWarning(D_HTTP_MANAGER) << "No custom response found for verdict CUSTOM_RESPONSE";
return FilterVerdict(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT);
}
return FilterVerdict(ver, state.getCurrentCustomResponse().unpack());
} else {
FilterVerdict aggregated_verdict(ver, state.getCurrWebUserResponse());
if (aggregated_verdict.getVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify();
}
return aggregated_verdict;
}
}
static void
compressAppSecLogs(LogBulkRest &bulk)
@@ -394,11 +420,12 @@ private:
}
I_Table *i_transaction_table;
static const ngx_http_cp_verdict_e default_verdict;
string custom_header = "";
static const ServiceVerdict default_verdict;
static const string app_sec_marker_key;
};
const ngx_http_cp_verdict_e HttpManager::Impl::default_verdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
const ServiceVerdict HttpManager::Impl::default_verdict(ServiceVerdict::TRAFFIC_VERDICT_DROP);
const string HttpManager::Impl::app_sec_marker_key = "app_sec_marker";
HttpManager::HttpManager() : Component("HttpManager"), pimpl(make_unique<Impl>()) {}
@@ -409,10 +436,10 @@ void HttpManager::init() { pimpl->init(); }
void
HttpManager::preload()
{
registerExpectedConfiguration<uint>("HTTP manager", "Previous Buffer Cache size");
registerExpectedConfiguration<uint>("HTTP manager", "Max Request Body Size");
registerExpectedConfiguration<uint>("HTTP manager", "Max Response Body Size");
registerExpectedConfiguration<string>("HTTP manager", "Request Size Limit Verdict");
registerExpectedConfiguration<string>("HTTP manager", "Response Size Limit Verdict");
registerExpectedConfigurationWithCache<uint>("assetId", "HTTP manager", "Previous Buffer Cache size");
registerExpectedConfigurationWithCache<uint>("assetId", "HTTP manager", "Max Request Body Size");
registerExpectedConfigurationWithCache<uint>("assetId", "HTTP manager", "Max Response Body Size");
registerExpectedConfigurationWithCache<string>("assetId", "HTTP manager", "Request Size Limit Verdict");
registerExpectedConfigurationWithCache<string>("assetId", "HTTP manager", "Response Size Limit Verdict");
registerConfigLoadCb([this] () { pimpl->sendPolicyLog(); });
}

50
components/http_manager/http_manager_opaque.cc
View File

@@ -27,7 +27,7 @@ HttpManagerOpaque::HttpManagerOpaque()
}
void
HttpManagerOpaque::setApplicationVerdict(const string &app_name, ngx_http_cp_verdict_e verdict)
HttpManagerOpaque::setApplicationVerdict(const string &app_name, ServiceVerdict verdict)
{
applications_verdicts[app_name] = verdict;
}
@@ -39,49 +39,61 @@ HttpManagerOpaque::setApplicationWebResponse(const string &app_name, string web_
applications_web_user_response[app_name] = web_user_response_id;
}
ngx_http_cp_verdict_e
void
HttpManagerOpaque::setCustomResponse(const std::string &app_name, const CustomResponse &custom_response)
{
dbgTrace(D_HTTP_MANAGER) << "Security app: " << app_name
<< ", has custom response: " << custom_response.getBody()
<< ", with code: " << custom_response.getStatusCode();
current_custom_response = custom_response;
}
ServiceVerdict
HttpManagerOpaque::getApplicationsVerdict(const string &app_name) const
{
auto verdict = applications_verdicts.find(app_name);
return verdict == applications_verdicts.end() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT : verdict->second;
return verdict == applications_verdicts.end() ? ServiceVerdict::TRAFFIC_VERDICT_INSPECT : verdict->second;
}
ngx_http_cp_verdict_e
ServiceVerdict
HttpManagerOpaque::getCurrVerdict() const
{
if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (manager_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
return manager_verdict;
}
uint accepted_apps = 0;
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
for (const auto &app_verdic_pair : applications_verdicts) {
switch (app_verdic_pair.second) {
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP:
case ServiceVerdict::TRAFFIC_VERDICT_DROP:
dbgTrace(D_HTTP_MANAGER) << "Verdict DROP for app: " << app_verdic_pair.first;
current_web_user_response = applications_web_user_response.at(app_verdic_pair.first);
dbgTrace(D_HTTP_MANAGER) << "current_web_user_response=" << current_web_user_response;
return app_verdic_pair.second;
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT:
case ServiceVerdict::TRAFFIC_VERDICT_INJECT:
// Sent in ResponseHeaders and ResponseBody.
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT;
verdict = ServiceVerdict::TRAFFIC_VERDICT_INJECT;
break;
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT:
case ServiceVerdict::TRAFFIC_VERDICT_ACCEPT:
accepted_apps++;
break;
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT:
case ServiceVerdict::TRAFFIC_VERDICT_INSPECT:
break;
case ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS:
case ServiceVerdict::LIMIT_RESPONSE_HEADERS:
// Sent in End Request.
verdict = ngx_http_cp_verdict_e::LIMIT_RESPONSE_HEADERS;
verdict = ServiceVerdict::LIMIT_RESPONSE_HEADERS;
break;
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT:
case ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT:
dbgTrace(D_HTTP_MANAGER) << "Verdict 'Irrelevant' is not yet supported. Returning Accept";
accepted_apps++;
break;
case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT:
case ServiceVerdict::TRAFFIC_VERDICT_DELAYED:
// Sent in Request Headers and Request Body.
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
verdict = ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
break;
case ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE:
verdict = ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE;
break;
default:
dbgAssert(false)
@@ -91,18 +103,18 @@ HttpManagerOpaque::getCurrVerdict() const
}
}
return accepted_apps == applications_verdicts.size() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT : verdict;
return accepted_apps == applications_verdicts.size() ? ServiceVerdict::TRAFFIC_VERDICT_ACCEPT : verdict;
}
std::set<std::string>
HttpManagerOpaque::getCurrentDropVerdictCausers() const
{
std::set<std::string> causers;
if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (manager_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
causers.insert(HTTP_MANAGER_NAME);
}
for (const auto &app_verdic_pair : applications_verdicts) {
bool was_dropped = app_verdic_pair.second == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
bool was_dropped = app_verdic_pair.second == ServiceVerdict::TRAFFIC_VERDICT_DROP;
dbgTrace(D_HTTP_MANAGER)
<< "The verdict from: " << app_verdic_pair.first
<< (was_dropped ? " is \"drop\"" : " is not \"drop\" ");

20
components/http_manager/http_manager_opaque.h
View File

@@ -18,7 +18,8 @@
#include "buffer.h"
#include "table_opaque.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "http_inspection_events.h"
static const std::string HTTP_MANAGER_NAME = "HTTP Manager";
@@ -27,17 +28,19 @@ class HttpManagerOpaque : public TableOpaqueSerialize<HttpManagerOpaque>
public:
HttpManagerOpaque();
void setApplicationVerdict(const std::string &app_name, ngx_http_cp_verdict_e verdict);
void setApplicationVerdict(const std::string &app_name, ServiceVerdict verdict);
void setApplicationWebResponse(const std::string &app_name, std::string web_user_response_id);
ngx_http_cp_verdict_e getApplicationsVerdict(const std::string &app_name) const;
void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; }
ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; }
ngx_http_cp_verdict_e getCurrVerdict() const;
void setCustomResponse(const std::string &app_name, const CustomResponse &custom_response);
ServiceVerdict getApplicationsVerdict(const std::string &app_name) const;
void setManagerVerdict(ServiceVerdict verdict) { manager_verdict = verdict; }
ServiceVerdict getManagerVerdict() const { return manager_verdict; }
ServiceVerdict getCurrVerdict() const;
const std::string & getCurrWebUserResponse() const { return current_web_user_response; };
std::set<std::string> getCurrentDropVerdictCausers() const;
void saveCurrentDataToCache(const Buffer &full_data);
void setUserDefinedValue(const std::string &value) { user_defined_value = value; }
Maybe<std::string> getUserDefinedValue() const { return user_defined_value; }
Maybe<CustomResponse> getCurrentCustomResponse() const { return current_custom_response; }
const Buffer & getPreviousDataCache() const { return prev_data_cache; }
uint getAggeregatedPayloadSize() const { return aggregated_payload_size; }
void updatePayloadSize(const uint curr_payload);
@@ -53,12 +56,13 @@ public:
static uint minVer() { return 0; }
private:
std::unordered_map<std::string, ngx_http_cp_verdict_e> applications_verdicts;
std::unordered_map<std::string, ServiceVerdict> applications_verdicts;
std::unordered_map<std::string, std::string> applications_web_user_response;
mutable std::string current_web_user_response;
ngx_http_cp_verdict_e manager_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict manager_verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
Buffer prev_data_cache;
uint aggregated_payload_size = 0;
Maybe<CustomResponse> current_custom_response = genError("uninitialized");
Maybe<std::string> user_defined_value = genError("uninitialized");
};

11
components/include/generic_rulebase/match_query.h
View File

@@ -68,8 +68,12 @@ public:
const std::vector<IpProtoRange> & getProtoValue() const { return ip_proto_value; }
const std::vector<MatchQuery> & getItems() const { return items; }
std::string getFirstValue() const { return first_value; }
MatchResult getMatch(const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs) const;
bool matchAttributes(const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs) const;
MatchResult getMatch(
const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs,
bool skip_irrelevant_key = false) const;
bool matchAttributes(
const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs,
bool skip_irrelevant_key = false) const;
bool matchException(const std::string &behaviorKey, const std::string &behaviorValue) const;
bool isKeyTypeIp() const;
bool isKeyTypePort() const;
@@ -82,7 +86,8 @@ public:
private:
bool matchAttributes(
const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs,
std::set<std::string> &matched_override_keywords) const;
std::set<std::string> &matched_override_keywords,
bool skip_irrelevant_key = false) const;
StaticKeys getKeyByName(const std::string &key_type_name);
bool matchAttributes(const std::set<std::string> &values,
std::set<std::string> &matched_override_keywords) const;

13
components/include/generic_rulebase/parameters_config.h
View File

@@ -190,7 +190,7 @@ public:
static void
preload()
{
registerExpectedConfiguration<ParameterException>("rulebase", "exception");
registerExpectedConfigurationWithCache<ParameterException>("assetId", "rulebase", "exception");
registerConfigLoadCb([](){ is_geo_location_exception_exists = is_geo_location_exception_being_loaded; });
registerConfigPrepareCb([](){ is_geo_location_exception_being_loaded = false; });
}
@@ -198,14 +198,20 @@ public:
void load(cereal::JSONInputArchive &archive_in);
std::set<ParameterBehavior>
getBehavior(const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs) const;
getBehavior(
const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs,
bool skip_irrelevant_key = false) const;
std::set<ParameterBehavior>
getBehavior(
const std::unordered_map<std::string, std::set<std::string>> &key_value_pairs,
std::set<std::string> &matched_override_keywords) const;
std::set<std::string> &matched_override_keywords,
bool skip_irrelevant_key = false) const;
static bool isGeoLocationExceptionExists() { return is_geo_location_exception_exists; }
const MatchQuery& getMatch() const { return match; }
bool isContainingKVPair() const { return is_containing_kv_pair; }
bool checkKVPair() const;
private:
class MatchBehaviorPair
@@ -221,6 +227,7 @@ private:
ParameterBehavior behavior;
static bool is_geo_location_exception_exists;
static bool is_geo_location_exception_being_loaded;
bool is_containing_kv_pair;
};
static const ParameterBehavior action_ignore(BehaviorKey::ACTION, BehaviorValue::IGNORE);

2
components/include/generic_rulebase/rulebase_config.h
View File

@@ -112,7 +112,7 @@ public:
static void
preload()
{
registerExpectedConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
registerExpectedConfigurationWithCache<BasicRuleConfig>("assetId", "rulebase", "rulesConfig");
registerExpectedSetting<std::vector<BasicRuleConfig>>("rulebase", "rulesConfig");
registerConfigLoadCb(BasicRuleConfig::updateCountMetric);
registerConfigPrepareCb([](){ BasicRuleConfig::assets_ids_aggregation.clear(); });

26
components/include/generic_rulebase/triggers_config.h
View File

@@ -52,7 +52,7 @@ public:
static void
preload()
{
registerExpectedConfiguration<WebTriggerConf>("rulebase", "webUserResponse");
registerExpectedConfigurationWithCache<WebTriggerConf>("triggerId", "rulebase", "webUserResponse");
}
/// \brief Load function to deserialize configuration from JSONInputArchive.
@@ -104,6 +104,14 @@ public:
return redirect_url;
}
/// \brief Get the content type for the trigger.
/// \return The content type for the trigger.
const std::string &
getContentType() const
{
return content_type;
}
/// \brief Check if the trigger should add an event ID to the header.
/// \return True if the trigger should add an event ID, otherwise false.
bool
@@ -120,6 +128,7 @@ private:
std::string details_level;
std::string response_body;
std::string redirect_url;
std::string content_type;
uint response_code;
bool add_event_id_to_header = false;
};
@@ -175,7 +184,8 @@ public:
static void
preload()
{
registerExpectedConfiguration<LogTriggerConf>("rulebase", "log");
//registerExpectedConfiguration<LogTriggerConf>("rulebase", "log");
registerExpectedConfigurationWithCache<LogTriggerConf>("triggerId", "rulebase", "log");
}
/// \brief LogGen operator for LogTriggerConf.
@@ -260,6 +270,15 @@ public:
return should_log_on_prevent.isSet(security_type);
}
/// \brief Check if should ignore exception log for the given security type.
/// \param security_type The security type to check.
/// \return True if should ignore exception, otherwise false.
bool
shouldIgnoreExceptionLog(SecurityType security_type) const
{
return should_log_exception.isSet(security_type);
}
/// \brief Check if the log is active on detect for the given security type.
/// \param security_type The security type to check.
/// \return True if the log is active on detect, otherwise false.
@@ -333,6 +352,7 @@ private:
Flags<ReportIS::StreamType> active_streams;
Flags<SecurityType> should_log_on_detect;
Flags<SecurityType> should_log_on_prevent;
Flags<SecurityType> should_log_exception;
Flags<SecurityType> log_geo_location;
Flags<WebLogFields> log_web_fields;
extendLoggingSeverity extend_logging_severity = extendLoggingSeverity::None;
@@ -349,7 +369,7 @@ public:
static void
preload()
{
registerExpectedConfiguration<ReportTriggerConf>("rulebase", "report");
registerExpectedConfigurationWithCache<ReportTriggerConf>("triggerId", "rulebase", "report");
}
/// \brief Load function to deserialize configuration from JSONInputArchive.

24
components/include/http_event_impl/filter_verdict.h
View File

@@ -22,13 +22,13 @@
class FilterVerdict
{
public:
FilterVerdict(ngx_http_cp_verdict_e _verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT)
FilterVerdict(ServiceVerdict _verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT)
:
verdict(_verdict)
{}
FilterVerdict(
ngx_http_cp_verdict_e _verdict,
ServiceVerdict _verdict,
const std::string &_web_reponse_id)
:
verdict(_verdict),
@@ -40,15 +40,21 @@ public:
verdict(_verdict.getVerdict()),
web_user_response_id(_verdict.getWebUserResponseByPractice())
{
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_INJECT) {
addModifications(_verdict.getModifications(), _event_idx);
}
}
FilterVerdict(ServiceVerdict _verdict, const CustomResponse &_custom_response)
:
verdict(_verdict),
custom_response(_custom_response)
{}
void
addModifications(const FilterVerdict &other)
{
if (other.verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INJECT) return;
if (other.verdict != ServiceVerdict::TRAFFIC_VERDICT_INJECT) return;
modifications.insert(modifications.end(), other.modifications.begin(), other.modifications.end());
total_modifications += other.total_modifications;
@@ -58,22 +64,24 @@ public:
addModifications(
const ModificationList &mods,
ModifiedChunkIndex _event_idx,
ngx_http_cp_verdict_e alt_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT)
ServiceVerdict alt_verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT)
{
total_modifications += mods.size();
modifications.push_back(EventModifications(_event_idx, mods));
if (alt_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) verdict = alt_verdict;
if (alt_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) verdict = alt_verdict;
}
uint getModificationsAmount() const { return total_modifications; }
ngx_http_cp_verdict_e getVerdict() const { return verdict; }
ServiceVerdict getVerdict() const { return verdict; }
const std::vector<EventModifications> & getModifications() const { return modifications; }
const std::string getWebUserResponseID() const { return web_user_response_id; }
Maybe<CustomResponse> getCustomResponse() const { return custom_response; }
private:
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
std::vector<EventModifications> modifications;
std::string web_user_response_id;
Maybe<CustomResponse> custom_response = genError("uninitialized");
uint total_modifications = 0;
};

74
components/include/http_event_impl/i_http_event_impl.h
View File

@@ -25,12 +25,12 @@
#include "debug.h"
#include "buffer.h"
#include "http_transaction_data.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
USE_DEBUG_FLAG(D_HTTP_MANAGER);
using ModificationType = ngx_http_modification_type_e;
using ModificationPosition = ngx_http_cp_inject_pos_t;
using ModificationType = HttpModificationType;
using ModificationPosition = NanoHttpCpInjectPos;
static const ModificationPosition injection_pos_irrelevant = INJECT_POS_IRRELEVANT;
@@ -185,12 +185,18 @@ class HttpHeader
{
public:
HttpHeader() = default;
HttpHeader(const Buffer &_key, const Buffer &_value, uint8_t _header_index, bool _is_last_header = false)
HttpHeader(
const Buffer &_key,
const Buffer &_value,
uint8_t _header_index,
bool _is_last_header = false,
bool _should_log = true)
:
key(_key),
value(_value),
is_last_header(_is_last_header),
header_index(_header_index)
header_index(_header_index),
should_log(_should_log)
{
}
@@ -203,7 +209,8 @@ public:
key,
value,
is_last_header,
header_index
header_index,
should_log
);
}
@@ -215,7 +222,8 @@ public:
key,
value,
is_last_header,
header_index
header_index,
should_log
);
}
// LCOV_EXCL_STOP
@@ -232,6 +240,8 @@ public:
<< std::to_string(header_index)
<< ", Is last header: "
<< (is_last_header ? "True" : "False")
<< ", Should log: "
<< (should_log ? "True" : "False")
<< ")";
}
@@ -241,12 +251,18 @@ public:
bool isLastHeader() const { return is_last_header; }
void setIsLastHeader() { is_last_header = true; }
uint8_t getHeaderIndex() const { return header_index; }
bool shouldLog() const { return should_log; }
void setShouldNotLog() {
dbgTrace(D_HTTP_MANAGER) << "Header '" << std::dumpHex(key) << "' marked as should not log";
should_log = false;
}
private:
Buffer key;
Buffer value;
bool is_last_header = false;
uint8_t header_index = 0;
bool should_log = true;
};
using BodyModification = Buffer;
@@ -362,23 +378,54 @@ private:
uint8_t body_chunk_index;
};
class CustomResponse
{
public:
CustomResponse(
const std::string& body,
uint16_t status_code,
const std::string& content_type = "application/json"
) :
body(body),
status_code(status_code),
content_type(content_type)
{}
std::string getBody() const { return body; }
uint16_t getStatusCode() const { return status_code; }
std::string getContentType() const { return content_type; }
private:
std::string body;
uint16_t status_code;
std::string content_type;
};
class EventVerdict
{
public:
EventVerdict() = default;
EventVerdict(ngx_http_cp_verdict_e event_verdict) : modifications(), verdict(event_verdict) {}
EventVerdict(ServiceVerdict event_verdict) : modifications(), verdict(event_verdict) {}
EventVerdict(const ModificationList &mods) : modifications(mods) {}
EventVerdict(const ModificationList &mods, ngx_http_cp_verdict_e event_verdict) :
EventVerdict(const ModificationList &mods, ServiceVerdict event_verdict) :
modifications(mods),
verdict(event_verdict)
{}
EventVerdict(
const CustomResponse &custom_response
) :
modifications(),
verdict(ServiceVerdict::TRAFFIC_VERDICT_CUSTOM_RESPONSE),
custom_response(custom_response)
{}
EventVerdict(
const ModificationList &mods,
ngx_http_cp_verdict_e event_verdict,
ServiceVerdict event_verdict,
std::string response_id) :
modifications(mods),
verdict(event_verdict),
@@ -390,17 +437,20 @@ public:
// LCOV_EXCL_STOP
const ModificationList & getModifications() const { return modifications; }
ngx_http_cp_verdict_e getVerdict() const { return verdict; }
ServiceVerdict getVerdict() const { return verdict; }
const std::string getWebUserResponseByPractice() const { return webUserResponseByPractice; }
void setWebUserResponseByPractice(const std::string id) {
dbgTrace(D_HTTP_MANAGER) << "current verdict web user response set to: " << id;
webUserResponseByPractice = id;
}
Maybe<CustomResponse> getCustomResponse() const { return custom_response; }
private:
ModificationList modifications;
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
std::string webUserResponseByPractice;
Maybe<CustomResponse> custom_response = genError("uninitialized");
};
#endif // __I_HTTP_EVENT_IMPL_H__

1
components/include/http_inspection_events.h
View File

@@ -31,6 +31,7 @@ public:
const Buffer & getValue() const { return req_header.getValue(); }
bool isLastHeader() const { return req_header.isLastHeader(); }
uint8_t getHeaderIndex() const { return req_header.getHeaderIndex(); }
bool shouldLog() const { return req_header.shouldLog(); }
template <class Archive>
void

11
components/include/i_static_resources_handler.h
View File

@@ -21,7 +21,16 @@
class I_StaticResourcesHandler
{
public:
virtual bool registerStaticResource(const std::string &resource_name, const std::string &resource_full_path) = 0;
virtual bool registerStaticResource(
const std::string &resource_name,
const std::string &resource_full_path,
bool overwrite_if_exists = false
) = 0;
virtual bool registerStaticResourceByContent(
const std::string &resource_name,
const std::string &file_content
) = 0;
protected:
virtual ~I_StaticResourcesHandler() {}

2
components/include/ips_comp.h
View File

@@ -2,13 +2,13 @@
#define __IPS_COMP_H__
#include "singleton.h"
#include "i_generic_rulebase.h"
#include "i_keywords_rule.h"
#include "i_table.h"
#include "i_mainloop.h"
#include "i_http_manager.h"
#include "i_environment.h"
#include "http_inspection_events.h"
#include "i_generic_rulebase.h"
#include "component.h"
class IPSComp

12
components/include/mock/mock_nginx_attachment.h
View File

@@ -7,9 +7,17 @@ class MockNginxAttachment:
public Singleton::Provide<I_StaticResourcesHandler>::From<MockProvider<I_StaticResourcesHandler>>
{
public:
MOCK_METHOD2(
MOCK_METHOD3(
registerStaticResource,
bool(const std::string &static_resource_name, const std::string &static_resource_path)
bool(const std::string &static_resource_name,
const std::string &static_resource_path,
bool overwrite_if_exists
)
);
MOCK_METHOD2(
registerStaticResourceByContent,
bool(const std::string &resource_name, const std::string &file_content)
);
};

8
components/include/nginx_intaker_metric.h
View File

@@ -14,7 +14,7 @@
#ifndef __NGINX_INTAKER_METRIC_H__
#define __NGINX_INTAKER_METRIC_H__
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "generic_metric.h"
#include "cpu/cpu_metric.h"
@@ -26,11 +26,11 @@ public:
void resetAllCounters();
ngx_http_plugin_metric_type_e EnumOfIndex(int i);
AttachmentMetricType EnumOfIndex(int i);
void addPluginMetricCounter(const ngx_http_cp_metric_data_t *recieved_metric_data);
void addPluginMetricCounter(const NanoHttpMetricData *recieved_metric_data);
uint64_t getPluginMetricCounter(ngx_http_plugin_metric_type_e _verdict) const;
uint64_t getPluginMetricCounter(AttachmentMetricType _verdict) const;
void notifyCPU() const { cpu_event.notify(); }

4
components/include/rate_limit_config.h
View File

@@ -120,6 +120,7 @@ public:
const std::vector<RateLimitRule> & getRateLimitRules() const { return rate_limit_rules; }
const RateLimitAction & getRateLimitMode() const { return mode; }
const std::string & getWebUserResponse() const { return web_user_response; }
RateLimitRule generateSiblingRateLimitRule(const RateLimitRule &rule);
@@ -150,6 +151,8 @@ public:
static bool isActive() { return is_active; }
void setWebUserResponse(const std::string& response_id) { web_user_response = response_id; }
static const std::map<RateLimitAction, std::string> rate_limit_action_to_string;
static const std::map<std::string, RateLimitAction> rate_limit_string_to_action;
@@ -160,6 +163,7 @@ private:
static bool is_active;
RateLimitAction mode;
std::vector<RateLimitRule> rate_limit_rules;
std::string web_user_response;
};
#endif // __RATE_LIMIT_CONFIG_H__

1
components/include/reverse_proxy_defaults.h
View File

@@ -44,6 +44,7 @@ static const std::string default_syslog_socket_address = "127.0.0.1:1514";
static const std::string rpm_full_load_path = "/tmp/rpm_full_load";
static const std::string rpm_partial_load_path = "/tmp/rpm_partial_load";
static const std::string first_rpm_policy_load_path = "/tmp/first_rpm_policy_load";
static const std::string readiness_file_path = "/tmp/readiness";
static const int default_port = 5555;

1
components/include/user_identifiers_config.h
View File

@@ -31,7 +31,6 @@ public:
std::vector<std::string> getHeaderValuesFromConfig(const std::string &header_key) const;
void setXFFValuesToOpaqueCtx(const HttpHeader &header, ExtractType type) const;
void setWafTagValuesToOpaqueCtx(const HttpHeader &header) const;
private:
class UsersIdentifiersConfig
{

615
components/nginx_message_reader/nginx_message_reader.cc
View File

@@ -51,12 +51,56 @@ static const boost::regex error_log_regex(
", (host: \".+?\")$"
);
static const boost::regex incident_log_regex(
"("
+ syslog_regex_string + ") "
+ "incidentLog\\s+"
"host=([^\\s]+)\\s+"
"host_port=(\\d+)\\s+"
"client_addr=([\\d\\.]+)\\s+"
"client_port=(\\d+)\\s+"
"time_local=\\[([^\\]]+)\\]\\s+"
"request=\"([^\"]+)\"\\s+"
"status=(\\d{3})\\s+"
"uri=\"([^\"]+)\"\\s+"
"request_id=([^\\s]+)\\s+"
"upstream_status=([^,\\s]+)(?:,\\s*[^\\s]+)?\\s+"
"upstream_response_time=([^,\\s]+)(?:,\\s*[^\\s]+)?\\s+"
"body_bytes_sent=(\\d+)\\s+"
"referer=\"([^\"]*)\"\\s+"
"user_agent=\"([^\"]*)\"\\s+"
"pf=([^\\s]*)\\s+"
"x_event_id=([^\\s]*)"
);
static const boost::regex generic_crit_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[crit\\] (.+)$"
);
static const boost::regex generic_emerg_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[emerg\\] (.+)$"
);
static const boost::regex generic_fallback_log_regex(
"("
+ syslog_regex_string + ") "
+ "(?:\\d{4}/\\d{2}/\\d{2} \\d{2}:\\d{2}:\\d{2} )?" // Optional nginx timestamp
+ "\\[(\\w+)\\] (.+)$"
);
static const boost::regex server_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+)|(\\w+\\.\\w+)");
static const boost::regex uri_regex("^/");
static const boost::regex port_regex("\\d+");
static const boost::regex response_code_regex("[0-9]{3}");
static const boost::regex http_method_regex("[A-Za-z]+");
static const string central_nginx_manager = "Central NGINX Manager";
class NginxMessageReader::Impl
{
public:
@@ -64,6 +108,15 @@ public:
init()
{
dbgFlow(D_NGINX_MESSAGE_READER);
if (Singleton::exists<I_Environment>()) {
auto name = Singleton::Consume<I_Environment>::by<Report>()->get<string>("Service Name");
if (name.ok()) {
dbgInfo(D_NGINX_MESSAGE_READER) << "Service name: " << *name;
service_name = *name;
}
}
I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<NginxMessageReader>();
mainloop->addOneTimeRoutine(
I_MainLoop::RoutineType::System,
@@ -97,6 +150,7 @@ public:
"429",
"accessControl.rateLimit.returnCode"
);
dbgTrace(D_NGINX_MESSAGE_READER) << "Selected rate-limit status code: " << rate_limit_status_code;
}
@@ -107,6 +161,7 @@ private:
RESPONSE_CODE,
HOST,
SOURCE,
SOURCE_PORT,
DESTINATION_IP,
DESTINATION_PORT,
EVENT_MESSAGE,
@@ -114,9 +169,21 @@ private:
ASSET_NAME,
RULE_NAME,
RULE_ID,
REFERENCE_ID,
LOG_TYPE,
PROXY_FAULT,
X_EVENT_ID,
MATCHED_REASON,
REMEDIATION,
COUNT
};
struct GenericLogInfo {
string timestamp;
string severity;
string message;
};
void
initSyslogServerSocket()
{
@@ -175,10 +242,12 @@ private:
bool log_sent;
if (isAccessLog(log)) {
log_sent = sendAccessLog(log);
} else if (isAlertErrorLog(log) || isErrorLog(log)) {
} else if (isAlertErrorLog(log) || isErrorLog(log) || isCritErrorLog(log) || isEmergErrorLog(log)) {
log_sent = sendErrorLog(log);
} else if (isIncidentLog(log)) {
log_sent = sendIncidentLog(log);
} else {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format for message: "<< log;
continue;
}
if (!log_sent) {
@@ -222,13 +291,74 @@ private:
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Error log" << log;
Maybe<EnumArray<LogInfo, string>> log_info = parseErrorLog(log);
if (!log_info.ok()) {
if (log_info.ok()) {
return sendLog(log_info.unpack());
}
if (service_name == central_nginx_manager) {
dbgDebug(D_NGINX_MESSAGE_READER) << "Detailed parsing failed, trying generic parsing";
Maybe<GenericLogInfo> generic_log = parseGenericErrorLog(log);
if (generic_log.ok()) {
return sendGenericLog(generic_log.unpack());
}
}
dbgWarning(D_NGINX_MESSAGE_READER)
<< "Failed parsing the NGINX logs. Error: "
<< log_info.getErr();
<< log_info.getErr()
<< service_name;
return false;
}
return sendLog(log_info.unpack());
static bool
isValidUuid(const string &uuid_str)
{
if (uuid_str.empty() || uuid_str == "-") {
return false;
}
if (uuid_str.length() != 36) {
return false;
}
if (uuid_str[8] != '-' || uuid_str[13] != '-' || uuid_str[18] != '-' || uuid_str[23] != '-') {
return false;
}
for (size_t i = 0; i < uuid_str.length(); ++i) {
if (i == 8 || i == 13 || i == 18 || i == 23) {
continue;
}
char c = uuid_str[i];
if (!((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'))) {
return false;
}
}
return true;
}
bool
sendIncidentLog(const string &log)
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Incident log" << log;
Maybe<EnumArray<LogInfo, string>> log_info = parseIncidentLog(log);
if (!log_info.ok()) {
dbgTrace(D_NGINX_MESSAGE_READER) << log_info.getErr();
return false;
}
auto unpacked_log_info = log_info.unpack();
if (isValidUuid(unpacked_log_info[LogInfo::X_EVENT_ID])) {
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Skipping incident log - valid x_event_id present: "
<< unpacked_log_info[LogInfo::X_EVENT_ID];
return true;
}
unpacked_log_info[LogInfo::LOG_TYPE] = "incidentLog";
return sendNotification(unpacked_log_info);
}
bool
@@ -253,7 +383,111 @@ private:
}
bool
sendLog(const EnumArray<LogInfo, string> &log_info)
isIncidentLog(const string &log) const
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'incident log'. Log: " << log;
return log.find("incidentLog") != string::npos;
}
bool
sendNotification(const EnumArray<LogInfo, string> &log_info)
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name;
if (
log_info[LogInfo::LOG_TYPE] == "incidentLog"
&& (log_info[LogInfo::RESPONSE_CODE][0] == '4' || log_info[LogInfo::RESPONSE_CODE][0] == '5')
) {
bool is_proxy_fault = (log_info[LogInfo::PROXY_FAULT] == "yes");
event_name = getEventName(is_proxy_fault);
} else {
return false;
}
LogGen log(
event_name,
ReportIS::Level::LOG,
ReportIS::Audience::SECURITY,
ReportIS::Severity::HIGH,
ReportIS::Priority::HIGH,
ReportIS::Tags::REVERSE_PROXY
);
log << LogField("eventConfidence", "High");
for (LogInfo field : makeRange<LogInfo>()) {
Maybe<string> string_field = convertLogFieldToString(field);
if (!string_field.ok()) {
dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " << string_field.getErr();
return false;
}
if (string_field.unpack().empty() || log_info[field].empty()) {
continue;
}
if (field == LogInfo::ASSET_ID || field == LogInfo::ASSET_NAME) {
log.addToOrigin(LogField(string_field.unpack(), log_info[field]));
continue;
}
if (field != LogInfo::DESTINATION_PORT && field != LogInfo::SOURCE_PORT) {
log << LogField(string_field.unpack(), log_info[field]);
continue;
}
try {
log << LogField(string_field.unpack(), stoi(log_info[field]));
} catch (const exception &e) {
dbgError(D_NGINX_MESSAGE_READER)
<< "Unable to convert port to numeric value: "
<< e.what();
log << LogField(string_field.unpack(), 0);
}
}
return true;
}
bool
isCritErrorLog(const string &log) const
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'crit log'. Log: " << log;
return log.find("[crit]") != string::npos;
}
bool
isEmergErrorLog(const string &log) const
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'emerg log'. Log: " << log;
return log.find("[emerg]") != string::npos;
}
Maybe<string>
getCNMEventName(const EnumArray<LogInfo, string> &log_info) const
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name;
switch (log_info[LogInfo::RESPONSE_CODE][0]) {
case '4': {
event_name = "NGINX Proxy Error: Invalid request or incorrect NGINX configuration - Request dropped."
" Please check the reverse proxy configuration of your relevant assets";
break;
}
case '5': {
event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration."
"If the issue persists please contact open-appsec support";
break;
}
default: {
dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code";
return genError("Irrelevant status code");
}
}
return event_name;
}
Maybe<string>
getRPMEventName(const EnumArray<LogInfo, string> &log_info) const
{
dbgFlow(D_NGINX_MESSAGE_READER);
string event_name;
@@ -271,20 +505,48 @@ private:
}
default: {
dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code";
return false;
return genError("Irrelevant status code");
}
}
return event_name;
}
Maybe<string> getEventName(const EnumArray<LogInfo, string> &log_info)
{
if (service_name != central_nginx_manager) {
dbgWarning(D_NGINX_MESSAGE_READER)
<< "Unknown service name: "
<< service_name
<< " Response will be sent as CNM";
}
return getCNMEventName(log_info);
}
bool
sendLog(const EnumArray<LogInfo, string> &log_info)
{
dbgFlow(D_NGINX_MESSAGE_READER);
Maybe<string> event_name = getEventName(log_info);
if (!event_name.ok()) {
dbgError(D_NGINX_MESSAGE_READER) << event_name.getErr();
return false;
}
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Nginx log's event name and response code: "
<< event_name
<< event_name.unpack()
<< ", "
<< log_info[LogInfo::RESPONSE_CODE];
LogGen log(
event_name,
event_name.unpack(),
ReportIS::Audience::SECURITY,
ReportIS::Severity::INFO,
ReportIS::Severity::HIGH,
ReportIS::Priority::LOW,
service_name == central_nginx_manager ?
ReportIS::Tags::CENTRAL_NGINX_MANAGER :
ReportIS::Tags::REVERSE_PROXY
);
log << LogField("eventConfidence", "High");
@@ -296,6 +558,10 @@ private:
return false;
}
if (string_field.unpack().empty() || log_info[field].empty()) {
continue;
}
if (field != LogInfo::DESTINATION_PORT) {
log << LogField(string_field.unpack(), log_info[field]);
continue;
@@ -313,6 +579,45 @@ private:
return true;
}
bool
sendGenericLog(const GenericLogInfo &log_info)
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Sending generic log";
string event_name = "NGINX Proxy Error: Request failed! Please verify your proxy configuration."
"If the issue persists please contact open-appsec support";
ReportIS::Severity severity = ReportIS::Severity::MEDIUM;
ReportIS::Priority priority = ReportIS::Priority::MEDIUM;
if (log_info.severity == "emerg" || log_info.severity == "crit") {
severity = ReportIS::Severity::CRITICAL;
priority = ReportIS::Priority::URGENT;
} else if (log_info.severity == "error" || log_info.severity == "alert") {
severity = ReportIS::Severity::HIGH;
priority = ReportIS::Priority::HIGH;
}
LogGen log(
event_name,
ReportIS::Audience::SECURITY,
severity,
priority,
ReportIS::Tags::CENTRAL_NGINX_MANAGER
);
log << LogField("eventConfidence", "High");
log << LogField("timestamp", log_info.timestamp);
log << LogField("httpResponseBody", formatGenericLogMessage(log_info));
return true;
}
string
formatGenericLogMessage(const GenericLogInfo &log_info)
{
return "[" + log_info.severity + "] " + log_info.message;
}
bool
sendRateLimitLog(const EnumArray<LogInfo, string> &log_info)
{
@@ -338,7 +643,6 @@ private:
string security_action = "Drop";
bool is_log_required = false;
// Prevent events checkbox (in triggers)
if (rate_limit_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) {
is_log_required = true;
}
@@ -373,6 +677,10 @@ private:
return false;
}
if (string_field.unpack().empty() || log_info[field].empty()) {
continue;
}
if (
field == LogInfo::HOST ||
field == LogInfo::URI ||
@@ -406,6 +714,55 @@ private:
return true;
}
static string
getStatusCodeMessage(const string &status_code)
{
static map<string, string> status_messages = {
// 4xx Client Error codes
{"400", "Bad Request - malformed syntax"},
{"401", "Unauthorized - authentication required"},
{"403", "Forbidden - access denied"},
{"404", "Not Found - resource does not exist"},
{"405", "Method Not Allowed - HTTP verb not permitted"},
{"408", "Request Timeout - client too slow"},
{"411", "Length Required - missing Content-Length"},
{"413", "Payload Too Large - body exceeds limit"},
{"414", "URI Too Long - request target exceeds limit"},
{"416", "Range Not Satisfiable - invalid byte range"},
{"429", "Too Many Requests - rate limit exceeded"},
{"431", "Header Fields Too Large - headers exceed limit"},
{"451", "Unavailable For Legal Reasons"},
// NGINX specific 4xx codes
{"494", "Request Header Too Large - NGINX internal"},
{"495", "SSL Certificate Error - invalid client cert"},
{"496", "SSL Certificate Required - none provided"},
{"497", "Plain HTTP sent to HTTPS port"},
// 5xx Server Error codes
{"500", "Internal Server Error"},
{"501", "Not Implemented - feature unsupported"},
{"502", "Bad Gateway - upstream connection failed"},
{"503", "Service Unavailable - server overloaded"},
{"504", "Gateway Timeout - upstream too slow"},
{"505", "HTTP Version Not Supported"},
{"507", "Insufficient Storage - WebDAV quota exceeded"}
};
auto it = status_messages.find(status_code);
return (it != status_messages.end()) ? it->second : "HTTP Error " + status_code;
}
static string
getEventName(bool is_proxy_fault)
{
if (!is_proxy_fault) {
return "Upstream Application Error";
}
return "Reverse Proxy Error";
}
Maybe<string>
convertLogFieldToString(LogInfo field)
{
@@ -420,7 +777,9 @@ private:
case LogInfo::HOST:
return string("httpHostName");
case LogInfo::SOURCE:
return string("httpSourceId");
return string("sourceip");
case LogInfo::SOURCE_PORT:
return string("sourcePort");
case LogInfo::DESTINATION_IP:
return string("destinationIp");
case LogInfo::DESTINATION_PORT:
@@ -435,6 +794,18 @@ private:
return string("ruleId");
case LogInfo::RULE_NAME:
return string("ruleName");
case LogInfo::REFERENCE_ID:
return string("eventReferenceId");
case LogInfo::MATCHED_REASON:
return string("matchreason");
case LogInfo::REMEDIATION:
return string("eventRemediation");
case LogInfo::LOG_TYPE:
return string("");
case LogInfo::PROXY_FAULT:
return string("");
case LogInfo::X_EVENT_ID:
return string("");
case LogInfo::COUNT:
dbgError(D_NGINX_MESSAGE_READER) << "LogInfo::COUNT is not allowed";
return genError("LogInfo::COUNT is not allowed");
@@ -512,7 +883,7 @@ private:
static_cast<uint16_t>(stoi(log_info[LogInfo::DESTINATION_PORT]))
);
} catch (const exception &e) {
dbgError(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what();
dbgWarning(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what();
}
ctx.registerValue<string>(HttpTransactionData::host_name_ctx, log_info[LogInfo::HOST]);
ctx.registerValue<string>(HttpTransactionData::uri_ctx, log_info[LogInfo::URI]);
@@ -525,12 +896,65 @@ private:
}
BasicRuleConfig context = rule_by_ctx.unpack();
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Adding context fields to log info. Asset ID: "
<< context.getAssetId()
<< ", Asset Name: "
<< context.getAssetName()
<< ", Rule ID: "
<< context.getRuleId()
<< ", Rule Name: "
<< context.getRuleName();
log_info[LogInfo::ASSET_ID] = context.getAssetId();
log_info[LogInfo::ASSET_NAME] = context.getAssetName();
log_info[LogInfo::RULE_ID] = context.getRuleId();
log_info[LogInfo::RULE_NAME] = context.getRuleName();
}
Maybe<GenericLogInfo>
parseGenericErrorLog(const string &log_line)
{
dbgFlow(D_NGINX_MESSAGE_READER) << "Parsing generic error log: " << log_line;
boost::smatch matcher;
GenericLogInfo generic_log;
if (isCritErrorLog(log_line)) {
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_crit_log_regex)) {
const int timestamp_index = 2;
const int message_index = 5;
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = "crit";
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
} else if (isEmergErrorLog(log_line)) {
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_emerg_log_regex)) {
const int timestamp_index = 2;
const int message_index = 5;
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = "emerg";
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
}
if (NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, generic_fallback_log_regex)) {
const int timestamp_index = 2;
const int severity_index = 5;
const int message_index = 6;
generic_log.timestamp = string(matcher[timestamp_index].first, matcher[timestamp_index].second);
generic_log.severity = string(matcher[severity_index].first, matcher[severity_index].second);
generic_log.message = string(matcher[message_index].first, matcher[message_index].second);
return generic_log;
}
dbgWarning(D_NGINX_MESSAGE_READER) << "Could not parse log with generic method: " << log_line;
return genError("Could not parse log with generic method");
}
Maybe<EnumArray<LogInfo, string>>
parseErrorLog(const string &log_line)
{
@@ -540,17 +964,28 @@ private:
boost::smatch matcher;
vector<string> result;
boost::regex selected_regex;
if (isAlertErrorLog(log_line)) {
selected_regex = alert_log_regex;
} else if (isErrorLog(log_line)) {
selected_regex = error_log_regex;
} else {
dbgWarning(D_NGINX_MESSAGE_READER) << "No matching log type found for log: " << log_line;
return genError("No matching log type found");
}
if (
!NGEN::Regex::regexSearch(
__FILE__,
__LINE__,
log_line,
matcher,
isAlertErrorLog(log_line) ? alert_log_regex : error_log_regex
selected_regex
)
) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
return genError("Unexpected nginx log format");
dbgWarning(D_NGINX_MESSAGE_READER) << "Detailed regex parsing failed for log: " << log_line;
return genError("Detailed regex parsing failed");
}
const int event_message_index = 6;
@@ -591,8 +1026,8 @@ private:
addContextFieldsToLogInfo(log_info);
if (!validateLog(log_info)) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
return genError("Unexpected nginx log format");
dbgWarning(D_NGINX_MESSAGE_READER) << "Log validation failed for detailed parsing";
return genError("Log validation failed for detailed parsing");
}
return log_info;
@@ -642,6 +1077,104 @@ private:
return log_info;
}
Maybe<EnumArray<LogInfo, string>>
parseIncidentLog(const string &log_line)
{
dbgTrace(D_NGINX_MESSAGE_READER) << "Parsing incident log line: " << log_line;
EnumArray<LogInfo, string> log_info(EnumArray<LogInfo, string>::Fill(), string(""));
boost::smatch matcher;
if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_line, matcher, incident_log_regex)) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx incident log format";
return genError("Unexpected nginx incident log format");
}
const int host_index = 5; // host=server_name
const int host_port_index = 6; // host_port=port
const int client_addr_index = 7; // client_addr=IP
const int client_port_index = 8; // client_port=port
const int request_index = 10; // request="METHOD URI HTTP/1.1"
const int status_index = 11; // status=CODE
const int uri_index = 12; // uri="PATH"
const int request_id_index = 13; // request_id=ID
const int proxy_fault_index = 19; // pf=yes or empty
const int x_event_id_index = 20; // x_event_id=value or empty
string host = string(matcher[host_index].first, matcher[host_index].second);
string host_port = string(matcher[host_port_index].first, matcher[host_port_index].second);
string uri = string(matcher[uri_index].first, matcher[uri_index].second);
string proxy_fault = string(matcher[proxy_fault_index].first, matcher[proxy_fault_index].second);
log_info[LogInfo::HOST] = host;
log_info[LogInfo::URI] = uri;
log_info[LogInfo::DESTINATION_PORT] = host_port;
log_info[LogInfo::PROXY_FAULT] = proxy_fault;
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Parsed host: "
<< host
<< ", host_port: "
<< host_port
<< ", uri: "
<< uri
<< ", proxy_fault: "
<< proxy_fault;
addContextFieldsToLogInfo(log_info);
if (log_info[LogInfo::ASSET_ID].empty() || log_info[LogInfo::ASSET_NAME].empty()) {
return genError("Asset ID or Asset Name is empty");
}
string client_addr = string(matcher[client_addr_index].first, matcher[client_addr_index].second);
string client_port = string(matcher[client_port_index].first, matcher[client_port_index].second);
string request = string(matcher[request_index].first, matcher[request_index].second);
string status = string(matcher[status_index].first, matcher[status_index].second);
string request_id = string(matcher[request_id_index].first, matcher[request_id_index].second);
string x_event_id = string(matcher[x_event_id_index].first, matcher[x_event_id_index].second);
dbgTrace(D_NGINX_MESSAGE_READER)
<< "Parsed client_addr: "
<< client_addr
<< ", client_port: "
<< client_port
<< ", request: "
<< request
<< ", status: "
<< status
<< ", request_id: "
<< request_id
<< ", proxy_fault: "
<< proxy_fault
<< ", x_event_id: "
<< x_event_id;
vector<string> request_parts;
boost::split(request_parts, request, boost::is_any_of(" "), boost::token_compress_on);
string http_method = request_parts.size() > 0 ? request_parts[0] : "";
log_info[LogInfo::REFERENCE_ID] = request_id;
log_info[LogInfo::RESPONSE_CODE] = status;
log_info[LogInfo::HTTP_METHOD] = http_method;
log_info[LogInfo::SOURCE] = client_addr;
log_info[LogInfo::SOURCE_PORT] = client_port;
log_info[LogInfo::DESTINATION_IP] = host;
log_info[LogInfo::X_EVENT_ID] = x_event_id;
if (!validateLog(log_info)) {
dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx incident log format after validation";
return genError("Unexpected nginx incident log format after validation");
}
if (proxy_fault == "yes") {
log_info[LogInfo::MATCHED_REASON] = getStatusCodeMessage(status);
log_info[LogInfo::REMEDIATION] = getIncidentLogRemediation(status);
}
return log_info;
}
static bool
validateLog(const EnumArray<LogInfo, string> &log_info)
{
@@ -708,8 +1241,54 @@ private:
return move(raw_log);
}
static string
getIncidentLogRemediation(const string &status_code)
{
static map<string, string> status_remediations = {
{"400", "Check request syntax and format; validate client input and HTTP headers"},
{"401", "Verify authentication credentials and configuration"},
{"403", "If this is a valid WAF block, no action is required; otherwise,"
" contact Check Point Support"},
{"404", "Validate the requested asset URI"},
{"405", "Ensure allowed HTTP methods are configured correctly"},
{"408", "Adjust `client_header_timeout` and `client_body_timeout`"
" in the assets advanced nginx server block"},
{"411", "Require a Content-Length header on requests with a body"},
{"413", "Increase `client_max_body_size` in the asset advanced nginx server block"},
{"414", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)"
" in the asset advanced nginx server block"},
{"416", "Validate Range request headers against actual file sizes"},
{"429", "Review rate limiting settings and implement proper backoff strategies"},
{"431", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)"
" in the asset advanced nginx server block"},
{"451", "Review content filtering policies and legal compliance requirements"},
{"494", "Increase `large_client_header_buffers` (e.g. `large_client_header_buffers 4 32k;`)"
" in the asset advanced nginx server block"},
{"495", "Confirm client SSL certificates are valid and properly configured"},
{"496", "Configure SSL client certificate requirements correctly"},
{"497", "Enforce HTTPS for secure endpoints"},
{"500", "Contact Check Point Support"},
{"501", "Ensure requested features are supported and configured correctly"},
{"502", "Verify upstream server connectivity and health; check proxy configuration"},
{"503", "Assess server capacity and load balancing; verify upstream availability"},
{"504", "Increase `proxy_connect_timeout` and `proxy_read_timeout`"
" in the asset advanced nginx configuration"},
{"505", "Ensure HTTP version compatibility between client and server"},
{"507", "Contact Check Point Support"}
};
auto it = status_remediations.find(status_code);
if (it != status_remediations.end()) {
return it->second + ". If this suggestion does not resolve the issue, please contact support.";
}
return "Please verify the nginx configuration of your relevant assets. "
"Please contact support if the issue persists.";
}
I_Socket::socketFd syslog_server_socket = -1;
string rate_limit_status_code = "429";
string service_name = "Unnamed Nano Service";
};
NginxMessageReader::NginxMessageReader() : Component("NginxMessageReader"), pimpl(make_unique<Impl>()) {}

108
components/security_apps/http_geo_filter/http_geo_filter.cc
View File

@@ -35,7 +35,8 @@ public:
init()
{
dbgTrace(D_GEO_FILTER) << "Init Http Geo filter component";
registerListener();
handleNewPolicy();
registerConfigLoadCb([this]() { handleNewPolicy(); });
}
void
@@ -44,6 +45,19 @@ public:
unregisterListener();
}
void
handleNewPolicy()
{
if (ParameterException::isGeoLocationExceptionExists()) {
registerListener();
return;
}
if (!ParameterException::isGeoLocationExceptionExists()) {
unregisterListener();
}
}
string getListenerName() const override { return "HTTP geo filter"; }
void
@@ -56,7 +70,7 @@ public:
<< "Load http geo filter default action. Action: "
<< default_action_maybe.unpack();
} else {
default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
dbgTrace(D_GEO_FILTER) << "No http geo filter default action. Action: Irrelevant";
}
}
@@ -66,7 +80,7 @@ public:
{
dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event";
if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
if (!event.isLastHeader()) return EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_INSPECT);
std::set<std::string> ip_set;
auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
auto maybe_xff = env->get<std::string>(HttpTransactionData::xff_vals_ctx);
@@ -101,15 +115,17 @@ public:
}
ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set);
if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
ServiceVerdict exception_verdict = getExceptionVerdict(ip_set);
if (exception_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) {
return EventVerdict(exception_verdict);
}
ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(ip_set);
if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
return EventVerdict(geo_lookup_verdict);
}
// deprecated for now
// ServiceVerdict geo_lookup_verdict = getGeoLookupVerdict(ip_set);
// if (geo_lookup_verdict != ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT) {
// return EventVerdict(geo_lookup_verdict);
// }
return EventVerdict(default_action);
}
@@ -146,7 +162,7 @@ private:
void
removeTrustedIpsFromXff(std::set<std::string> &xff_set)
{
auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
auto identify_config = getConfigurationWithCache<UsersAllIdentifiersConfig>(
"rulebase",
"usersIdentifiers"
);
@@ -189,33 +205,34 @@ private:
return os.str();
}
ngx_http_cp_verdict_e
convertActionToVerdict(const string &action) const
{
if (action == "accept") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
if (action == "drop") return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
}
ngx_http_cp_verdict_e
ServiceVerdict
convertBehaviorValueToVerdict(const BehaviorValue &behavior_value) const
{
if (behavior_value == BehaviorValue::ACCEPT || behavior_value == BehaviorValue::IGNORE) {
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (behavior_value == BehaviorValue::DROP || behavior_value == BehaviorValue::REJECT) {
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
}
ngx_http_cp_verdict_e
// LCOV_EXCL_START Reason: deprecated for now
ServiceVerdict
convertActionToVerdict(const string &action) const
{
if (action == "accept") return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
if (action == "drop") return ServiceVerdict::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
ServiceVerdict
getGeoLookupVerdict(const std::set<std::string> &sources)
{
auto maybe_geo_config = getConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
if (!maybe_geo_config.ok()) {
dbgTrace(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
return ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
}
GeoConfig geo_config = maybe_geo_config.unpack();
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
@@ -252,12 +269,12 @@ private:
<< ", country code: "
<< country_code;
generateVerdictLog(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
ServiceVerdict::TRAFFIC_VERDICT_ACCEPT,
geo_config.getId(),
true,
geo_location_data
);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (geo_config.isBlockedCountry(country_code)) {
dbgTrace(D_GEO_FILTER)
@@ -266,12 +283,12 @@ private:
<< ", country code: "
<< country_code;
generateVerdictLog(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
ServiceVerdict::TRAFFIC_VERDICT_DROP,
geo_config.getId(),
true,
geo_location_data
);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
}
dbgTrace(D_GEO_FILTER)
@@ -286,22 +303,23 @@ private:
);
return convertActionToVerdict(geo_config.getDefaultAction());
}
// LCOV_EXCL_STOP
Maybe<pair<ngx_http_cp_verdict_e, string>>
Maybe<pair<ServiceVerdict, string>>
getBehaviorsVerdict(
const unordered_map<string, set<string>> &behaviors_map_to_search,
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data)
{
bool is_matched = false;
ParameterBehavior matched_behavior;
ngx_http_cp_verdict_e matched_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
ServiceVerdict matched_verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<HttpGeoFilter>();
set<ParameterBehavior> behaviors_set = i_rulebase->getBehavior(behaviors_map_to_search);
dbgTrace(D_GEO_FILTER) << "get verdict from: " << behaviors_set.size() << " behaviors";
for (const ParameterBehavior &behavior : behaviors_set) {
matched_verdict = convertBehaviorValueToVerdict(behavior.getValue());
if (
matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP
matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP
){
dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId();
generateVerdictLog(
@@ -310,10 +328,10 @@ private:
false,
geo_location_data
);
return pair<ngx_http_cp_verdict_e, string>(matched_verdict, behavior.getId());
return pair<ServiceVerdict, string>(matched_verdict, behavior.getId());
}
else if (
matched_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT
matched_verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT
){
dbgTrace(D_GEO_FILTER) << "behavior verdict: ACCEPT, exception id: " << behavior.getId();
matched_behavior = behavior;
@@ -321,19 +339,19 @@ private:
}
}
if (is_matched) {
return pair<ngx_http_cp_verdict_e, string>(
ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
return pair<ServiceVerdict, string>(
ServiceVerdict::TRAFFIC_VERDICT_ACCEPT,
matched_behavior.getId()
);
}
return genError("No exception matched to HTTP geo filter rule");
}
ngx_http_cp_verdict_e
ServiceVerdict
getExceptionVerdict(const std::set<std::string> &sources) {
pair<ngx_http_cp_verdict_e, string> curr_matched_behavior;
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
pair<ServiceVerdict, string> curr_matched_behavior;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
@@ -389,7 +407,7 @@ private:
if (matched_behavior_maybe.ok()) {
curr_matched_behavior = matched_behavior_maybe.unpack();
verdict = curr_matched_behavior.first;
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
return verdict;
}
}
@@ -402,13 +420,13 @@ private:
if (matched_behavior_maybe.ok()) {
curr_matched_behavior = matched_behavior_maybe.unpack();
verdict = curr_matched_behavior.first;
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP) {
return verdict;
}
}
}
if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) {
if (verdict == ServiceVerdict::TRAFFIC_VERDICT_ACCEPT) {
generateVerdictLog(
verdict,
curr_matched_behavior.second,
@@ -421,7 +439,7 @@ private:
void
generateVerdictLog(
const ngx_http_cp_verdict_e &verdict,
const ServiceVerdict &verdict,
const string &matched_id,
bool is_geo_filter,
const EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data,
@@ -430,7 +448,7 @@ private:
{
dbgTrace(D_GEO_FILTER) << "Generate Log for verdict - HTTP geo filter";
auto &trigger = getConfigurationWithDefault(default_triger, "rulebase", "log");
bool is_prevent = verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
bool is_prevent = verdict == ServiceVerdict::TRAFFIC_VERDICT_DROP;
string matched_on = is_geo_filter ? "geoFilterPracticeId" : "exceptionId";
LogGen log = trigger(
"Web Request - HTTP Geo Filter",
@@ -469,7 +487,7 @@ private:
<< LogField("sourceCountryName", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME]);
}
ngx_http_cp_verdict_e default_action = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
ServiceVerdict default_action = ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT;
};
HttpGeoFilter::HttpGeoFilter() : Component("HttpGeoFilter"), pimpl(make_unique<HttpGeoFilter::Impl>()) {}
@@ -491,6 +509,6 @@ void
HttpGeoFilter::preload()
{
registerExpectedConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
registerExpectedConfiguration<UsersAllIdentifiersConfig>("rulebase", "usersIdentifiers");
registerExpectedConfigurationWithCache<UsersAllIdentifiersConfig>("assetId", "rulebase", "usersIdentifiers");
registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); });
}

14
components/security_apps/ips/include/ips_signatures.h
View File

@@ -21,13 +21,13 @@
#include <vector>
#include "config.h"
#include "i_generic_rulebase.h"
#include "i_first_tier_agg.h"
#include "ips_entry.h"
#include "ips_enums.h"
#include "log_generator.h"
#include "parsed_context.h"
#include "pm_hook.h"
#include "i_generic_rulebase.h"
#define DEFAULT_IPS_YIELD_COUNT 500
@@ -402,6 +402,7 @@ public:
LogTriggerConf getTrigger() const;
std::set<ParameterBehavior>
getBehavior(const std::unordered_map<std::string, std::set<std::string>> &exceptions_dict) const;
@@ -410,10 +411,21 @@ private:
/// \param ips_state The IPS entry.
ActionResults getAction(const IPSEntry &ips_state) const;
void sendLog(
const Buffer &context_buffer,
const IPSEntry &ips_state,
const std::tuple<
IPSSignatureSubTypes::SignatureAction,
std::string, std::vector<std::string>
> &override_action,
bool is_prevent
) const;
std::shared_ptr<CompleteSignature> signature;
SignatureAction action;
std::string trigger_id;
std::string exception_id;
mutable bool bSupressLog = false;
};
} // namespace IPSSignatureSubTypes

12
components/security_apps/ips/ips_comp.cc
View File

@@ -13,7 +13,7 @@
#include "virtual_modifiers.h"
#include "helper.h"
#include "ips_common_types.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
using namespace std;
@@ -46,9 +46,9 @@ class IPSComp::Impl
public Listener<HttpResponseBodyEvent>,
public Listener<EndTransactionEvent>
{
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
class SigsFirstTierAgg
{
@@ -383,8 +383,8 @@ IPSComp::preload()
registerExpectedResource<SnortSignaturesResource>("IPSSnortSigs", "protections");
registerExpectedConfiguration<IPSConfiguration>("IPS", "IpsConfigurations");
registerExpectedConfiguration<uint>("IPS", "Max Field Size");
registerExpectedConfiguration<IPSSignatures>("IPS", "IpsProtections");
registerExpectedConfiguration<SnortSignatures>("IPSSnortSigs", "SnortProtections");
registerExpectedConfigurationWithCache<IPSSignatures>("assetId", "IPS", "IpsProtections");
registerExpectedConfigurationWithCache<SnortSignatures>("assetId", "IPSSnortSigs", "SnortProtections");
registerExpectedConfigFile("ips", Config::ConfigFileType::Policy);
registerExpectedConfigFile("ips", Config::ConfigFileType::Data);
registerExpectedConfigFile("snort", Config::ConfigFileType::Policy);

32
components/security_apps/ips/ips_signatures.cc
View File

@@ -321,6 +321,9 @@ SignatureAndAction::getAction(const IPSEntry &ips_state) const
override_actions.insert(behavior.getValue());
const string &override_id = behavior.getId();
if (!override_id.empty()) override_ids.push_back(override_id);
} else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) {
dbgTrace(D_IPS) << "setting bSupressLog due to override behavior: " << behavior.getId();
bSupressLog = true;
}
}
@@ -358,8 +361,10 @@ set<ParameterBehavior>
SignatureAndAction::getBehavior(const unordered_map<string, set<string>> &exceptions_dict) const
{
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<IPSComp>();
if (exception_id.empty()) return i_rulebase->getBehavior(exceptions_dict);
if (exception_id.empty()) {
dbgTrace(D_RULEBASE_CONFIG) << "No exception id provided, using default behavior";
return i_rulebase->getBehavior(exceptions_dict);
}
return i_rulebase->getParameterException(exception_id).getBehavior(exceptions_dict);
}
@@ -453,10 +458,25 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
return false;
}
dbgDebug(D_IPS) << "Signature matched - sending log";
bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT;
if(bSupressLog) {
dbgTrace(D_IPS) << "Signature matched - not sending log due to exception behavior";
} else {
sendLog(context_buffer, ips_state, override_action, is_prevent);
}
return is_prevent;
}
void SignatureAndAction::sendLog(
const Buffer &context_buffer,
const IPSEntry &ips_state,
const tuple<IPSSignatureSubTypes::SignatureAction, string, vector<string>> &override_action,
bool is_prevent
) const
{
dbgFlow(D_IPS) << "Signature matched - sending log";
auto trigger = getTrigger();
bool is_prevent = get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::PREVENT;
auto severity = signature->getSeverity() < IPSLevel::HIGH ? Severity::HIGH : Severity::CRITICAL;
if (get<0>(override_action) == IPSSignatureSubTypes::SignatureAction::DETECT) severity = Severity::INFO;
@@ -560,7 +580,7 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
uint res_size = res_body.ok() && trigger.isWebLogFieldActive(::res_body) ? res_body.unpack().size() : 0;
if (req_size + res_size > max_size) {
if (req_size + 500 > max_size) {
res_size = std::min(500u, res_size);
res_size = min(500u, res_size);
req_size = max_size - res_size;
} else {
res_size = max_size - req_size;
@@ -574,8 +594,6 @@ SignatureAndAction::isMatchedPrevent(const Buffer &context_buffer, const set<PMP
if (!get<2>(override_action).empty()) log.addToOrigin(LogField("exceptionIdList", get<2>(override_action)));
log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
return is_prevent;
}
void

6
components/security_apps/ips/ips_ut/component_ut.cc
View File

@@ -109,11 +109,11 @@ operator ==(const EventVerdict &first, const EventVerdict &second)
return first.getVerdict() == second.getVerdict();
}
const EventVerdict ComponentTest::inspect(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
const EventVerdict ComponentTest::inspect(ServiceVerdict::TRAFFIC_VERDICT_INSPECT);
const EventVerdict ComponentTest::accept(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT);
const EventVerdict ComponentTest::accept(ServiceVerdict::TRAFFIC_VERDICT_ACCEPT);
const EventVerdict ComponentTest::drop(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
const EventVerdict ComponentTest::drop(ServiceVerdict::TRAFFIC_VERDICT_DROP);
TEST_F(ComponentTest, check_init_fini_do_not_crush)
{

115
components/security_apps/ips/ips_ut/signatures_ut.cc
View File

@@ -19,6 +19,7 @@
#include "generic_rulebase/generic_rulebase_context.h"
#include "encryptor.h"
#include "mock/mock_table.h"
USE_DEBUG_FLAG(D_IPS);
using namespace testing;
using namespace std;
@@ -194,6 +195,101 @@ public:
gen_ctx->activate();
}
// Loads an exception with log: ignore behavior, as in the provided JSON
void loadExceptionsIgnoreLog()
{
BasicRuleConfig::preload();
registerExpectedConfiguration<ParameterException>("rulebase", "exception");
string test_config(
"{"
" \"rulebase\": {"
" \"rulesConfig\": ["
" {"
" \"context\": \"All()\","
" \"priority\": 1,"
" \"ruleId\": \"5eaef0726765c30010bae8bb\","
" \"ruleName\": \"Acme web API\","
" \"assetId\": \"5e243effd858007660b758ad\","
" \"assetName\": \"Acme Power API\","
" \"parameters\": ["
" {"
" \"parameterId\": \"6c3867be-4da5-42c2-93dc-8f509a764003\","
" \"parameterType\": \"exceptions\","
" \"parameterName\": \"exception\""
" }"
" ],"
" \"zoneId\": \"\","
" \"zoneName\": \"\""
" }"
" ],"
" \"exception\": ["
" {"
" \"context\": \"Any(parameterId(6c3867be-4da5-42c2-93dc-8f509a764003))\","
" \"exceptions\": ["
" {"
" \"match\": {"
" \"type\": \"operator\","
" \"op\": \"and\","
" \"items\": ["
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"sourceIdentifier\","
" \"value\": [\"1.1.1.1\"]"
" }"
" ]"
" },"
" \"behavior\": {"
" \"key\": \"log\","
" \"value\": \"ignore\""
" }"
" },"
" {"
" \"match\": {"
" \"type\": \"operator\","
" \"op\": \"or\","
" \"items\": ["
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"protectionName\","
" \"value\": [\"Test1\"]"
" },"
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"protectionName\","
" \"value\": [\"Test2\"]"
" },"
" {"
" \"type\": \"condition\","
" \"op\": \"equals\","
" \"key\": \"sourceIdentifier\","
" \"value\": [\"1.1.1.1\"]"
" }"
" ]"
" },"
" \"behavior\": {"
" \"key\": \"action\","
" \"value\": \"accept\""
" }"
" }"
" ]"
" }"
" ]"
" }"
"}"
);
istringstream ss(test_config);
auto i_config = Singleton::Consume<Config::I_Config>::from(config);
i_config->loadConfiguration(ss);
gen_ctx = make_unique<GenericRulebaseContext>();
gen_ctx->activate();
}
void
load(const IPSSignaturesResource &policy, const string &severity, const string &confidence)
{
@@ -261,6 +357,7 @@ public:
IPSSignaturesResource performance_signatures3;
IPSSignaturesResource single_broken_signature;
NiceMock<MockTable> table;
NiceMock<MockLogging> logs;
MockAgg mock_agg;
private:
@@ -273,7 +370,6 @@ private:
ConfigComponent config;
Encryptor encryptor;
AgentDetails details;
StrictMock<MockLogging> logs;
IPSEntry ips_state;
string signature1 =
@@ -524,6 +620,23 @@ TEST_F(SignatureTest, basic_load_of_signatures)
EXPECT_FALSE(sigs.isEmpty("HTTP_REQUEST_BODY"));
}
TEST_F(SignatureTest, ignore_exception_suppresses_log)
{
load(single_signature2, "Low or above", "Low");
loadExceptionsIgnoreLog();
expectLog("\"protectionId\": \"Test3\"", "\"eventSeverity\": \"Critical\"");
EXPECT_TRUE(checkData("gggddd"));
ScopedContext ctx;
ctx.registerValue<string>("sourceIdentifiers", "1.1.1.1");
// No log should be sent when the exception matches
EXPECT_CALL(logs, sendLog(_)).Times(0);
EXPECT_FALSE(checkData("gggddd"));
}
TEST_F(SignatureTest, single_signature_matching_override)
{
load(single_signature, "Low or above", "Low");

18
components/security_apps/layer_7_access_control/layer_7_access_control.cc
View File

@@ -7,7 +7,7 @@
#include "config.h"
#include "cache.h"
#include "http_inspection_events.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "log_generator.h"
@@ -117,12 +117,12 @@ public:
if (!isAppEnabled()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
if (!event.isLastHeader()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Inspect verdict";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
return handleEvent();
@@ -133,7 +133,7 @@ public:
{
if (!isAppEnabled()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Returning Accept verdict as the Layer-7 Access Control app is disabled";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Handling wait verdict";
@@ -217,13 +217,13 @@ Layer7AccessControl::Impl::queryIpReputation(const string &source_ip)
if (!ip_reputation.ok()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Scheduling Intelligence query - returning Wait verdict";
scheduleIntelligenceQuery(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
return ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
}
if (!ip_reputation.unpack().isMalicious()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Accepting IP: " << source_ip;
ip_reputation_cache.deleteEntry(source_ip);
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
return generateLog(source_ip, ip_reputation.unpack());
@@ -246,7 +246,7 @@ Layer7AccessControl::Impl::handleEvent()
}
dbgWarning(D_L7_ACCESS_CONTROL) << "Could not extract the Client IP address from context";
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
void
@@ -354,11 +354,11 @@ Layer7AccessControl::Impl::generateLog(const string &source_ip, const Intelligen
if (isPrevent()) {
dbgTrace(D_L7_ACCESS_CONTROL) << "Dropping IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
return ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
dbgTrace(D_L7_ACCESS_CONTROL) << "Detecting IP: " << source_ip;
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
return ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
Maybe<LogField, Context::Error>

8
components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc
View File

@@ -49,10 +49,10 @@ public:
void registerTransactionData();
void verifyReport(const Report &report, const string &source_identifier, const string &security_action);
const EventVerdict drop_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
const EventVerdict accept_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
const EventVerdict inspect_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
const EventVerdict wait_verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT;
const EventVerdict drop_verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
const EventVerdict accept_verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
const EventVerdict inspect_verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
const EventVerdict wait_verdict = ServiceVerdict::TRAFFIC_VERDICT_DELAYED;
Layer7AccessControl l7_access_control;
::Environment env;
ConfigComponent config;

168
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@@ -68,6 +68,15 @@ checkSAMLPortal(const string &command_output)
return string("false");
}
Maybe<string>
checkIdaPDP(const string &command_output)
{
if (command_output.find("is_collecting_identities (true)") != string::npos) {
return string("true");
}
return string("false");
}
Maybe<string>
checkInfinityIdentityEnabled(const string &command_output)
{
@@ -139,6 +148,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
return command_output;
}
Maybe<string>
checkIsCME(const string &command_output)
{
if (command_output == "" ) return string("false");
return command_output;
}
Maybe<string>
getOtlpAgentGaiaOsRole(const string &command_output)
{
@@ -147,6 +164,129 @@ getOtlpAgentGaiaOsRole(const string &command_output)
return command_output;
}
// Helper function for case-insensitive substring search
inline bool
containsIgnoreCase(const string& text, const string& pattern) {
string lowerText = text;
string lowerPattern = pattern;
transform(lowerText.begin(), lowerText.end(), lowerText.begin(), ::tolower);
transform(lowerPattern.begin(), lowerPattern.end(), lowerPattern.begin(), ::tolower);
return lowerText.find(lowerPattern) != string::npos;
}
inline Maybe<string>
extractValue(const string& line, const string& field) {
size_t colonPos = line.find(':');
if (colonPos == string::npos) {
return Maybe<string>(Error<string>("no match"));
}
string key = line.substr(0, colonPos);
string value = line.substr(colonPos + 1);
key.erase(0, key.find_first_not_of(" \t"));
key.erase(key.find_last_not_of(" \t") + 1);
value.erase(0, value.find_first_not_of(" \t"));
value.erase(value.find_last_not_of(" \t") + 1);
if (containsIgnoreCase(key, field)) {
return Maybe<string>(value);
}
return Maybe<string>(Error<string>("no match"));
}
inline std::pair<std::string, std::string>
parseDmidecodeOutput(const std::string& dmidecodeOutput) {
string manufacturer;
string product;
istringstream stream(dmidecodeOutput);
string line;
while (getline(stream, line)) {
if (manufacturer.empty()) {
auto extractedManufacturer = extractValue(line, "Manufacturer");
if (extractedManufacturer.ok() && !extractedManufacturer->empty()) {
manufacturer = *extractedManufacturer;
}
}
if (product.empty()) {
auto extractedProduct = extractValue(line, "Product Name");
if (extractedProduct.ok() && !extractedProduct->empty()) {
product = *extractedProduct;
}
}
if (!manufacturer.empty() && !product.empty()) {
break;
}
}
return make_pair(manufacturer, product);
}
Maybe<string>
getAiopCgnsHardwareType(const string &command_output)
{
if (command_output == "" ) return string("NA");
auto pair = parseDmidecodeOutput(command_output);
if (containsIgnoreCase(pair.first, "Amazon")) {
return string("AWS");
}
if (containsIgnoreCase(pair.first, "Microsoft")) {
return string("Azure");
}
if (containsIgnoreCase(pair.first, "Google")) {
return string("Google Cloud");
}
if (containsIgnoreCase(pair.first, "Oracle")) {
return string("OCI");
}
if (containsIgnoreCase(pair.first, "Alibaba")) {
return string("Alibaba");
}
if (containsIgnoreCase(pair.second, "VMware")) {
return string("VMware");
}
if (containsIgnoreCase(pair.first, "OpenStack")) {
return string("OpenStack");
}
// Check for KVM (manufacturer OR product)
if (containsIgnoreCase(pair.first, "QEMU") || containsIgnoreCase(pair.second, "KVM")) {
return string("KVM");
}
if (containsIgnoreCase(pair.first, "Xen")) {
return string("Xen");
}
if (containsIgnoreCase(pair.first, "Nutanix")) {
return string("Nutanix");
}
return string("NA");
}
Maybe<string>
getAiopsCgnsCloudVendor(const string &command_output)
{
if (command_output == "" ) return string("NA");
string platform = "NA";
istringstream stream(command_output);
string line;
while (getline(stream, line)) {
if (line.find("platform") != string::npos) {
size_t colonPos = line.find(' ');
if (colonPos != string::npos) {
platform = line.substr(colonPos + 1);
platform.erase(0, platform.find_first_not_of(" \t"));
platform.erase(platform.find_last_not_of(" \t") + 1);
break;
}
}
}
return platform;
}
Maybe<string>
getQUID(const string &command_output)
{
@@ -158,10 +298,23 @@ getQUID(const string &command_output)
return command_output;
}
// Handler for a comma-separated list of QUIDs
Maybe<string>
getIsAiopsRunning(const string &command_output)
getQUIDList(const string &command_output)
{
if (command_output == "" ) return string("false");
if (command_output.empty()) {
return string("false");
}
std::istringstream ss(command_output);
std::string quid;
while (std::getline(ss, quid, ',')) {
const auto res = getQUID(quid);
if (!res.ok()) {
return res; // Return the error directly with context from getQUID
}
}
return command_output;
}
@@ -349,6 +502,15 @@ getGWIPAddress(const string &command_output)
return getAttr(command_output, "IP Address was not found");
}
Maybe<string>
getGWIPv6Address(const string &command_output)
{
if (command_output.empty() || command_output == "null") {
return genError("IPv6 Address was not found");
}
return string(command_output);
}
Maybe<string>
getGWVersion(const string &command_output)
{
@@ -366,7 +528,7 @@ checkIfSdwanRunning(const string &command_output)
Maybe<string>
getClusterObjectIP(const string &command_output)
{
return getAttr(command_output, "Cluster object IP was not found");
return command_output;
}
Maybe<string>

59
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@@ -46,18 +46,23 @@ SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
"FS_PATH=<FILESYSTEM-PREFIX>; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] "
"&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
checkIsInstallHorizonTelemetrySucceeded)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"${PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
SHELL_CMD_HANDLER("isCME", "[ -d /opt/CPcme ] && echo 'true' || echo 'false'", checkIsCME)
#endif
#if defined(gaia)
SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''",
getQUID)
SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
"IS_MDS=$(cpprod_util CPPROD_IsConfigured PROVIDER-1 2>/dev/null | tr -d ' ');"
"if [ \"${IS_MDS}\" = \"1\" ]; then "
"DOMAIN_NAME=$(echo \"${FS_PATH}\" | grep -o -E \"domain-[^/]+\" | sed 's|domain-||');"
"[ -z \"${DOMAIN_NAME}\" ] && echo '' && exit 0;"
"sed \"s|###DOMAIN_NAME###|${DOMAIN_NAME}|g\" /opt/CPotelcol/quid_api/get_mds_quid.json"
" > /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME};"
"[ -f /opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} ] && "
"python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mds_quid.json.${DOMAIN_NAME} 2>/dev/null | jq -r .message[0].MDS_QUID || echo '';"
"else "
"VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
"[ -z \"${VS_ID}\" ] && "
"(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');"
@@ -66,7 +71,8 @@ SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
" > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); "
"[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && "
"(python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
"/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');"
"fi",
getQUID)
SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
@@ -76,9 +82,21 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("MHO_QUID",
"[ -d /opt/CPquid ] && "
"python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_mho_quid.json 2>/dev/null | "
"jq -r '[.message[]? | select(.MHO_QUID != \"\") | .MHO_QUID] | join(\",\")' 2>/dev/null || "
"echo ''",
getQUIDList)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
"&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
getOtlpAgentGaiaOsRole)
SHELL_CMD_HANDLER("AIOPS_CGNS_HW_TYPE", ""
"command -v dmidecode &>/dev/null && dmidecode -t 1 2>/dev/null",
getAiopCgnsHardwareType)
SHELL_CMD_HANDLER("AIOPS_CGNS_CLOUD_VENDOR",
"cat /etc/cloud-version 2>/dev/null",
getAiopsCgnsCloudVendor)
SHELL_CMD_HANDLER("ETH_MGMT_IP",
"FS_PATH=<FILESYSTEM-PREFIX>;"
"VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
@@ -104,7 +122,12 @@ SHELL_CMD_HANDLER("MGMT_QUID", "echo ''", getQUID)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "echo 'SMB'", getOtlpAgentGaiaOsRole)
#endif
#if defined(gaia) || defined(smb) || defined(smb_thx_v3) || defined(smb_sve_v2) || defined(smb_mrv_v1)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"hasSDWan",
"[ $(cpprod_util CPPROD_IsMgmtMachine) -eq 1 ] && echo '0' ||"
"([ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0')",
checkHasSDWan
)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
"jq -r .can_update_sdwan_data /tmp/cpsdwan_getdata_orch.json",
@@ -131,10 +154,11 @@ SHELL_CMD_HANDLER(
)
SHELL_CMD_HANDLER(
"cpProductIntegrationMgmtParentObjectIP",
"obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";"
"[ $(cpprod_util FwIsHighAvail) -eq 1 ] && "
"(obj=\"$(jq -r .cluster_name /tmp/cpsdwan_getdata_orch.json)\";"
" awk -v obj=\"$obj\" '$1 == \":\" && $2 == \"(\" obj, $1 == \":ip_address\" { if ($1 == \":ip_address\")"
" { gsub(/[()]/, \"\", $2); print $2; exit; } }'"
" $FWDIR/state/local/FW1/local.gateway_cluster",
" $FWDIR/state/local/FW1/local.gateway_cluster) || echo \"\"",
getClusterObjectIP
)
SHELL_CMD_HANDLER(
@@ -146,7 +170,16 @@ SHELL_CMD_HANDLER("is_legacy_qos_blade_enabled",
"cpprod_util CPPROD_GetValue FG1 ProdActive 1 | grep -q '^1$' "
"&& (cpprod_util CPPROD_GetValue FG1 FgSDWAN 1 | grep -q '^1$' && echo false || echo true) || "
"echo false",
checkQosLegacyBlade)
checkQosLegacyBlade
)
SHELL_CMD_HANDLER(
"IPv6 Address",
"( [ $(cpprod_util FwIsHighAvail) -eq 1 ] && [ $(cpprod_util FwIsVSX) -eq 1 ]"
"&& (jq -r .cluster_main_ipv6 /tmp/cpsdwan_getdata_orch.json) )"
"|| ( [ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" )"
"|| (jq -r .main_ipv6 /tmp/cpsdwan_getdata_orch.json)",
getGWIPv6Address
)
#endif //gaia || smb
#if defined(gaia)
@@ -154,6 +187,10 @@ SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedB
SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade)
SHELL_CMD_HANDLER("hasVPNBlade", "enabled_blades", checkVPNBlade)
SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal)
SHELL_CMD_HANDLER("hasIdaPdpEnabled",
"cat $FWDIR/database/myself_objects.C | grep is_collecting_identities",
checkIdaPDP
)
SHELL_CMD_HANDLER("hasInfinityIdentityEnabled",
"cat $FWDIR/database/myself_objects.C | grep get_identities_from_infinity_identity",
checkInfinityIdentityEnabled

41
components/security_apps/orchestration/health_check/health_check.cc
View File

@@ -154,7 +154,8 @@ private:
static const map<string, pair<string, int>> ip_port_defaults_map = {
{"Azure", make_pair(getenv("DOCKER_RPM_ENABLED") ? "" : "168.63.129.16", 8117)},
{"Aws", make_pair("", 8117)},
{"Local", make_pair("", 8117)}
{"Local", make_pair("", 8117)},
{"VMware", make_pair("", 8117)}
};
auto cloud_vendor_maybe = getSetting<string>("reverseProxy", "cloudVendorName");
@@ -271,6 +272,12 @@ private:
return HealthCheckStatus::UNHEALTHY;
}
if (checkReadinessFilesExist()) {
dbgTrace(D_HEALTH_CHECK)
<< "Readiness file exists, instance not ready for traffic, returning unhealthy status";
return HealthCheckStatus::UNHEALTHY;
}
if (NGEN::Filesystem::exists(rpm_full_load_path)) {
dbgTrace(D_HEALTH_CHECK) << "RPM is fully loaded";
return i_service_controller->getServicesPolicyStatus()
@@ -289,6 +296,24 @@ private:
return HealthCheckStatus::UNHEALTHY;
}
bool
checkReadinessFilesExist()
{
string readiness_dir = readiness_file_path.substr(0, readiness_file_path.find_last_of('/'));
string readiness_filename = NGEN::Filesystem::getFileName(readiness_file_path);
auto directory_files = NGEN::Filesystem::getDirectoryFiles(readiness_dir);
if (!directory_files.ok()) return false;
for (const string& filename : directory_files.unpack()) {
if (NGEN::Strings::startsWith(filename, readiness_filename)) {
return true;
}
}
return false;
}
bool
nginxContainerIsRunning()
{
@@ -304,7 +329,19 @@ private:
return false;
}
return (*maybe_result).find(nginx_container_name) != string::npos;
bool container_running = (*maybe_result).find(nginx_container_name) != string::npos;
if (!container_running) {
dbgTrace(D_HEALTH_CHECK) << "Nginx container is not running";
return false;
}
if (checkReadinessFilesExist()) {
dbgTrace(D_HEALTH_CHECK) << "Readiness file exists on host machine, not ready for traffic";
return false;
}
dbgTrace(D_HEALTH_CHECK) << "Nginx container is running and no readiness files found - ready for traffic";
return true;
}
void

6
components/security_apps/orchestration/manifest_controller/manifest_diff_calculator.cc
View File

@@ -158,6 +158,12 @@ ManifestDiffCalculator::buildInstallationQueue(
installation_queue.push_back(orchestration_it->second);
}
auto shared_libs_it = new_packages.find("sharedLibs");
if (shared_libs_it != new_packages.end()) {
installation_queue.push_back(shared_libs_it->second);
}
auto wlp_standalone_it = new_packages.find("wlpStandalone");
if (wlp_standalone_it != new_packages.end()){
installation_queue.push_back(wlp_standalone_it->second);

5
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -1535,6 +1535,11 @@ private:
if (i_details_resolver->compareCheckpointVersion(8200, greater_equal<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER82", "true");
}
if (i_details_resolver->compareCheckpointVersion(8200, equal_to<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "true");
} else {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionR82", "false");
}
#endif // gaia || smb
if (agent_data_report == curr_agent_data_report) {

23
components/security_apps/prometheus/prometheus_comp.cc
View File

@@ -40,7 +40,13 @@ struct ServiceData
class PrometheusMetricData
{
public:
PrometheusMetricData(const string &n, const string &t, const string &d) : name(n), type(t), description(d) {}
PrometheusMetricData(const string &n, const string &u, const string &t, const string &d)
:
name(n),
unique_name(u),
type(t),
description(d)
{}
void
addElement(const string &labels, const string &value)
@@ -55,7 +61,9 @@ public:
string representative_name = "";
if (!name.empty()) {
auto metric_name = convertMetricName(name);
string metric_name;
if (!unique_name.empty()) metric_name = convertMetricName(unique_name);
if (metric_name.empty()) metric_name = convertMetricName(name);
!metric_name.empty() ? representative_name = metric_name : representative_name = name;
}
@@ -73,6 +81,7 @@ public:
private:
string name;
string unique_name;
string type;
string description;
map<string, string> metric_labels_to_values;
@@ -98,6 +107,7 @@ public:
for(auto &metric : metrics) {
auto &metric_object = getDataObject(
metric.name,
metric.unique_name,
metric.type,
metric.description
);
@@ -107,11 +117,14 @@ public:
private:
PrometheusMetricData &
getDataObject(const string &name, const string &type, const string &description)
getDataObject(const string &name, const string &unique_name, const string &type, const string &description)
{
auto elem = prometheus_metrics.find(name);
auto elem = prometheus_metrics.find(unique_name);
if (elem == prometheus_metrics.end()) {
elem = prometheus_metrics.emplace(name, PrometheusMetricData(name, type, description)).first;
elem = prometheus_metrics.emplace(
unique_name,
PrometheusMetricData(name, unique_name, type, description)
).first;
}
return elem->second;

58
components/security_apps/prometheus/prometheus_metric_names.h
View File

@@ -71,37 +71,37 @@ convertMetricName(const std::string &original_metric_name)
{"responseBodySizeUponTimeoutMaxSample", "response_body_size_max"},
{"responseBodySizeUponTimeoutMinSample", "response_body_size_min"},
// WaapTelemetrics
{"reservedNgenA", "total_requests_counter"},
{"reservedNgenB", "unique_sources_counter"},
{"reservedNgenC", "requests_blocked_by_force_and_exception_counter"},
{"reservedNgenD", "requests_blocked_by_waf_counter"},
{"reservedNgenE", "requests_blocked_by_open_api_counter"},
{"reservedNgenF", "requests_blocked_by_bot_protection_counter"},
{"reservedNgenG", "requests_threat_level_info_and_no_threat_counter"},
{"reservedNgenH", "requests_threat_level_low_counter"},
{"reservedNgenI", "requests_threat_level_medium_counter"},
{"reservedNgenJ", "requests_threat_level_high_counter"},
{"reservedNgenA_WAAP telemetry", "total_requests_counter"},
{"reservedNgenB_WAAP telemetry", "unique_sources_counter"},
{"reservedNgenC_WAAP telemetry", "requests_blocked_by_force_and_exception_counter"},
{"reservedNgenD_WAAP telemetry", "requests_blocked_by_waf_counter"},
{"reservedNgenE_WAAP telemetry", "requests_blocked_by_open_api_counter"},
{"reservedNgenF_WAAP telemetry", "requests_blocked_by_bot_protection_counter"},
{"reservedNgenG_WAAP telemetry", "requests_threat_level_info_and_no_threat_counter"},
{"reservedNgenH_WAAP telemetry", "requests_threat_level_low_counter"},
{"reservedNgenI_WAAP telemetry", "requests_threat_level_medium_counter"},
{"reservedNgenJ_WAAP telemetry", "requests_threat_level_high_counter"},
// WaapTrafficTelemetrics
{"reservedNgenA", "post_requests_counter"},
{"reservedNgenB", "get_requests_counter"},
{"reservedNgenC", "put_requests_counter"},
{"reservedNgenD", "patch_requests_counter"},
{"reservedNgenE", "delete_requests_counter"},
{"reservedNgenF", "other_requests_counter"},
{"reservedNgenG", "2xx_status_code_responses_counter"},
{"reservedNgenH", "4xx_status_code_responses_counter"},
{"reservedNgenI", "5xx_status_code_responses_counter"},
{"reservedNgenJ", "requests_time_latency_average"},
{"reservedNgenA_WAAP traffic telemetry", "post_requests_counter"},
{"reservedNgenB_WAAP traffic telemetry", "get_requests_counter"},
{"reservedNgenC_WAAP traffic telemetry", "put_requests_counter"},
{"reservedNgenD_WAAP traffic telemetry", "patch_requests_counter"},
{"reservedNgenE_WAAP traffic telemetry", "delete_requests_counter"},
{"reservedNgenF_WAAP traffic telemetry", "other_requests_counter"},
{"reservedNgenG_WAAP traffic telemetry", "status_code_2xx_responses_counter"},
{"reservedNgenH_WAAP traffic telemetry", "status_code_4xx_responses_counter"},
{"reservedNgenI_WAAP traffic telemetry", "status_code_5xx_responses_counter"},
{"reservedNgenJ_WAAP traffic telemetry", "requests_time_latency_average"},
// WaapAttackTypesMetrics
{"reservedNgenA", "sql_injection_attacks_type_counter"},
{"reservedNgenB", "vulnerability_scanning_attacks_type_counter"},
{"reservedNgenC", "path_traversal_attacks_type_counter"},
{"reservedNgenD", "ldap_injection_attacks_type_counter"},
{"reservedNgenE", "evasion_techniques_attacks_type_counter"},
{"reservedNgenF", "remote_code_execution_attacks_type_counter"},
{"reservedNgenG", "xml_extern_entity_attacks_type_counter"},
{"reservedNgenH", "cross_site_scripting_attacks_type_counter"},
{"reservedNgenI", "general_attacks_type_counter"},
{"reservedNgenA_WAAP attack type telemetry", "sql_injection_attacks_type_counter"},
{"reservedNgenB_WAAP attack type telemetry", "vulnerability_scanning_attacks_type_counter"},
{"reservedNgenC_WAAP attack type telemetry", "path_traversal_attacks_type_counter"},
{"reservedNgenD_WAAP attack type telemetry", "ldap_injection_attacks_type_counter"},
{"reservedNgenE_WAAP attack type telemetry", "evasion_techniques_attacks_type_counter"},
{"reservedNgenF_WAAP attack type telemetry", "remote_code_execution_attacks_type_counter"},
{"reservedNgenG_WAAP attack type telemetry", "xml_extern_entity_attacks_type_counter"},
{"reservedNgenH_WAAP attack type telemetry", "cross_site_scripting_attacks_type_counter"},
{"reservedNgenI_WAAP attack type telemetry", "general_attacks_type_counter"},
// AssetsMetric
{"numberOfProtectedApiAssetsSample", "api_assets_counter"},
{"numberOfProtectedWebAppAssetsSample", "web_api_assets_counter"},

30
components/security_apps/rate_limit/rate_limit.cc
View File

@@ -9,7 +9,7 @@
#include "i_mainloop.h"
#include "i_time_get.h"
#include "rate_limit_config.h"
#include "nginx_attachment_common.h"
#include "nano_attachment_common.h"
#include "http_inspection_events.h"
#include "Waf2Util.h"
#include "generic_rulebase/evaluators/asset_eval.h"
@@ -329,7 +329,7 @@ public:
auto asset_location =
Singleton::Consume<I_GeoLocation>::by<RateLimit>()->lookupLocation(maybe_source_ip.unpack());
if (!asset_location.ok()) {
dbgWarning(D_RATE_LIMIT)
dbgDebug(D_RATE_LIMIT)
<< "Rate limit lookup location failed for source: "
<< source_ip
<< ", Error: "
@@ -400,7 +400,19 @@ public:
if (calcRuleAction(rule) == RateLimitAction::PREVENT) {
dbgTrace(D_RATE_LIMIT) << "Received DROP verdict, this request will be blocked by rate limit";
return DROP;
EventVerdict verdict = DROP;
ScopedContext rate_limit_ctx;
rate_limit_ctx.registerValue<GenericConfigId>(AssetMatcher::ctx_key, asset_id);
auto maybe_rate_limit_config = getConfiguration<RateLimitConfig>("rulebase", "rateLimit");
if (maybe_rate_limit_config.ok()) {
const string &web_user_response_id = maybe_rate_limit_config.unpack().getWebUserResponse();
if (!web_user_response_id.empty()) {
verdict.setWebUserResponseByPractice(web_user_response_id);
dbgTrace(D_RATE_LIMIT) << "Set web user response: " << web_user_response_id;
}
}
return verdict;
}
dbgTrace(D_RATE_LIMIT) << "Received DROP in detect mode, will not block.";
@@ -490,7 +502,7 @@ public:
return;
}
auto maybe_rule_by_ctx = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
auto maybe_rule_by_ctx = getConfigurationWithCache<BasicRuleConfig>("rulebase", "rulesConfig");
if (!maybe_rule_by_ctx.ok()) {
dbgWarning(D_RATE_LIMIT)
<< "rule was not found by the given context. Reason: "
@@ -732,9 +744,9 @@ public:
I_EnvDetails* i_env_details = nullptr;
private:
static constexpr auto DROP = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
static constexpr auto DROP = ServiceVerdict::TRAFFIC_VERDICT_DROP;
static constexpr auto ACCEPT = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
static constexpr auto INSPECT = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
RateLimitAction practice_action;
string rate_limit_lua_script_hash;
@@ -761,8 +773,8 @@ RateLimit::~RateLimit() = default;
void
RateLimit::preload()
{
registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
registerExpectedConfigurationWithCache<WaapConfigApplication>("assetId", "WAAP", "WebApplicationSecurity");
registerExpectedConfigurationWithCache<WaapConfigAPI>("assetId", "WAAP", "WebAPISecurity");
registerExpectedConfigFile("waap", Config::ConfigFileType::Policy);
registerExpectedConfiguration<RateLimitConfig>("rulebase", "rateLimit");
registerExpectedConfigFile("accessControlV2", Config::ConfigFileType::Policy);

11
components/security_apps/rate_limit/rate_limit_config.cc
View File

@@ -120,6 +120,17 @@ RateLimitConfig::load(cereal::JSONInputArchive &ar)
dbgWarning(D_RATE_LIMIT) << "Failed to load single Rate Limit JSON config. Error: " << e.what();
ar.setNextName(nullptr);
}
try {
string rate_limit_response;
ar(cereal::make_nvp("webUserResponseId", rate_limit_response));
setWebUserResponse(rate_limit_response);
} catch(const std::exception& e) {
dbgDebug(D_RATE_LIMIT)
<< "No webUserResponseId defined for Rate Limit config. Using default. Error: "
<< e.what();
ar.setNextName(nullptr);
}
}
RateLimitRule

27
components/security_apps/waap/include/i_serialize.h
View File

@@ -12,6 +12,7 @@
// limitations under the License.
#pragma once
#include <chrono>
#include <fstream>
#include "i_time_get.h"
@@ -19,12 +20,18 @@
#include "i_messaging.h"
#include "i_mainloop.h"
#include "i_agent_details.h"
#include "compression_utils.h"
#include <sstream>
#include <vector>
static const uint max_send_obj_retries = 3;
static const std::chrono::microseconds wait_next_attempt(5000000);
USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
// Forward declarations
class WaapComponent;
class RestGetFile : public ClientRest
{
public:
@@ -71,6 +78,7 @@ public:
virtual void postProcessedData() = 0;
virtual void pullProcessedData(const std::vector<std::string> &files) = 0;
virtual void updateState(const std::vector<std::string> &files) = 0;
virtual Maybe<std::string> getRemoteStateFilePath() const { return genError("No remote state file path defined"); }
};
class I_Backup {
@@ -86,7 +94,7 @@ class SerializeToFileBase :
public I_Serializable
{
public:
SerializeToFileBase(std::string filePath);
SerializeToFileBase(const std::string &filePath);
virtual ~SerializeToFileBase();
virtual void saveData();
@@ -96,13 +104,13 @@ protected:
// saved file name for testing
std::string m_filePath;
private:
void loadFromFile(std::string filePath);
void loadFromFile(const std::string &filePath); // Updated to match implementation
};
class SerializeToFilePeriodically : public SerializeToFileBase
{
public:
SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, std::string filePath);
SerializeToFilePeriodically(std::chrono::seconds pollingIntervals, const std::string &filePath);
virtual ~SerializeToFilePeriodically();
void setInterval(std::chrono::seconds newInterval);
@@ -131,7 +139,7 @@ public:
virtual void restore();
virtual void syncWorker();
bool shouldNotSync() const;
void setInterval(std::chrono::seconds newInterval);
std::chrono::seconds getIntervalDuration() const;
void setRemoteSyncEnabled(bool enabled);
@@ -143,7 +151,7 @@ protected:
std::string getUri();
size_t getIntervalsCount();
void incrementIntervalsCount();
bool isBase();
bool isBase() const;
template<typename T>
bool sendObject(T &obj, HTTPMethod method, std::string uri)
@@ -242,7 +250,7 @@ protected:
}
template<typename T>
bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, std::string uri)
bool sendNoReplyObjectWithRetry(T &obj, HTTPMethod method, const std::string &uri)
{
I_MainLoop *mainloop= Singleton::Consume<I_MainLoop>::by<WaapComponent>();
for (uint i = 0; i < max_send_obj_retries; i++)
@@ -270,10 +278,16 @@ protected:
private:
bool localSyncAndProcess();
void updateStateFromRemoteService();
Maybe<std::string> getStateTimestampByListing();
bool checkAndUpdateStateTimestamp(const std::string& currentStateTimestamp);
RemoteFilesList getProcessedFilesList();
RemoteFilesList getRemoteProcessedFilesList();
std::string getLearningHost();
std::string getSharedStorageHost();
std::string getStateTimestampPath();
Maybe<std::string> getStateTimestamp();
Maybe<void> updateStateFromRemoteFile();
bool shouldSendSyncNotification() const;
I_MainLoop* m_pMainLoop;
std::chrono::microseconds m_waitForSync;
@@ -287,3 +301,4 @@ private:
Maybe<std::string> m_shared_storage_host;
Maybe<std::string> m_learning_host;
};

2
components/security_apps/waap/include/i_transaction.h
View File

@@ -89,7 +89,7 @@ public:
virtual double getOtherModelScore() const = 0;
virtual const std::vector<double> getScoreArray() const = 0;
virtual Waap::CSRF::State& getCsrfState() = 0;
virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;
virtual ServiceVerdict getUserLimitVerdict() = 0;
virtual const std::string getUserLimitVerdictStr() const = 0;
virtual const std::string getViolatedUserLimitTypeStr() const = 0;
virtual void checkShouldInject() = 0;

8
components/security_apps/waap/include/i_waapConfig.h
View File

@@ -40,6 +40,13 @@ enum class AttackMitigationMode
PREVENT,
UNKNOWN
};
enum class WaapConfigType {
Unknown = 0,
Application,
API
};
class IWaapConfig {
public:
virtual const std::string& get_AssetId() const = 0;
@@ -65,6 +72,7 @@ public:
virtual const std::shared_ptr<Waap::RateLimiting::Policy>& get_ErrorLimitingPolicy() const = 0;
virtual const std::shared_ptr<Waap::SecurityHeaders::Policy>& get_SecurityHeadersPolicy() const = 0;
virtual const std::shared_ptr<Waap::UserLimits::Policy>& get_UserLimitsPolicy() const = 0;
virtual WaapConfigType getType() const = 0;
virtual void printMe(std::ostream& os) const = 0;
};

4
components/security_apps/waap/reputation/reputation_features_agg.cc
View File

@@ -30,7 +30,7 @@ template <typename EventType>
class DefaultListener : public Listener<EventType>
{
public:
DefaultListener(EventVerdict defaultVerdict = EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT))
DefaultListener(EventVerdict defaultVerdict = EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT))
:
m_default_verdict(defaultVerdict)
{}
@@ -58,7 +58,7 @@ class ReputationFeaturesAgg::Impl
public:
Impl()
:
DefaultListener<ResponseCodeEvent>(EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT)),
DefaultListener<ResponseCodeEvent>(EventVerdict(ServiceVerdict::TRAFFIC_VERDICT_IRRELEVANT)),
m_agg_entries()
{
}

8
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -93,6 +93,14 @@ add_library(waap_clib
ParserBinaryFile.cc
RegexComparator.cc
RequestsMonitor.cc
WaapHyperscanEngine.cc
buffered_compressed_stream.cc
UnifiedIndicatorsContainer.cc
)
add_library(
UnifiedIndicatorsContainer
UnifiedIndicatorsContainer.cc
)
add_definitions("-Wno-unused-function")

526
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

@@ -18,6 +18,7 @@
#include "ConfidenceFile.h"
#include "i_agent_details.h"
#include "i_mainloop.h"
#include "buffered_compressed_stream.h"
#include <sys/stat.h>
#include <math.h>
#include <dirent.h> // For DIR, opendir, readdir, closedir
@@ -32,13 +33,15 @@ USE_DEBUG_FLAG(D_WAAP);
#define BUSY_WAIT_TIME chrono::microseconds(100000) // 0.1 seconds
#define WAIT_LIMIT 10
#define BENIGN_PARAM_FACTOR 2
#define MAX_TRACKING_KEYS 1000 // Maximum number of keys to track
double logn(double x, double n)
{
return log(x) / log(n);
}
ConfidenceCalculator::ConfidenceCalculator(size_t minSources,
ConfidenceCalculator::ConfidenceCalculator(
size_t minSources,
size_t minIntervals,
chrono::minutes intervalDuration,
double ratioThreshold,
@@ -65,12 +68,17 @@ ConfidenceCalculator::ConfidenceCalculator(size_t minSources,
m_ignoreSources(ignoreSrc),
m_tuning(tuning),
m_estimated_memory_usage(0),
m_post_index(0),
m_mainLoop(Singleton::Consume<I_MainLoop>::by<WaapComponent>()),
m_routineId(0),
m_filesToRemove()
m_filesToRemove(),
m_indicator_tracking_keys(),
m_tracking_keys_received(false)
{
restore();
extractLowConfidenceKeys(m_confidence_level);
// Start asynchronous deletion of existing carry-on data files
garbageCollector();
}
@@ -103,10 +111,11 @@ void ConfidenceCalculator::hardReset()
m_estimated_memory_usage = 0;
m_confidence_level.clear();
m_confident_sets.clear();
m_indicator_tracking_keys.clear();
m_tracking_keys_received = false;
remove(m_filePath.c_str());
}
void ConfidenceCalculator::reset()
{
if (m_time_window_logger) {
@@ -138,15 +147,16 @@ bool ConfidenceCalculator::reset(ConfidenceCalculatorParams& params)
class WindowLogPost : public RestGetFile
{
public:
WindowLogPost(ConfidenceCalculator::KeyValSourcesLogger& _window_logger)
: window_logger(_window_logger)
WindowLogPost(std::shared_ptr<ConfidenceCalculator::KeyValSourcesLogger> _window_logger_ptr)
{
// Initialize the RestParam with a reference to the shared data
// do not copy the shared pointer, but move it to avoid copying - do not use the ptr after it
window_logger = std::move(*_window_logger_ptr);
}
~WindowLogPost()
{
window_logger.get().clear();
window_logger.get().rehash(0);
// Container will be automatically cleaned up when RestParam goes out of scope
}
private:
@@ -169,73 +179,9 @@ private:
S2C_PARAM(ConfidenceCalculator::KeyValSourcesLogger, window_logger)
};
// Function to handle compression
void compressDataWrapper(const string& uncompressed_data, size_t chunk_size, vector<unsigned char>& compressed_data) {
auto compression_stream = initCompressionStream();
size_t offset = 0;
while (offset < uncompressed_data.size()) {
size_t current_chunk_size = std::min(chunk_size, uncompressed_data.size() - offset);
bool is_last = (offset + current_chunk_size >= uncompressed_data.size());
CompressionResult chunk_res = compressData(
compression_stream,
CompressionType::GZIP,
static_cast<uint32_t>(current_chunk_size),
reinterpret_cast<const unsigned char*>(uncompressed_data.c_str() + offset),
is_last ? 1 : 0
);
if (!chunk_res.ok) {
finiCompressionStream(compression_stream);
throw runtime_error("Compression failed");
}
if (chunk_res.output && chunk_res.num_output_bytes > 0) {
compressed_data.insert(
compressed_data.end(),
chunk_res.output,
chunk_res.output + chunk_res.num_output_bytes
);
free(chunk_res.output);
}
offset += current_chunk_size;
}
finiCompressionStream(compression_stream);
}
Maybe<void> ConfidenceCalculator::writeToFile(const string& path, const vector<unsigned char>& data)
{
ofstream file(path, ios::binary);
if (!file.is_open()) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << path
<< ", errno: " << errno << ", strerror: " << strerror(errno);
return genError("Failed to open file");
}
// Write compressed data to file in chunks to avoid large memory usage
const uint CHUNK_SIZE = getProfileAgentSettingWithDefault<uint>(
64 * 1024, // 64 KiB
"appsecLearningSettings.writeChunkSize"
);
size_t offset = 0;
while (offset < data.size()) {
size_t current_chunk_size = min(static_cast<size_t>(CHUNK_SIZE), data.size() - offset);
file.write(reinterpret_cast<const char *>(data.data()) + offset, current_chunk_size);
offset += current_chunk_size;
m_mainLoop->yield(false);
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Write progress: " << offset << "/" << data.size()
<< " bytes (" << (offset * 100 / data.size()) << "%) - yielded";
}
file.close();
return Maybe<void>();
}
void ConfidenceCalculator::saveTimeWindowLogger()
{
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Saving the time window logger to: " << m_path_to_backup;
if (m_path_to_backup != "") // remove old file from exceed memory cap flow
{
remove(m_path_to_backup.c_str());
@@ -257,69 +203,34 @@ void ConfidenceCalculator::saveTimeWindowLogger()
m_path_to_backup = temp_filename;
stringstream ss;
{
cereal::JSONOutputArchive archive(ss);
archive(cereal::make_nvp("logger", *m_time_window_logger));
}
m_mainLoop->yield(false);
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "JSON serialized, size: " << ss.str().size() << " bytes";
string data = ss.str();
const uint COMPRESSED_CHUNK_SIZE = getProfileAgentSettingWithDefault<uint>(
16 * 1024, // 16KB
"appsecLearningSettings.compressionChunkSize"
);
auto compression_stream = initCompressionStream();
size_t offset = 0;
vector<unsigned char> compressed_data;
bool ok = true;
while (offset < data.size()) {
size_t chunk_size = min(static_cast<size_t>(COMPRESSED_CHUNK_SIZE), data.size() - offset);
bool is_last = (offset + chunk_size >= data.size());
CompressionResult chunk_res = compressData(
compression_stream,
CompressionType::GZIP,
static_cast<uint32_t>(chunk_size),
reinterpret_cast<const unsigned char *>(data.c_str() + offset),
is_last ? 1 : 0
);
if (!chunk_res.ok) {
ok = false;
break;
}
if (chunk_res.output && chunk_res.num_output_bytes > 0) {
compressed_data.insert(
compressed_data.end(),
chunk_res.output,
chunk_res.output + chunk_res.num_output_bytes
);
free(chunk_res.output);
}
offset += chunk_size;
m_mainLoop->yield(false);
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Compression progress: " << offset << "/" << data.size()
<< " bytes processed (" << (offset * 100 / data.size()) << "%) - yielded";
}
finiCompressionStream(compression_stream);
if (!ok) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress data";
try {
ofstream file(m_path_to_backup, ios::binary);
if (!file.is_open()) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << m_path_to_backup
<< ", errno: " << errno << ", strerror: " << strerror(errno);
m_time_window_logger_backup = m_time_window_logger;
return;
}
auto maybeError = writeToFile(m_path_to_backup, compressed_data);
if (!maybeError.ok()) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to write the backup file: " << m_path_to_backup;
BufferedCompressedOutputStream compressed_out(file);
{
cereal::JSONOutputArchive archive(compressed_out);
archive(cereal::make_nvp("logger", *m_time_window_logger));
}
compressed_out.close();
file.close();
m_mainLoop->yield(false);
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "JSON serialized and compressed to file: " << m_path_to_backup;
}
catch (const std::exception &e) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize and compress data: " << e.what();
m_time_window_logger_backup = m_time_window_logger;
m_path_to_backup = "";
} else {
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Finished writing the backup file: " << m_path_to_backup;
return;
}
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Finished writing the backup file: " << m_path_to_backup;
}
shared_ptr<ConfidenceCalculator::KeyValSourcesLogger> ConfidenceCalculator::loadTimeWindowLogger()
@@ -330,107 +241,41 @@ shared_ptr<ConfidenceCalculator::KeyValSourcesLogger> ConfidenceCalculator::load
return nullptr;
}
ifstream file(m_path_to_backup, ios::binary);
ifstream file(m_path_to_backup);
if (!file.is_open()) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << m_path_to_backup
<< ", errno: " << errno << ", strerror: " << strerror(errno);
return nullptr;
}
stringstream buffer;
// Read the file in chunks to avoid large memory usage
const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault<uint>(
16 * 1024,
"appsecLearningSettings.readChunkSize"); // 16 KiB
vector<char> chunk(READ_CHUNK_SIZE);
size_t chunk_size = static_cast<size_t>(READ_CHUNK_SIZE);
size_t total_bytes_read = 0;
size_t chunks_read = 0;
while (file.peek() != EOF) {
file.read(chunk.data(), chunk_size);
streamsize bytesRead = file.gcount();
if (bytesRead > 0) {
buffer.write(chunk.data(), bytesRead);
total_bytes_read += bytesRead;
chunks_read++;
}
m_mainLoop->yield(false);
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Read chunk " << chunks_read
<< " (" << total_bytes_read << " bytes total) - yielded";
}
file.close();
remove(m_path_to_backup.c_str());
m_path_to_backup = "";
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Completed reading " << total_bytes_read
<< " bytes in " << chunks_read << " chunks";
string compressed_data = buffer.str();
auto compression_stream = initCompressionStream();
chunk_size = static_cast<size_t>(
getProfileAgentSettingWithDefault<uint>(
32 * 1024, // 32KiB
"appsecLearningSettings.compressionChunkSize"
)
);
size_t offset = 0;
string decompressed_data;
while (offset < compressed_data.size()) {
size_t current_chunk_size = min(chunk_size, compressed_data.size() - offset);
DecompressionResult res = decompressData(
compression_stream,
current_chunk_size,
reinterpret_cast<const unsigned char *>(compressed_data.c_str() + offset)
);
if (!res.ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress data from file: " << m_path_to_backup;
finiCompressionStream(compression_stream);
return nullptr;
}
decompressed_data.append(reinterpret_cast<const char *>(res.output), res.num_output_bytes);
free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
offset += current_chunk_size;
// Yield control after processing each chunk
m_mainLoop->yield(false);
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompression progress: " << offset << "/" << compressed_data.size()
<< " bytes (" << (offset * 100 / compressed_data.size()) << "%) - yielded";
}
finiCompressionStream(compression_stream);
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Decompressed data size: " << decompressed_data.size()
<< " bytes (compression ratio: " << (float)decompressed_data.size() / compressed_data.size() << "x)";
stringstream decompressed_stream(decompressed_data);
auto window_logger = make_shared<KeyValSourcesLogger>();
try {
cereal::JSONInputArchive archive(decompressed_stream);
BufferedCompressedInputStream compressed_in(file);
cereal::JSONInputArchive archive(compressed_in);
archive(cereal::make_nvp("logger", *window_logger));
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Successfully deserialized logger from JSON";
} catch (cereal::Exception &e) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the time window logger: " << e.what();
file.close();
return nullptr;
}
file.close();
return window_logger;
}
bool ConfidenceCalculator::postData()
{
if (m_time_window_logger->empty())
{
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping";
return true; // Nothing to post
}
saveTimeWindowLogger();
m_mainLoop->yield(false);
WindowLogPost currentWindow(*m_time_window_logger);
WindowLogPost currentWindow(m_time_window_logger);
m_mainLoop->yield(false);
m_time_window_logger = make_shared<ConfidenceCalculator::KeyValSourcesLogger>();
string url = getPostDataUrl() + to_string(m_post_index++);
@@ -465,7 +310,7 @@ void ConfidenceCalculator::pullData(const vector<string>& files)
string url = getPostDataUrl();
string sentFile = url.erase(0, strlen("/storage/waap/"));
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "pulling files, skipping: " << sentFile;
for (auto file : files)
for (const auto &file : files) // Use const reference to avoid copying
{
if (file == sentFile)
{
@@ -483,13 +328,13 @@ void ConfidenceCalculator::pullData(const vector<string>& files)
}
KeyValSourcesLogger remoteLogger = getWindow.getWindowLogger().unpack();
for (auto& log : remoteLogger)
for (const auto &log : remoteLogger) // Use const reference
{
const string &key = log.first;
for (auto& entry : log.second)
for (const auto &entry : log.second) // Use const reference
{
const string &value = entry.first;
for (auto & source : entry.second)
for (const auto &source : entry.second) // Use const reference
{
(*m_time_window_logger_backup)[key][value].insert(source);
}
@@ -504,7 +349,7 @@ void ConfidenceCalculator::processData()
m_post_index = 0;
if (m_time_window_logger_backup == nullptr || m_time_window_logger_backup->empty())
{
if (m_path_to_backup != "")
if (!m_path_to_backup.empty())
{
m_time_window_logger_backup = loadTimeWindowLogger();
m_mainLoop->yield(false);
@@ -523,10 +368,10 @@ void ConfidenceCalculator::processData()
// clear temp data
m_time_window_logger_backup->clear();
m_time_window_logger_backup.reset();
if (m_path_to_backup != "")
if (!m_path_to_backup.empty())
{
remove(m_path_to_backup.c_str());
m_path_to_backup = "";
m_path_to_backup.clear();
}
}
@@ -539,11 +384,11 @@ void ConfidenceCalculator::updateState(const vector<string>& files)
m_time_window_logger_backup->clear();
m_time_window_logger_backup.reset();
}
if (m_path_to_backup != "")
{
remove(m_path_to_backup.c_str());
m_path_to_backup = "";
}
Maybe<string> ConfidenceCalculator::getRemoteStateFilePath() const
{
return m_remotePath + "/remote/confidence.data";
}
void ConfidenceCalculator::pullProcessedData(const vector<string> &files)
@@ -552,10 +397,10 @@ void ConfidenceCalculator::pullProcessedData(const vector<string>& files)
m_post_index = 0;
bool is_first_pull = true;
bool is_ok = false;
for (auto file : files)
for (const auto &file : files) // Use const reference to avoid copying
{
ConfidenceFileDecryptor getConfFile;
bool res = sendObjectWithRetry(getConfFile,
bool res = sendObject(getConfFile,
HTTPMethod::GET,
getUri() + "/" + file);
is_ok |= res;
@@ -564,15 +409,32 @@ void ConfidenceCalculator::pullProcessedData(const vector<string>& files)
mergeFromRemote(getConfFile.getConfidenceSet().unpackMove(), is_first_pull);
is_first_pull = false;
}
if (res && getConfFile.getTrackingKeys().ok())
{
auto trackingKeys = getConfFile.getTrackingKeys().unpackMove();
m_indicator_tracking_keys = unordered_set<string>(trackingKeys.begin(), trackingKeys.end());
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Received tracking keys: " << m_indicator_tracking_keys.size();
m_tracking_keys_received = true;
}
if (res && getConfFile.getConfidenceLevels().ok())
{
// write to disk the confidence levels
saveConfidenceLevels(getConfFile.getConfidenceLevels());
saveConfidenceLevels(getConfFile.getConfidenceLevels().unpackMove());
}
else
{
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get tracking keys from file: " << file;
}
}
// is_ok = false -> no file was downloaded and merged
if (!is_ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to get the remote state";
return;
}
if (m_path_to_backup != "")
{
remove(m_path_to_backup.c_str());
m_path_to_backup = "";
}
}
@@ -939,6 +801,8 @@ size_t ConfidenceCalculator::getLastConfidenceUpdate()
void ConfidenceCalculator::log(const Key &key, const Val &value, const string &source)
{
// Only track in time window logger if we should track this parameter
if (shouldTrackParameter(key, value)) {
auto &sources_set = (*m_time_window_logger)[key][value];
auto result = sources_set.insert(source);
if (result.second) {
@@ -958,6 +822,7 @@ void ConfidenceCalculator::log(const Key &key, const Val &value, const string &s
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "memory usage: " << m_estimated_memory_usage <<
"/" << m_params.maxMemoryUsage;
}
if (value != m_null_obj)
{
@@ -987,6 +852,64 @@ void ConfidenceCalculator::logSourceHit(const Key &key, const string &source)
log(key, m_null_obj, source);
}
void ConfidenceCalculator::setIndicatorTrackingKeys(const std::vector<std::string>& keys)
{
m_indicator_tracking_keys.clear();
for (const auto& key : keys) {
m_indicator_tracking_keys.insert(key);
}
m_tracking_keys_received = true;
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - received " << keys.size() << " indicator tracking keys from service";
}
void ConfidenceCalculator::markKeyAsConfident(const Key &key)
{
// This method is kept for API compatibility but doesn't affect conditional tracking
// The confidence set is managed independently and not altered by conditional tracking feature
// Add the key to confident sets if not already present
if (m_confident_sets.find(key) == m_confident_sets.end()) {
size_t current_time = chrono::duration_cast<chrono::seconds>(
Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime()
).count();
m_confident_sets[key] = std::make_pair(ValuesSet(), current_time);
}
}
bool ConfidenceCalculator::shouldTrackParameter(const Key &key, const Val &value)
{
// For backward compatibility: if tracking list hasn't been received from service yet, track all
if (!m_tracking_keys_received) {
return true;
}
// Should NOT track if key->value combination is already in confidence set
if (is_confident(key, value)) {
return !m_params.learnPermanently; // If learnPermanently is true, we don't track confident values
}
if (!m_params.learnPermanently && m_confident_sets.find(key) != m_confident_sets.end()) {
m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list
return true;
}
// Should NOT track if key is not in tracking list AND value is null obj
bool keyInTrackingList = (m_indicator_tracking_keys.find(key) != m_indicator_tracking_keys.end());
if (!keyInTrackingList && value == m_null_obj) {
return false;
}
if (!keyInTrackingList) {
// If tracking list is full, do not track this key
m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - tracking key: " << key << ", value: " << value;
}
// Should track if:
// 1. Key is in tracking list, OR
// 2. New value (not null obj) AND key->value not in confidence set (already checked above)
return keyInTrackingList || (value != m_null_obj);
}
void ConfidenceCalculator::removeBadSources(SourcesSet &sources, const vector<string>* badSources)
{
if (badSources == nullptr)
@@ -1026,86 +949,28 @@ void ConfidenceCalculator::loadConfidenceLevels()
string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount() - 1) % 2) + ".gz";
ifstream file(file_path, ios::binary);
if (!file.is_open()) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open the file: " << file_path;
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open the file: " << file_path <<
", errno: " << errno << ", strerror: " << strerror(errno);
return;
}
stringstream buffer;
// Read the file in chunks to avoid large memory usage
const uint READ_CHUNK_SIZE = getProfileAgentSettingWithDefault<uint>(
16 * 1024,
"appsecLearningSettings.readChunkSize"); // 16 KiB
vector<char> chunk(READ_CHUNK_SIZE);
size_t chunk_size = static_cast<size_t>(READ_CHUNK_SIZE);
while (file.peek() != EOF) {
file.read(chunk.data(), chunk_size);
streamsize bytesRead = file.gcount();
if (bytesRead > 0) {
buffer.write(chunk.data(), bytesRead);
}
m_mainLoop->yield(false);
}
file.close();
string compressed_data = buffer.str();
auto compression_stream = initCompressionStream();
DecompressionResult res;
res.ok = true;
res.output = nullptr;
res.num_output_bytes = 0;
size_t offset = 0;
const size_t CHUNK_SIZE = static_cast<size_t>(
getProfileAgentSettingWithDefault<uint>(
16 * 1024, // 16KiB
"appsecLearningSettings.compressionChunkSize"
)
);
vector<char> decompressed_data_vec;
while (offset < compressed_data.size()) {
size_t current_chunk_size = min(CHUNK_SIZE, compressed_data.size() - offset);
DecompressionResult chunk_res = decompressData(
compression_stream,
current_chunk_size,
reinterpret_cast<const unsigned char *>(compressed_data.c_str() + offset)
);
if (!chunk_res.ok) {
res.ok = false;
break;
}
if (chunk_res.output && chunk_res.num_output_bytes > 0) {
// Append directly to the vector to avoid extra copies
size_t old_size = decompressed_data_vec.size();
decompressed_data_vec.resize(old_size + chunk_res.num_output_bytes);
memcpy(decompressed_data_vec.data() + old_size, chunk_res.output, chunk_res.num_output_bytes);
free(chunk_res.output);
}
offset += current_chunk_size;
m_mainLoop->yield(false);
}
finiCompressionStream(compression_stream);
if (!res.ok) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to decompress the confidence levels data.";
return;
}
string decompressed_data(decompressed_data_vec.begin(), decompressed_data_vec.end());
stringstream decompressed_stream(decompressed_data);
try {
BufferedCompressedInputStream decompressed_stream(file);
cereal::JSONInputArchive archive(decompressed_stream);
archive(cereal::make_nvp("confidence_levels", m_confidence_level));
} catch (runtime_error &e) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk: " << e.what();
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR)
<< "Failed to load the confidence levels, owner: "
<< m_owner << ", error: " << e.what();
}
file.close();
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - loaded the confidence levels from disk, latest index: " << m_latest_index <<
", intervals count: " << getIntervalsCount();
m_mainLoop->yield(false);
if (m_confidence_level.empty())
{
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to load the confidence levels from disk";
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "confidence levels are empty, owner: " << m_owner <<
", file: " << file_path;
}
}
@@ -1117,34 +982,42 @@ void ConfidenceCalculator::saveConfidenceLevels()
void ConfidenceCalculator::saveConfidenceLevels(Maybe<ConfidenceCalculator::ConfidenceLevels> confidenceLevels)
{
string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount()) % 2) + ".gz";
stringstream serialized_data;
try {
if (!confidenceLevels.ok())
{
cereal::JSONOutputArchive archive(serialized_data);
// if confidence levels are not available, use the current confidence level
extractLowConfidenceKeys(m_confidence_level);
}
else
{
// if confidence levels are empty, use the current confidence level
extractLowConfidenceKeys(confidenceLevels.unpackMove());
}
string file_path = m_filePath + ".levels." + to_string((m_latest_index + getIntervalsCount()) % 2) + ".gz";
ofstream file(file_path, ios::binary);
if (!file.is_open()) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to open file: " << file_path
<< ", errno: " << errno << ", strerror: " << strerror(errno);
return;
}
{
try {
BufferedCompressedOutputStream compressed_out(file);
cereal::JSONOutputArchive archive(compressed_out);
if (confidenceLevels.ok()) {
archive(cereal::make_nvp("confidence_levels", confidenceLevels.unpackMove()));
} else {
}
else
{
archive(cereal::make_nvp("confidence_levels", m_confidence_level));
}
}
} catch (runtime_error &e) {
file.close();
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to serialize the confidence levels: " << e.what();
return;
}
m_mainLoop->yield(false);
string uncompressed_data = serialized_data.str();
const size_t CHUNK_SIZE = 16 * 1024; // 16 KiB
vector<unsigned char> compressed_data;
try {
compressDataWrapper(uncompressed_data, CHUNK_SIZE, compressed_data);
writeToFile(file_path, compressed_data);
} catch (const runtime_error& e) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to compress the confidence levels data: " << e.what();
}
file.close();
m_mainLoop->yield(false);
m_confidence_level.clear();
@@ -1176,7 +1049,6 @@ void ConfidenceCalculator::calculateInterval()
}
int itr = 0;
for (auto sourcesCtrItr : *m_time_window_logger_backup)
{
if (++itr % 20 == 0) {
@@ -1296,6 +1168,7 @@ void ConfidenceCalculator::garbageCollector()
I_MainLoop *mainLoop = Singleton::Consume<I_MainLoop>::by<WaapComponent>();
mainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::Offline,
// LCOV_EXCL_START
[this, mainLoop]() {
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - running garbage collection of carry-on data files";
@@ -1406,6 +1279,47 @@ void ConfidenceCalculator::garbageCollector()
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - finished garbage collection.";
},
// LCOV_EXCL_STOP
"ConfidenceCalculator garbage collection"
);
}
void ConfidenceCalculator::extractLowConfidenceKeys(const ConfidenceLevels& confidence_levels)
{
const double LOW_CONFIDENCE_THRESHOLD = 100.0;
size_t keys_added = 0;
m_tracking_keys_received = true; // Ensure tracking keys are considered received
for (const auto& keyEntry : confidence_levels) {
const std::string& key = keyEntry.first;
const auto& valueConfidenceMap = keyEntry.second;
if (!m_params.learnPermanently) {
m_indicator_tracking_keys.insert(key); // Ensure the key is in the tracking list
continue;
}
// Check if any value for this key has confidence level below threshold
bool hasLowConfidence = false;
for (const auto& valueEntry : valueConfidenceMap) {
if (valueEntry.second < LOW_CONFIDENCE_THRESHOLD) {
hasLowConfidence = true;
break;
}
}
// If key has low confidence values, add it to tracking keys
if (hasLowConfidence) {
auto result = m_indicator_tracking_keys.insert(key);
if (result.second) { // Key was newly inserted
keys_added++;
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - added key '" << key << "' to tracking list (has confidence < " <<
LOW_CONFIDENCE_THRESHOLD << ")";
}
}
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Owner: " << m_owner <<
" - added " << keys_added << " keys with low confidence values to tracking list";
}

16
components/security_apps/waap/waap_clib/ConfidenceCalculator.h
View File

@@ -66,6 +66,8 @@ struct ConfidenceCalculatorParams
friend std::ostream & operator<<(std::ostream &os, const ConfidenceCalculatorParams &ccp);
};
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
class ConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase
{
public:
@@ -111,12 +113,11 @@ public:
virtual void postProcessedData();
virtual void pullProcessedData(const std::vector<std::string> &files);
virtual void updateState(const std::vector<std::string> &files);
virtual Maybe<std::string> getRemoteStateFilePath() const override;
virtual void serialize(std::ostream &stream);
virtual void deserialize(std::istream &stream);
Maybe<void> writeToFile(const std::string& path, const std::vector<unsigned char>& data);
void mergeFromRemote(const ConfidenceSet &remote_confidence_set, bool is_first_pull);
bool is_confident(const Key &key, const Val &value) const;
@@ -135,6 +136,13 @@ public:
static void mergeConfidenceSets(ConfidenceSet &confidence_set,
const ConfidenceSet &confidence_set_to_merge,
size_t &last_indicators_update);
// Methods for conditional parameter tracking (INXT-46771)
void setIndicatorTrackingKeys(const std::vector<std::string>& keys);
void markKeyAsConfident(const Key &key);
bool shouldTrackParameter(const Key &key, const Val &value);
void extractLowConfidenceKeys(const ConfidenceLevels& confidence_levels);
private:
void loadVer0(cereal::JSONInputArchive &archive);
void loadVer1(cereal::JSONInputArchive &archive);
@@ -178,4 +186,8 @@ private:
I_MainLoop *m_mainLoop;
I_MainLoop::RoutineID m_routineId;
std::vector<std::string> m_filesToRemove;
// Additional member variables for conditional tracking (INXT-46771)
std::unordered_set<std::string> m_indicator_tracking_keys;
bool m_tracking_keys_received;
};

6
components/security_apps/waap/waap_clib/ConfidenceFile.cc
View File

@@ -29,6 +29,12 @@ Maybe<ConfidenceCalculator::ConfidenceLevels> ConfidenceFileDecryptor::getConfid
return genError("failed to get confidence levels");
}
Maybe<std::vector<std::string>> ConfidenceFileDecryptor::getTrackingKeys() const
{
if (!tracking_keys.get().empty()) return tracking_keys.get();
return genError("failed to get tracking keys");
}
ConfidenceFileEncryptor::ConfidenceFileEncryptor(const ConfidenceCalculator::ConfidenceSet &_confidence_set,
const ConfidenceCalculator::ConfidenceLevels &_confidence_levels) :

3
components/security_apps/waap/waap_clib/ConfidenceFile.h
View File

@@ -23,10 +23,13 @@ public:
getConfidenceSet() const;
Maybe<ConfidenceCalculator::ConfidenceLevels>
getConfidenceLevels() const;
Maybe<std::vector<std::string>>
getTrackingKeys() const;
private:
S2C_PARAM(ConfidenceCalculator::ConfidenceSet, confidence_set);
S2C_OPTIONAL_PARAM(ConfidenceCalculator::ConfidenceLevels, confidence_levels);
S2C_OPTIONAL_PARAM(std::vector<std::string>, tracking_keys);
};
class ConfidenceFileEncryptor : public RestGetFile

4
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -414,8 +414,10 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
if (valueStats.canSplitPipe || valueStats.canSplitSemicolon) {
std::string key = IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction);
if (m_pWaapAssetState->m_filtersMngr != nullptr) {
m_pWaapAssetState->m_filtersMngr->pushSample(key, cur_val, m_pTransaction);
}
}
// Detect and decode UTF-16 data
Waap::Util::decodeUtf16Value(valueStats, cur_val);
@@ -1090,7 +1092,7 @@ DeepParser::createInternalParser(
int offset = -1;
auto pWaapAssetState = m_pTransaction->getAssetState();
std::shared_ptr<Signatures> signatures = m_pWaapAssetState->getSignatures();
if (pWaapAssetState != nullptr) {
if (pWaapAssetState != nullptr && pWaapAssetState->m_filtersMngr != nullptr) {
// Find out learned type
std::set<std::string> paramTypes = pWaapAssetState->m_filtersMngr->getParameterTypes(
IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction)

9
components/security_apps/waap/waap_clib/IndicatorsFilterBase.cc
View File

@@ -14,6 +14,7 @@
#include "i_indicatorsFilter.h"
#include "IndicatorsFilterBase.h"
#include "Waf2Engine.h"
#include "maybe_res.h"
IndicatorFilterBase::IndicatorFilterBase(const std::string& confidence_path,
const std::string& trusted_path,
@@ -113,12 +114,12 @@ bool IndicatorFilterBase::isTrustedSourceOfType(const std::string& source,
}
std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction)
Maybe<std::string> IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction)
{
if (m_policy == nullptr)
{
dbgTrace(D_WAAP) << "Policy for trusted sources is not set";
return "";
return genError<std::string>("Policy for trusted sources is not set");
}
auto trustedTypes = m_policy->getTrustedTypes();
std::string xFwdVal;
@@ -171,7 +172,7 @@ std::string IndicatorFilterBase::getTrustedSource(IWaf2Transaction* pTransaction
}
}
return "";
return genError<std::string>("No trusted source found");
}
void IndicatorFilterBase::registerKeyword(const std::string& key,
@@ -191,3 +192,5 @@ void IndicatorFilterBase::registerKeyword(const std::string& key,
m_trusted_confidence_calc.log(key, keyword, trusted_src);
}
}
//TODO: update states function (use getRemoteStateFilePath)

3
components/security_apps/waap/waap_clib/IndicatorsFilterBase.h
View File

@@ -19,6 +19,7 @@
#include "TrustedSourcesConfidence.h"
#include "ConfidenceCalculator.h"
#include "TuningDecisions.h"
#include "maybe_res.h"
class IndicatorFilterBase : public I_IndicatorsFilter
{
@@ -39,8 +40,8 @@ public:
bool setTrustedSrcParameter(std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter> policy);
void reset();
Maybe<std::string> getTrustedSource(IWaf2Transaction* pTransaction);
protected:
std::string getTrustedSource(IWaf2Transaction* pTransaction);
void registerKeyword(const std::string& key,
const std::string& keyword,
const std::string& source,

279
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc
View File

@@ -19,44 +19,168 @@
#include "FpMitigation.h"
#include "Waf2Engine.h"
#include "WaapKeywords.h"
#include "config.h"
IndicatorsFiltersManager::IndicatorsFiltersManager(const std::string& remotePath, const std::string &assetId,
USE_DEBUG_FLAG(D_WAAP_LEARN);
static constexpr int DEFAULT_SOURCES_LIMIT = 1000;
using namespace std;
// // Helper class for posting unified indicators
// class UnifiedIndicatorsLogPost : public RestGetFile {
// public:
// UnifiedIndicatorsLogPost(shared_ptr<UnifiedIndicatorsContainer> container_ptr)
// {
// window_logger = move(*container_ptr);
// }
// private:
// C2S_PARAM(UnifiedIndicatorsContainer, window_logger);
// };
IndicatorsFiltersManager::IndicatorsFiltersManager(const string& remotePath, const string &assetId,
I_WaapAssetState* pWaapAssetState)
:
SerializeToFileBase(pWaapAssetState->getWaapDataDir() + "/6.data"),
SerializeToLocalAndRemoteSyncBase(
chrono::minutes(120),
chrono::seconds(300),
pWaapAssetState->getWaapDataDir() + "/6.data",
(remotePath == "") ? remotePath : remotePath + "/CentralizedData",
assetId,
"IndicatorsFiltersManager"
),
m_pWaapAssetState(pWaapAssetState),
m_ignoreSources(pWaapAssetState->getWaapDataDir(), remotePath, assetId),
m_tuning(remotePath),
m_matchedOverrideKeywords()
m_matchedOverrideKeywords(),
m_isLeading(getSettingWithDefault<bool>(true, "features", "learningLeader")),
m_sources_limit(DEFAULT_SOURCES_LIMIT),
m_uniqueSources(),
m_unifiedIndicators(make_shared<UnifiedIndicatorsContainer>())
{
restore();
m_keywordsFreqFilter = std::make_unique<KeywordIndicatorFilter>(
m_keywordsFreqFilter = make_unique<KeywordIndicatorFilter>(
pWaapAssetState->getWaapDataDir(),
remotePath,
assetId,
&m_ignoreSources,
&m_tuning);
m_typeFilter = std::make_unique<TypeIndicatorFilter>(pWaapAssetState, remotePath, assetId, &m_tuning);
m_typeFilter = make_unique<TypeIndicatorFilter>(pWaapAssetState, remotePath, assetId, &m_tuning);
m_uniqueSources.reserve(m_sources_limit);
registerConfigLoadCb([this](){
updateSourcesLimit();
updateLearningLeaderFlag();
});
}
IndicatorsFiltersManager::~IndicatorsFiltersManager()
{
}
void IndicatorsFiltersManager::registerKeywords(const std::string& key,
bool IndicatorsFiltersManager::shouldRegister(
const string& key,
const Waap::Keywords::KeywordsSet& keywords,
const IWaf2Transaction* pTransaction)
{
// Check if the learning leader flag is true
if (!m_isLeading) {
dbgDebug(D_WAAP_LEARN) << "Learning leader flag is false. Skipping source ID assertion.";
return false;
}
// if key needs tracking
if (m_keywordsFreqFilter->shouldTrack(key, keywords) ||
m_typeFilter->shouldTrack(key, pTransaction)) {
dbgDebug(D_WAAP_LEARN) << "Key '" << key << "' needs tracking.";
} else {
dbgTrace(D_WAAP_LEARN) << "Key '" << key << "' does not need tracking.";
return false;
}
auto sourceId = pTransaction->getSourceIdentifier();
// Check if the database has reached its limit and source is unknown
if (m_uniqueSources.size() >= static_cast<size_t>(m_sources_limit) &&
m_uniqueSources.find(sourceId) == m_uniqueSources.end() ) {
dbgDebug(D_WAAP_LEARN) << "Database limit reached. Cannot insert new source ID '" << sourceId << "'.";
return false;
}
// Insert the sourceId into the database when did not reached the limit
// If limit is reached, we know that sourceId is already in the database
auto insertResult = m_uniqueSources.insert(sourceId);
if (insertResult.second) {
dbgDebug(D_WAAP_LEARN) << "Inserted new source ID '" << sourceId << "' into the database.";
} else {
dbgTrace(D_WAAP_LEARN) << "source ID '" << sourceId << "' exists in database.";
}
return true;
}
void IndicatorsFiltersManager::registerKeywords(
const string& key,
Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pWaapTransaction)
{
// Check should register - if false, do not collect data
if (!shouldRegister(key, keywords, pWaapTransaction)) {
return;
}
const std::string& sourceId = pWaapTransaction->getSourceIdentifier();
if (m_tuning.getDecision(pWaapTransaction->getLastScanParamName(), PARAM_NAME) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getLastScanSample(), PARAM_VALUE) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getUri(), URL) == MALICIOUS ||
m_tuning.getDecision(pWaapTransaction->getSourceIdentifier(), SOURCE) == MALICIOUS)
m_tuning.getDecision(sourceId, SOURCE) == MALICIOUS)
{
dbgDebug(D_WAAP_LEARN) << "Skipping registration due to tuning decision (malicious)";
return;
}
// TODO: add configuration to choose central logging and return, else do legacy
if(getProfileAgentSettingWithDefault<bool>(false, "agent.learning.centralLogging")) {
dbgDebug(D_WAAP_LEARN) << "Central logging is enabled.";
// Build unified entry
UnifiedIndicatorsContainer::Entry entry;
entry.key = key;
entry.sourceId = pWaapTransaction->getSourceIdentifier();
// Use the new getTrustedSource method for proper trusted source checking
if (m_keywordsFreqFilter) {
auto trustedSourceResult = m_keywordsFreqFilter->getTrustedSource(pWaapTransaction);
entry.isTrusted = trustedSourceResult.ok();
if (entry.isTrusted) {
dbgDebug(D_WAAP_LEARN) << "Entry is from trusted source: " << trustedSourceResult.unpack();
}
} else {
entry.isTrusted = false;
}
for (const auto &kw : keywords) {
entry.indicators.push_back(kw);
}
// Add parameter types as TYPE indicators if applicable (skip url# keys)
if (key.rfind("url#", 0) != 0) {
string sample = pWaapTransaction->getLastScanSample();
auto sampleTypes = m_pWaapAssetState->getSampleType(sample);
entry.types.insert(entry.types.end(), sampleTypes.begin(), sampleTypes.end());
}
// Push to unified container
m_unifiedIndicators->addEntry(entry);
return;
}
dbgTrace(D_WAAP_LEARN) << "Central logging is disabled. Using legacy filters.";
// Legacy behavior (optional): keep existing filters updates for backward compatibility
if (!keywords.empty())
{
m_ignoreSources.log(pWaapTransaction->getSourceIdentifier(), key, keywords);
m_ignoreSources.log(sourceId, key, keywords);
}
m_keywordsFreqFilter->registerKeywords(key, keywords, pWaapTransaction);
if (key.rfind("url#", 0) == 0)
{
@@ -73,7 +197,7 @@ void IndicatorsFiltersManager::registerKeywords(const std::string& key,
}
}
bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const std::string &keyword) const
bool IndicatorsFiltersManager::shouldFilterKeyword(const string &key, const string &keyword) const
{
bool shouldFilter = false;
if (m_keywordsFreqFilter != nullptr)
@@ -99,14 +223,14 @@ bool IndicatorsFiltersManager::shouldFilterKeyword(const std::string &key, const
return shouldFilter;
}
void IndicatorsFiltersManager::serialize(std::ostream& stream)
void IndicatorsFiltersManager::serialize(ostream& stream)
{
cereal::JSONOutputArchive archive(stream);
archive(cereal::make_nvp("version", 1), cereal::make_nvp("trustedSrcParams", m_trustedSrcParams));
}
void IndicatorsFiltersManager::deserialize(std::istream& stream)
void IndicatorsFiltersManager::deserialize(istream& stream)
{
cereal::JSONInputArchive archive(stream);
@@ -116,10 +240,10 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream)
{
archive(cereal::make_nvp("version", version));
}
catch (std::runtime_error & e) {
catch (runtime_error & e) {
archive.setNextName(nullptr);
version = 0;
dbgDebug(D_WAAP) << "Can't load file version: " << e.what();
dbgDebug(D_WAAP_LEARN) << "Can't load file version: " << e.what();
}
switch (version)
@@ -131,12 +255,12 @@ void IndicatorsFiltersManager::deserialize(std::istream& stream)
archive(cereal::make_nvp("trustedSrcParams", m_trustedSrcParams));
break;
default:
dbgWarning(D_WAAP) << "unknown file format version: " << version;
dbgWarning(D_WAAP_LEARN) << "unknown file format version: " << version;
break;
}
}
std::set<std::string> IndicatorsFiltersManager::getParameterTypes(const std::string& canonicParam) const
set<string> IndicatorsFiltersManager::getParameterTypes(const string& canonicParam) const
{
return m_typeFilter->getParamTypes(canonicParam);
}
@@ -166,18 +290,18 @@ bool IndicatorsFiltersManager::loadPolicy(IWaapConfig* pConfig)
}
else
{
dbgWarning(D_WAAP) << "Failed to get configuration";
dbgWarning(D_WAAP_LEARN) << "Failed to get configuration";
}
return pConfig != NULL;
}
void IndicatorsFiltersManager::filterVerbose(const std::string &param,
std::vector<std::string>& filteredKeywords,
std::map<std::string, std::vector<std::string>>& filteredKeywordsVerbose)
void IndicatorsFiltersManager::filterVerbose(const string &param,
vector<string>& filteredKeywords,
map<string, vector<string>>& filteredKeywordsVerbose)
{
static std::string typeFilterName = "type indicators filter";
static std::string keywordsFilterName = "keywords frequency indicators filter";
static string typeFilterName = "type indicators filter";
static string keywordsFilterName = "keywords frequency indicators filter";
filteredKeywordsVerbose[typeFilterName];
filteredKeywordsVerbose[keywordsFilterName];
auto types = getParameterTypes(param);
@@ -209,12 +333,12 @@ void IndicatorsFiltersManager::reset()
}
std::string IndicatorsFiltersManager::extractUri(const std::string& referer, const IWaf2Transaction* pTransaction)
string IndicatorsFiltersManager::extractUri(const string& referer, const IWaf2Transaction* pTransaction)
{
std::string url;
string url;
size_t pos = referer.find("://");
if (pos == std::string::npos || (pos + 3) > referer.size())
if (pos == string::npos || (pos + 3) > referer.size())
{
url = referer;
}
@@ -223,11 +347,11 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
url = referer.substr(pos + 3);
}
pos = url.find('/');
if (pos == std::string::npos)
if (pos == string::npos)
{
return url;
}
std::string host = url.substr(0, pos);
string host = url.substr(0, pos);
if (host == pTransaction->getHdrContent("host"))
{
return url.substr(pos);
@@ -235,13 +359,13 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
return url;
}
std::string IndicatorsFiltersManager::generateKey(const std::string& location,
const std::string& param_name,
string IndicatorsFiltersManager::generateKey(const string& location,
const string& param_name,
const IWaf2Transaction* pTransaction)
{
std::string key = location;
static const std::string delim = "#";
std::string param = normalize_param(param_name);
string key = location;
static const string delim = "#";
string param = normalize_param(param_name);
if (location == "header" || location == "cookie" || location == "url_param")
{
@@ -268,8 +392,8 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location,
}
else if (location == "referer")
{
std::string referer = pTransaction->getHdrContent("referer");
std::string uri = extractUri(referer, pTransaction);
string referer = pTransaction->getHdrContent("referer");
string uri = extractUri(referer, pTransaction);
key = "url" + delim + normalize_uri(uri);
}
else
@@ -279,10 +403,10 @@ std::string IndicatorsFiltersManager::generateKey(const std::string& location,
return key;
}
std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& canonicKey, IWaf2Transaction* pTransaction)
string IndicatorsFiltersManager::getLocationFromKey(const string& canonicKey, IWaf2Transaction* pTransaction)
{
std::vector<std::string> known_locations = { "header", "cookie", "url", "body", "referer", "url_param" };
std::string delim = "#";
vector<string> known_locations = { "header", "cookie", "url", "body", "referer", "url_param" };
string delim = "#";
for (auto location : known_locations)
{
if (canonicKey.find(location + delim) == 0)
@@ -294,9 +418,9 @@ std::string IndicatorsFiltersManager::getLocationFromKey(const std::string& cano
}
void IndicatorsFiltersManager::filterKeywords(
const std::string &key,
const string &key,
Waap::Keywords::KeywordsSet& keywords,
std::vector<std::string>& filteredKeywords)
vector<string>& filteredKeywords)
{
for (auto keyword = keywords.begin(); keyword != keywords.end(); )
{
@@ -313,10 +437,15 @@ void IndicatorsFiltersManager::filterKeywords(
}
void IndicatorsFiltersManager::pushSample(
const std::string& key,
const std::string& sample,
const string& key,
const string& sample,
IWaf2Transaction* pTransaction)
{
// Check learning leader flag - if false, do not collect data
if (!m_isLeading) {
return;
}
if (key.rfind("url#", 0) == 0)
{
return;
@@ -324,7 +453,75 @@ void IndicatorsFiltersManager::pushSample(
m_typeFilter->registerKeywords(key, sample, pTransaction);
}
std::set<std::string> & IndicatorsFiltersManager::getMatchedOverrideKeywords(void)
set<string> & IndicatorsFiltersManager::getMatchedOverrideKeywords(void)
{
return m_matchedOverrideKeywords;
}
void IndicatorsFiltersManager::updateLearningLeaderFlag() {
m_isLeading = getSettingWithDefault<bool>(true, "features", "learningLeader");
dbgDebug(D_WAAP_LEARN) << "Updating learning leader flag from configuration: " << (m_isLeading ? "true" : "false");
}
void IndicatorsFiltersManager::updateSourcesLimit()
{
int new_limit = getProfileAgentSettingWithDefault<int>(DEFAULT_SOURCES_LIMIT, "agent.learning.sourcesLimit");
if (new_limit != m_sources_limit) {
m_sources_limit = new_limit;
m_uniqueSources.reserve(m_sources_limit);
}
}
bool IndicatorsFiltersManager::postData()
{
dbgDebug(D_WAAP_LEARN) << "Posting indicators data";
// Example: post unified indicators data if present
if (m_unifiedIndicators->getKeyCount() == 0) {
dbgDebug(D_WAAP_LEARN) << "No unified indicators to post, skipping";
return true; // Nothing to post
}
// Post unified indicators using REST client with C2S_PARAM
UnifiedIndicatorsLogPost logPost(m_unifiedIndicators);
string postUrl = getPostDataUrl();
dbgTrace(D_WAAP_LEARN) << "Posting unified indicators to: " << postUrl;
bool ok = sendNoReplyObjectWithRetry(logPost, HTTPMethod::PUT, postUrl);
if (!ok) {
dbgError(D_WAAP_LEARN) << "Failed to post unified indicators to: " << postUrl;
}
m_unifiedIndicators = make_shared<UnifiedIndicatorsContainer>();
m_uniqueSources.clear();
return ok;
}
void IndicatorsFiltersManager::pullData(const vector<string>& files)
{
// Phase 2 : backup sync flow
// Add logic for pulling data from a remote service
}
void IndicatorsFiltersManager::processData()
{
// Phase 2 : backup sync flow
// Add logic for processing pulled data
// call filters with ptr to unified data to process data from m_unifiedIndicators
}
void IndicatorsFiltersManager::postProcessedData()
{
// Add logic for posting processed data to a remote service
}
void IndicatorsFiltersManager::pullProcessedData(const vector<string>& files)
{
// Add logic for pulling processed data from a remote service
}
// TODO: Phase 3 implement getRemoteStateFilePath to return the base dir
void IndicatorsFiltersManager::updateState(const vector<string>& files)
{
// files is a list of single file base dir
// TODO phase 3: call each filter to update internal states
}

27
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h
View File

@@ -24,16 +24,18 @@
#include <cereal/cereal.hpp>
#include <cereal/types/memory.hpp>
#include <cereal/archives/json.hpp>
#include "UnifiedIndicatorsContainer.h"
using namespace Waap::Parameters;
class IWaf2Transaction;
struct Waf2ScanResult;
class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToFileBase
class IndicatorsFiltersManager : public I_IndicatorsFilter, public SerializeToLocalAndRemoteSyncBase
{
public:
IndicatorsFiltersManager(const std::string &remotePath, const std::string &assetId,
I_WaapAssetState* pWaapAssetState);
~IndicatorsFiltersManager();
virtual void registerKeywords(const std::string &key, Waap::Keywords::KeywordsSet &keywords,
@@ -59,13 +61,36 @@ public:
virtual void deserialize(std::istream &stream);
virtual std::set<std::string> getParameterTypes(const std::string &canonicParam) const;
// New required functions from SerializeToLocalAndRemoteSyncBase
virtual bool postData() override;
virtual void pullData(const std::vector<std::string>& files) override;
virtual void processData() override;
virtual void postProcessedData() override;
virtual void pullProcessedData(const std::vector<std::string>& files) override;
virtual void updateState(const std::vector<std::string>& files) override;
// Getter for unified indicators (for testing)
const UnifiedIndicatorsContainer& getUnifiedIndicators() const { return *m_unifiedIndicators; }
private:
static std::string extractUri(const std::string &referer, const IWaf2Transaction* pTransaction);
void updateLearningLeaderFlag();
bool shouldRegister(
const std::string& key,
const Waap::Keywords::KeywordsSet& keywords,
const IWaf2Transaction* pTransaction
);
void updateSourcesLimit();
std::unique_ptr<KeywordIndicatorFilter> m_keywordsFreqFilter;
std::unique_ptr<TypeIndicatorFilter> m_typeFilter;
I_WaapAssetState* m_pWaapAssetState;
std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter> m_trustedSrcParams;
ScannerDetector m_ignoreSources;
TuningDecision m_tuning;
std::set<std::string> m_matchedOverrideKeywords;
bool m_isLeading;
int m_sources_limit = 0;
std::unordered_set<std::string> m_uniqueSources;
std::shared_ptr<UnifiedIndicatorsContainer> m_unifiedIndicators;
};

299
components/security_apps/waap/waap_clib/KeyStack.cc
View File

@@ -20,36 +20,73 @@
USE_DEBUG_FLAG(D_WAAP);
KeyStack::KeyStack(const char* name)
:m_name(name), m_nameDepth(0) {
:m_name(name), m_nameDepth(0), m_total_length(0), m_using_buffer(true),
m_str_cache_valid(false), m_first_cache_valid(false) {
m_buffer[0] = '\0';
m_positions.reserve(16); // Reserve reasonable capacity
m_lengths.reserve(16);
m_fallback_stack.reserve(16);
}
void KeyStack::push(const char* subkey, size_t subkeySize, bool countDepth) {
m_stack.push_back(m_key.size());
void
KeyStack::push(const char *subkey, size_t subkeySize, bool countDepth)
{
bool dot_needed = !m_positions.empty() && subkey != nullptr && subkeySize > 0;
if (m_using_buffer) {
// Calculate space needed: subkey + dot (if not first) + null terminator
size_t dot_size = dot_needed ? 1 : 0;
size_t needed_space = subkeySize + dot_size + 1; // +1 for null terminator
// Prefix all subkeys (except the first) with '.'
if (!m_key.empty()) {
m_key += '.';
if (can_fit_in_buffer(needed_space)) {
// Fast path: use fixed buffer
if (dot_needed) {
m_buffer[m_total_length] = '.';
m_total_length++;
}
m_key += std::string(subkey, subkeySize);
m_positions.push_back(m_total_length);
m_lengths.push_back(subkeySize);
memcpy(m_buffer + m_total_length, subkey, subkeySize);
m_total_length += subkeySize;
m_buffer[m_total_length] = '\0';
} else {
// Switch to fallback mode
switch_to_fallback();
// Continue with fallback logic below
}
}
if (!m_using_buffer) {
// Slow path: use dynamic string
m_fallback_stack.push_back(m_fallback_key.size());
if (dot_needed) {
m_fallback_key.append(1, '.');
}
m_fallback_key.append(subkey, subkeySize);
}
if (countDepth) {
m_nameDepth++;
}
// Invalidate cache since key structure changed
invalidate_cache();
dbgTrace(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::push(): '"
<< std::string(subkey, subkeySize)
<< "' => full_key='"
<< std::string(m_key.data(), m_key.size())
<< c_str()
<< "'";
}
void KeyStack::pop(const char* log, bool countDepth) {
// Keep depth balanced even if m_key[] buffer is full
if (m_key.empty() || m_stack.empty()) {
if (m_using_buffer) {
if (m_positions.empty()) {
dbgDebug(D_WAAP)
<< "KeyStack("
<< m_name
@@ -58,18 +95,56 @@ void KeyStack::pop(const char* log, bool countDepth) {
return;
}
// Remove last subkey from buffer
m_total_length = m_positions.back();
// Only remove dot if:
// 1. There are multiple elements (not the first)
// 2. The element being popped had content (length > 0, meaning a dot was added)
// 3. The character before current position is actually a dot (safety check)
if (m_positions.size() > 1 && m_lengths.back() > 0) {
if (m_total_length > 0 && m_buffer[m_total_length - 1] == '.') {
m_total_length -= 1; // Remove the dot
}
} else if (m_positions.size() == 1) {
m_total_length = 0; // First element, no dot to remove
}
m_positions.pop_back();
m_lengths.pop_back();
m_buffer[m_total_length] = '\0';
} else {
// Fallback mode
if (m_fallback_key.empty() || m_fallback_stack.empty()) {
dbgDebug(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): [ERROR] ATTEMPT TO POP FROM EMPTY KEY STACK! "
<< log;
return;
}
// Remove last subkey
m_fallback_key.erase(m_fallback_stack.back());
m_fallback_stack.pop_back();
// Try to switch back to buffer if possible
if (m_fallback_key.size() + 1 < MAX_KEY_SIZE) {
rebuild_buffer_from_fallback();
}
}
if (countDepth) {
m_nameDepth--;
}
// Remove last subkey.
m_key.erase(m_stack.back());
m_stack.pop_back();
// Invalidate cache since key structure changed
invalidate_cache();
dbgTrace(D_WAAP)
<< "KeyStack("
<< m_name
<< ")::pop(): full_key='"
<< std::string(m_key.data(), (int)m_key.size())
<< c_str()
<< "': pop_key="
<< log
<< "'";
@@ -81,6 +156,200 @@ void KeyStack::print(std::ostream &os) const
<< "KeyStack("
<< m_name
<< ")::show(): full_key='"
<< std::string(m_key.data(), (int)m_key.size())
<< c_str()
<< "'";
}
void KeyStack::clear() {
if (m_using_buffer) {
m_positions.clear();
m_lengths.clear();
m_total_length = 0;
m_buffer[0] = '\0';
} else {
m_fallback_key.clear();
m_fallback_stack.clear();
m_using_buffer = true;
m_total_length = 0;
m_buffer[0] = '\0';
}
m_nameDepth = 0;
invalidate_cache();
}
size_t KeyStack::size() const {
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
return 0;
}
// Return size from second subkey onwards
return m_total_length - m_positions[1];
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
return 0;
}
// m_fallback_stack[1] points to the dot preceding the 2nd subkey.
// Exclude the dot itself from the reported size.
if (m_fallback_stack[1] + 1 >= m_fallback_key.size()) {
return 0; // Defensive: nothing after the dot
}
return m_fallback_key.size() - (m_fallback_stack[1] + 1);
}
}
const char* KeyStack::c_str() const {
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
return "";
}
// Return pointer to second subkey (skip first + dot)
return m_buffer + m_positions[1];
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
return "";
}
// m_fallback_stack[1] points to the dot. Skip it for consistency with buffer mode.
static thread_local std::string temp_result;
size_t start = m_fallback_stack[1] + 1;
if (start >= m_fallback_key.size()) {
temp_result.clear();
} else {
temp_result = m_fallback_key.substr(start);
}
return temp_result.c_str();
}
}
const std::string KeyStack::str() const {
if (m_str_cache_valid) {
return m_cached_str;
}
if (m_using_buffer) {
if (m_positions.size() <= 1 || m_positions[1] >= m_total_length) {
// No second element or second element has no content
m_cached_str = std::string();
} else {
// Return string from second subkey onwards
m_cached_str = std::string(m_buffer + m_positions[1], m_total_length - m_positions[1]);
}
} else {
// Fallback mode
if (m_fallback_stack.size() <= 1) {
m_cached_str = std::string();
} else {
size_t start = m_fallback_stack[1] + 1; // Skip the dot
if (start >= m_fallback_key.size()) {
m_cached_str.clear();
} else {
m_cached_str = m_fallback_key.substr(start);
}
}
}
m_str_cache_valid = true;
return m_cached_str;
}
const std::string KeyStack::first() const {
if (m_first_cache_valid) {
return m_cached_first;
}
if (m_using_buffer) {
if (m_positions.empty()) {
m_cached_first = std::string();
} else if (m_positions.size() == 1) {
// Only one subkey, return the whole buffer content
m_cached_first = std::string(m_buffer, m_lengths[0]);
} else {
// Multiple subkeys, return first one
m_cached_first = std::string(m_buffer + m_positions[0], m_lengths[0]);
}
} else {
// Fallback mode
if (m_fallback_stack.empty()) {
m_cached_first = std::string();
} else if (m_fallback_stack.size() == 1) {
m_cached_first = m_fallback_key;
} else {
// m_fallback_stack[1] points to the dot; substring up to dot (exclude it)
size_t dot_pos = m_fallback_stack[1];
if (dot_pos == 0 || dot_pos > m_fallback_key.size()) {
m_cached_first.clear();
} else {
m_cached_first = m_fallback_key.substr(0, dot_pos);
}
}
}
m_first_cache_valid = true;
return m_cached_first;
}
bool KeyStack::can_fit_in_buffer(size_t additional_size) const {
return (m_total_length + additional_size) < MAX_KEY_SIZE;
}
void KeyStack::switch_to_fallback() {
// Copy buffer content to fallback string
m_fallback_key.assign(m_buffer, m_total_length);
// Convert positions to stack format used by fallback
m_fallback_stack.clear();
for (size_t i = 0; i < m_positions.size(); ++i) {
if (i == 0) {
m_fallback_stack.push_back(0);
} else {
// Position after dot
m_fallback_stack.push_back(m_positions[i] - 1);
}
}
m_using_buffer = false;
invalidate_cache();
}void KeyStack::rebuild_buffer_from_fallback() {
if (m_fallback_key.size() + 1 >= MAX_KEY_SIZE) {
return; // Still too big for buffer
}
// Copy fallback content back to buffer
memcpy(m_buffer, m_fallback_key.c_str(), m_fallback_key.size());
m_total_length = m_fallback_key.size();
m_buffer[m_total_length] = '\0';
// Rebuild positions and lengths by parsing the buffer
m_positions.clear();
m_lengths.clear();
size_t pos = 0;
while (pos < m_total_length) {
m_positions.push_back(pos);
// Find length of current subkey
size_t start = pos;
while (pos < m_total_length && m_buffer[pos] != '.') {
pos++;
}
m_lengths.push_back(pos - start);
if (pos < m_total_length) {
pos++; // Skip the dot
}
}
// Clear fallback data
m_fallback_key.clear();
m_fallback_stack.clear();
m_using_buffer = true;
invalidate_cache();
}
void KeyStack::invalidate_cache() {
m_str_cache_valid = false;
m_first_cache_valid = false;
}

77
components/security_apps/waap/waap_clib/KeyStack.h
View File

@@ -21,62 +21,49 @@
// Represent string (key) that is concatenation of substrings (subkeys) separated by '.' character.
// Mostly emulates API of C++ std::string class, with addition of push() and pop() methods
// that append individual subkey and delete last subkey from the string efficiently.
// Uses fixed buffer for performance with fallback to dynamic string for long keys.
class KeyStack {
public:
KeyStack(const char *name);
void push(const char *subkey, size_t subkeySize, bool countDepth=true);
void pop(const char* log, bool countDepth=true);
bool empty() const { return m_key.empty(); }
void clear() { m_key.clear(); m_stack.clear(); }
bool empty() const { return m_using_buffer ? m_positions.empty() : m_fallback_key.empty(); }
void clear();
void print(std::ostream &os) const;
size_t depth() const { return m_nameDepth; }
size_t size() const {
return str().size();
}
const char *c_str() const {
// If pushed none - return empty string.
// If pushed once - still return empty string (the once-pushed subkey will only be returned
// by the first() method.
// If pushed twice or more - return all subkeys starting from the second one.
// Also, even if pushed 2 or more times, but pushed empty strings as subkeys,
// then it could happen that m_key is still empty, in which case we should still return empty string.
if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) {
return "";
}
size_t size() const;
const char *c_str() const;
const std::string str() const;
const std::string first() const;
return m_key.c_str() + m_stack[1] + 1;
}
const std::string str() const {
// If pushed none - return empty string.
// If pushed once - still return empty string (the once-pushed subkey will only be returned
// by the first() method.
// If pushed twice or more - return all subkeys starting from the second one.
// Also, even if pushed 2 or more times, but pushed empty strings as subkeys,
// then it could happen that m_key is still empty, in which case we should still return empty string.
if (m_stack.size() <= 1 || m_stack[1] + 1 >= m_key.size()) {
return "";
}
return m_key.substr(m_stack[1] + 1);
}
const std::string first() const {
if (m_stack.size() == 0) {
return "";
}
else if (m_stack.size() == 1) {
return m_key;
}
else {
// m_stack.size() > 1, so m_stack[1] is valid
return m_key.substr(0, m_stack[1]);
}
}
private:
static const size_t MAX_KEY_SIZE = 1024;
const char *m_name;
std::string m_key;
std::vector<size_t> m_stack; // position of individual key name starts in m_key,
// used to backtrack 1 key at a time.
int m_nameDepth;
// Fixed buffer approach for common case (fast path)
char m_buffer[MAX_KEY_SIZE];
std::vector<size_t> m_positions; // Start positions of each subkey in buffer
std::vector<size_t> m_lengths; // Length of each subkey
size_t m_total_length;
bool m_using_buffer;
// Fallback to dynamic approach for long keys (slow path)
std::string m_fallback_key;
std::vector<size_t> m_fallback_stack;
// Caching for frequently accessed methods
mutable std::string m_cached_str;
mutable std::string m_cached_first;
mutable bool m_str_cache_valid;
mutable bool m_first_cache_valid;
// Helper methods
void switch_to_fallback();
void rebuild_buffer_from_fallback();
bool can_fit_in_buffer(size_t additional_size) const;
void invalidate_cache();
};
#endif // __KEYSTACK_H__0a8039e6

16
components/security_apps/waap/waap_clib/KeywordIndicatorFilter.cc
View File

@@ -12,11 +12,11 @@
// limitations under the License.
#include "KeywordIndicatorFilter.h"
#include "i_transaction.h"
#include "waap.h"
#include "WaapConfigApi.h"
#include "WaapConfigApplication.h"
#include "FpMitigation.h"
#include "i_transaction.h"
#include <boost/algorithm/string/predicate.hpp>
#include <boost/algorithm/string/trim.hpp>
@@ -98,12 +98,24 @@ bool KeywordIndicatorFilter::loadParams(std::shared_ptr<Waap::Parameters::WaapPa
return m_confidence_calc.reset(params);
}
bool KeywordIndicatorFilter::shouldTrack(const std::string& key, const Waap::Keywords::KeywordsSet& keywords)
{
for (const auto& keyword : keywords)
{
if (m_confidence_calc.shouldTrackParameter(key, keyword))
{
return true;
}
}
return m_confidence_calc.shouldTrackParameter(key, "");
}
void KeywordIndicatorFilter::registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction)
{
std::string source(pTransaction->getSourceIdentifier());
std::string trusted_source = getTrustedSource(pTransaction);
auto trusted_source_maybe = getTrustedSource(pTransaction);
std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : "";
if (keywords.empty())
{
registerSource(key, source);

2
components/security_apps/waap/waap_clib/KeywordIndicatorFilter.h
View File

@@ -36,10 +36,12 @@ public:
size_t minIntervals = CONFIDENCE_MIN_INTERVALS,
std::chrono::minutes intervalDuration = CONFIDENCE_WINDOW_INTERVAL,
double ratioThreshold = CONFIDENCE_THRESHOLD);
~KeywordIndicatorFilter();
virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction);
bool shouldTrack(const std::string& key, const Waap::Keywords::KeywordsSet& keywords);
virtual bool shouldFilterKeyword(const std::string &key, const std::string &keyword) const;

1
components/security_apps/waap/waap_clib/KeywordTypeValidator.cc
View File

@@ -54,6 +54,7 @@ void KeywordTypeValidator::deserialize(std::istream& stream)
}
catch (std::runtime_error & e) {
dbgWarning(D_WAAP) << "failed to deserialize keyword types validator file. Error: " << e.what();
throw e;
}
}

2
components/security_apps/waap/waap_clib/ParserUrlEncode.cc
View File

@@ -92,7 +92,7 @@ ParserUrlEncode::push(const char *buf, size_t len)
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32) {
if (static_cast<unsigned char>(c) < 32u || static_cast<unsigned char>(c) > 126u) {
dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c;
m_state = s_error;
return i;

240
components/security_apps/waap/waap_clib/ScannerDetector.cc
View File

@@ -19,7 +19,8 @@ USE_DEBUG_FLAG(D_WAAP);
#define SYNC_WAIT_TIME std::chrono::seconds(300) // 5 minutes in seconds
#define INTERVAL std::chrono::minutes(120)
#define EQUAL_VALUES_COUNT_THRESHOLD 2
#define MAX_RETENTION 5
#define MAX_RETENTION 2
#define DEFAULT_MAX_SOURCES 256
ScannerDetector::ScannerDetector(const std::string &localPath, const std::string &remotePath,
const std::string &assetId) :
@@ -27,9 +28,11 @@ ScannerDetector::ScannerDetector(const std::string& localPath, const std::string
localPath + "/11.data",
(remotePath == "") ? remotePath : remotePath + "/ScannersDetector",
assetId,
"ScannerDetector")
"ScannerDetector"),
m_current_accumulator(std::make_shared<SourceKeyValsMap>()),
m_maxSources(getProfileAgentSettingWithDefault<uint>(-1, "scannerDetector.maxSources"))
{
m_sources_monitor.push_front(SourceKeyValsMap());
dbgTrace(D_WAAP) << "ScannerDetector constructor: m_maxSources set to " << m_maxSources;
}
bool ScannerDetector::ready()
@@ -47,9 +50,57 @@ std::vector<std::string>* ScannerDetector::getSourcesToIgnore()
return &m_sources;
}
void ScannerDetector::log(const std::string& source, const std::string& key, Waap::Keywords::KeywordsSet& keywords)
void ScannerDetector::log(
const std::string &source,
const std::string &key,
Waap::Keywords::KeywordsSet &keywords)
{
m_sources_monitor.front()[source][key].insert(keywords.begin(), keywords.end());
if (m_maxSources == uint(-1)) {
m_maxSources = getProfileAgentSettingWithDefault<uint>(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources");
dbgTrace(D_WAAP) << "log: m_maxSources set to " << m_maxSources;
}
// Add to accumulator for processing - same as original
(*m_current_accumulator)[source][key].insert(keywords.begin(), keywords.end());
// Optimized O(1) cache update - just add the key directly to the source cache
auto currentTime = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto cache_it = m_sourceCache.find(source);
if (cache_it != m_sourceCache.end()) {
// Source exists, just add the key - O(1) operation
cache_it->second.keys.insert(key);
cache_it->second.lastUpdate = currentTime;
cache_it->second.accessCount++;
// Move to front of LRU - O(1) operation
auto lru_it = m_lruMap.find(source);
if (lru_it != m_lruMap.end()) {
m_lruOrder.erase(lru_it->second);
m_lruOrder.push_front(source);
m_lruMap[source] = m_lruOrder.begin();
}
dbgTrace(D_WAAP) << "log: Updated existing source " << source << " with key " << key;
return;
}
// New source - check if cache is full
if (m_sourceCache.size() >= m_maxSources) {
evictLeastImportantSource();
}
// Add new source - O(1) operations
SourceInfo newSource(source, currentTime);
newSource.keys.insert(key);
m_sourceCache[source] = newSource;
// Add to front of LRU list - O(1) operation
m_lruOrder.push_front(source);
m_lruMap[source] = m_lruOrder.begin();
dbgTrace(D_WAAP) << "log: Added new source " << source << " with key " << key
<< " (cache size: " << m_sourceCache.size() << ")";
}
void ScannerDetector::loadParams(std::shared_ptr<Waap::Parameters::WaapParameters> pParams)
@@ -59,13 +110,16 @@ void ScannerDetector::loadParams(std::shared_ptr<Waap::Parameters::WaapParameter
setInterval(std::chrono::minutes(std::stoul(interval)));
std::string remoteSyncStr = pParams->getParamVal("remoteSync", "true");
setRemoteSyncEnabled(!boost::iequals(remoteSyncStr, "false"));
m_maxSources = getProfileAgentSettingWithDefault<uint>(DEFAULT_MAX_SOURCES, "scannerDetector.maxSources");
dbgTrace(D_WAAP) << "loadParams: m_maxSources set to " << m_maxSources;
}
class SourcesMonitorPost : public RestGetFile
{
public:
SourcesMonitorPost(ScannerDetector::SourceKeyValsMap &_monitor)
: monitor(_monitor)
: monitor(std::move(_monitor))
{
}
@@ -92,19 +146,19 @@ private:
bool ScannerDetector::postData()
{
m_sources_monitor_backup = m_sources_monitor.front();
m_sources_monitor.push_front(SourceKeyValsMap());
std::string url = getPostDataUrl();
dbgTrace(D_WAAP) << "Sending the data to: " << url;
SourcesMonitorPost currentWindow(m_sources_monitor_backup);
bool ok = sendNoReplyObjectWithRetry(currentWindow,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP) << "Failed to post collected data to: " << url;
if (m_current_accumulator->empty()) {
dbgDebug(D_WAAP) << "No data to post, skipping";
return true;
}
SourcesMonitorPost postMonitor(*m_current_accumulator);
bool ok = sendNoReplyObjectWithRetry(postMonitor,
HTTPMethod::PUT,
getPostDataUrl());
if (ok) {
m_current_accumulator = std::make_shared<SourceKeyValsMap>();
}
return ok;
}
@@ -113,10 +167,10 @@ void ScannerDetector::pullData(const std::vector<std::string>& files)
std::string url = getPostDataUrl();
std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
dbgTrace(D_WAAP) << "pulling files, skipping: " << sentFile;
for (auto file : files)
{
if (file == sentFile)
for (const auto &file : files) // Use const reference
{
if (file == sentFile) {
continue;
}
dbgTrace(D_WAAP) << "Pulling the file: " << file;
@@ -131,31 +185,18 @@ void ScannerDetector::pullData(const std::vector<std::string>& files)
}
SourceKeyValsMap remoteMonitor = getMonitor.getSourcesMonitor().unpack();
for (const auto& srcData : remoteMonitor)
{
for (const auto& keyData : srcData.second)
{
m_sources_monitor_backup[srcData.first][keyData.first].insert(
keyData.second.begin(),
keyData.second.end());
}
}
// update the sources monitor in previous "time window"
auto temp = m_sources_monitor.front();
m_sources_monitor.pop_front();
m_sources_monitor.pop_front();
m_sources_monitor.push_front(m_sources_monitor_backup);
m_sources_monitor.push_front(temp);
mergeMonitors(*m_current_accumulator, remoteMonitor);
}
}
void ScannerDetector::postProcessedData()
{
// Empty implementation as in original
}
void ScannerDetector::updateState(const std::vector<std::string>&)
{
// Empty implementation as in original
}
void ScannerDetector::pullProcessedData(const std::vector<std::string> &files)
@@ -163,72 +204,97 @@ void ScannerDetector::pullProcessedData(const std::vector<std::string>& files)
(void)files;
}
void ScannerDetector::mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom)
void ScannerDetector::evictLeastImportantSource()
{
for (const auto& srcData : mergeFrom)
if (m_lruOrder.empty()) {
return;
}
// Enhanced eviction: scan last N sources in LRU and evict the one with the smallest key count
constexpr size_t N = 10; // Number of candidates to consider
auto it = m_lruOrder.rbegin();
auto it_end = m_lruOrder.rend();
size_t checked = 0;
std::string evictCandidate;
size_t minKeyCount = std::numeric_limits<size_t>::max();
auto candidateIt = m_lruOrder.rbegin();
for (; it != it_end && checked < N; ++it, ++checked) {
const std::string &source = *it;
auto cacheIt = m_sourceCache.find(source);
size_t keyCount = (cacheIt != m_sourceCache.end()) ? cacheIt->second.keys.size() : 0;
if (keyCount < minKeyCount) {
minKeyCount = keyCount;
evictCandidate = source;
candidateIt = it;
}
}
if (evictCandidate.empty()) {
// fallback to classic LRU
evictCandidate = m_lruOrder.back();
candidateIt = m_lruOrder.rbegin();
}
// Remove from all data structures - O(1) operations
m_sourceCache.erase(evictCandidate);
m_lruMap.erase(evictCandidate);
// Erase from m_lruOrder using base iterator
m_lruOrder.erase(std::next(candidateIt).base());
// Remove evicted source from current accumulator
m_current_accumulator->erase(evictCandidate);
dbgTrace(D_WAAP) << "evictLeastImportantSource: Evicted " << evictCandidate
<< " (key count: " << minKeyCount << ", cache size: " << m_sourceCache.size() << ")";
}
void ScannerDetector::mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom)
{
for (const auto& keyData : srcData.second)
{
dbgTrace(D_WAAP) << "merging src: " << srcData.first << ", key: " << keyData.first <<
", keywords: " << Waap::Util::setToString(keyData.second);
mergeTo[srcData.first][keyData.first].insert(keyData.second.begin(), keyData.second.end());
for (const auto &sourceEntry : mergeFrom) { // Use const reference
const std::string &source = sourceEntry.first;
for (const auto &keyEntry : sourceEntry.second) { // Use const reference
const std::string &key = keyEntry.first;
for (const auto &value : keyEntry.second) { // Use const reference
mergeTo[source][key].insert(value);
}
}
}
}
void ScannerDetector::processData()
{
if (m_sources_monitor_backup.empty())
{
m_sources_monitor_backup = m_sources_monitor.front();
m_sources_monitor.push_front(SourceKeyValsMap());
dbgTrace(D_WAAP) << "processData: Processing accumulated sources";
// Move current data to monitor deque for analysis
if (!m_current_accumulator->empty()) {
m_sources_monitor.push_front(m_current_accumulator);
m_current_accumulator = std::make_shared<SourceKeyValsMap>();
}
if (m_sources_monitor.size() > 2)
{
auto monitorItr = m_sources_monitor.begin()++;
for (monitorItr++; monitorItr != m_sources_monitor.end(); monitorItr++)
{
mergeMonitors(m_sources_monitor_backup, *monitorItr);
}
// Merge all monitors into a single monitor, but only include cached sources
SourceKeyValsMap mergedMonitor;
for (const auto &monitor : m_sources_monitor) {
mergeMonitors(mergedMonitor, *monitor);
}
if (m_sources_monitor.size() == MAX_RETENTION) {
m_sources_monitor.pop_back(); // Keep only the latest MAX_RETENTION cycles
}
// Analyze cached sources to identify scanners
m_sources.clear();
for (auto source : m_sources_monitor_backup)
{
if (source.second.size() <= 2)
{
continue;
}
std::map<std::string, std::set<std::string>>& keyVals = source.second;
for (auto key = keyVals.begin(); key != keyVals.end(); key++)
{
auto otherKey = key;
int counter = 0;
for (++otherKey; otherKey != keyVals.end(); otherKey++)
{
if (key->second != otherKey->second)
{
continue;
}
dbgTrace(D_WAAP) << "source monitor: src: " << source.first << ", key_1: " << key->first << ", key_2: "
<< otherKey->first << ", vals: " << Waap::Util::setToString(key->second);
counter++;
}
if (counter >= EQUAL_VALUES_COUNT_THRESHOLD)
{
dbgDebug(D_WAAP) << "source: " << source.first << " will be ignored";
m_sources.push_back(source.first);
break;
}
// Simple threshold-based scanner detection
const uint SCANNER_KEY_THRESHOLD = 3;
for (const auto &sourceInfo : mergedMonitor) {
const std::string &source = sourceInfo.first;
const auto &keys = sourceInfo.second;
if (keys.size() >= SCANNER_KEY_THRESHOLD) {
dbgTrace(D_WAAP) << "processData: Source " << source
<< " flagged as scanner (keyCount=" << keys.size() << ")";
m_sources.push_back(source);
}
}
if (m_sources_monitor.size() > MAX_RETENTION)
{
m_sources_monitor.pop_back();
}
m_sources_monitor_backup.clear();
dbgTrace(D_WAAP) << "processData: Found " << m_sources.size() << " scanners out of "
<< m_sourceCache.size() << " sources in cache";
m_lastSync = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
}

53
components/security_apps/waap/waap_clib/ScannersDetector.h
View File

@@ -11,19 +11,51 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __SCANNERS_DETECTOR_H__
#define __SCANNERS_DETECTOR_H__
#ifndef __OPTIMIZED_SCANNERS_DETECTOR_H__
#define __OPTIMIZED_SCANNERS_DETECTOR_H__
#include "WaapKeywords.h"
#include "i_serialize.h"
#include "i_ignoreSources.h"
#include "WaapParameters.h"
#include <chrono>
#include <unordered_map>
#include <unordered_set>
#include <list>
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
class ScannerDetector : public SerializeToLocalAndRemoteSyncBase, public I_IgnoreSources
{
public:
typedef std::map<std::string, std::map<std::string, std::set<std::string>>> SourceKeyValsMap;
ScannerDetector(const std::string& localPath, const std::string& remotePath = "", const std::string &assetId = "");
struct SourceInfo {
std::string source;
std::unordered_set<std::string> keys;
std::chrono::microseconds lastUpdate;
uint32_t accessCount; // Track access frequency for LFU eviction
// Default constructor for container requirements
SourceInfo() : lastUpdate(std::chrono::microseconds(0)), accessCount(0) {}
SourceInfo(const std::string &src, std::chrono::microseconds time)
: source(src), lastUpdate(time), accessCount(1) {}
uint getKeyCount() const { return keys.size(); }
void updateKeys(const std::set<std::string> &newKeys) {
keys.clear();
keys.insert(newKeys.begin(), newKeys.end());
lastUpdate = std::chrono::duration_cast<std::chrono::microseconds>(
std::chrono::steady_clock::now().time_since_epoch());
accessCount++;
}
};
ScannerDetector(
const std::string &localPath,
const std::string &remotePath = "",
const std::string &assetId = "");
virtual bool ready();
virtual std::vector<std::string>* getSourcesToIgnore();
@@ -42,14 +74,21 @@ public:
virtual void deserialize(std::istream &stream);
private:
void mergeMonitors(SourceKeyValsMap& mergeTo, SourceKeyValsMap& mergeFrom);
void evictLeastImportantSource();
void mergeMonitors(SourceKeyValsMap &mergeTo, const SourceKeyValsMap &mergeFrom);
std::list<SourceKeyValsMap> m_sources_monitor; // list of map source -> key -> set of indicators
SourceKeyValsMap m_sources_monitor_backup; // stores data of the last window to process
// Optimized data structures
std::unordered_map<std::string, SourceInfo> m_sourceCache;
std::list<std::string> m_lruOrder;
std::unordered_map<std::string, std::list<std::string>::iterator> m_lruMap;
// Original data structures for compatibility
std::shared_ptr<SourceKeyValsMap> m_current_accumulator;
std::deque<std::shared_ptr<SourceKeyValsMap>> m_sources_monitor;
SourceKeyValsMap m_sources_monitor_backup;
std::vector<std::string> m_sources;
std::chrono::microseconds m_lastSync;
uint m_maxSources;
};
#endif

1
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -113,6 +113,7 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
restore();
}
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores) :
m_scoreTrigger(0),
m_fpStore(),

421
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -16,6 +16,7 @@
#include "Waf2Util.h"
#include "WaapAssetState.h"
#include "i_instance_awareness.h"
#include "buffered_compressed_stream.h"
#include <boost/regex.hpp>
#include <sstream>
#include <fstream>
@@ -33,7 +34,7 @@ USE_DEBUG_FLAG(D_WAAP_SERIALIZE);
namespace ch = std::chrono;
using namespace std;
typedef ch::duration<size_t, std::ratio<86400>> days;
typedef ch::duration<size_t, ratio<86400>> days;
// Define interval between successful sync times
static const ch::minutes assetSyncTimeSliceLength(10);
@@ -44,26 +45,40 @@ static const string defaultSharedStorageHost = "appsec-shared-storage-svc";
#define SHARED_STORAGE_HOST_ENV_NAME "SHARED_STORAGE_HOST"
#define LEARNING_HOST_ENV_NAME "LEARNING_HOST"
void yieldIfPossible(const string& func, int line)
{
// Check if we are in the main loop
if (Singleton::exists<I_MainLoop>() &&
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->getCurrentRoutineId().ok())
{
// If we are not in the main loop, yield to allow other routines to run
// This is important for the main loop to be able to process other events
// and avoid blocking the entire system.
dbgTrace(D_WAAP_SERIALIZE) << "Yielding to main loop from: " << func << ":" << line;
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
}
}
#define YIELD_IF_POSSIBLE() yieldIfPossible(__FUNCTION__, __LINE__)
bool RestGetFile::loadJson(const string &json)
{
string json_str;
// Try streaming approach first - handles both encryption and compression
try {
dbgTrace(D_WAAP_SERIALIZE) << "Attempting to use streaming approach for JSON loading, data size: "
<< json.size() << " bytes";
stringstream json_stream(json);
// if input json is big then yield to allow other routines to run
if (json.size() > 1000000) {
dbgTrace(D_WAAP_SERIALIZE) << "Input JSON is large, yielding to allow other routines to run";
YIELD_IF_POSSIBLE();
}
BufferedCompressedInputStream decompressed_stream(json_stream);
{
cereal::JSONInputArchive json_archive(decompressed_stream);
load(json_archive);
}
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded JSON using streaming approach";
return true;
}
catch (const exception &e) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach: " << e.what()
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
}
catch (...) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load JSON using streaming approach"
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
}
// Legacy approach: manual decryption and decompression
string json_str;
json_str = json;
if (!Waap::Util::isGzipped(json_str))
{
@@ -87,7 +102,7 @@ bool RestGetFile::loadJson(const string& json)
finiCompressionStream(compression_stream);
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Yielded after decompression in loadJson, decompressed size: "
dbgTrace(D_WAAP_SERIALIZE) << "Yielded after legacy decompression in loadJson, decompressed size: "
<< json_str.size() << " bytes";
return ClientRest::loadJson(json_str);
@@ -95,76 +110,42 @@ bool RestGetFile::loadJson(const string& json)
Maybe<string> RestGetFile::genJson() const
{
Maybe<string> json = ClientRest::genJson();
YIELD_IF_POSSIBLE();
if (json.ok())
stringstream output_stream;
try
{
string data = json.unpack();
// Get chunk size from profile settings for compression chunks
const size_t COMPRESSED_CHUNK_SIZE = static_cast<size_t>(
getProfileAgentSettingWithDefault<uint>(64 * 1024, "appsecLearningSettings.compressionChunkSize"));
auto compression_stream = initCompressionStream();
size_t offset = 0;
std::vector<unsigned char> compressed_data;
bool ok = true;
size_t chunk_count = 0;
// Process data in chunks for compression
while (offset < data.size()) {
size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
bool is_last = (offset + chunk_size >= data.size());
CompressionResult chunk_res = compressData(
compression_stream,
CompressionType::GZIP,
static_cast<uint32_t>(chunk_size),
reinterpret_cast<const unsigned char *>(data.c_str() + offset),
is_last ? 1 : 0
);
if (!chunk_res.ok) {
ok = false;
break;
BufferedCompressedOutputStream compressed_out(output_stream);
{
cereal::JSONOutputArchive json_archive(compressed_out, cereal::JSONOutputArchive::Options::NoIndent());
save(json_archive);
}
compressed_out.close();
}
catch (const exception &e)
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to generate JSON: " << e.what();
return genError("Failed to generate JSON: " + string(e.what()));
}
return output_stream.str();
}
if (chunk_res.output && chunk_res.num_output_bytes > 0) {
compressed_data.insert(
compressed_data.end(),
chunk_res.output,
chunk_res.output + chunk_res.num_output_bytes
);
free(chunk_res.output);
chunk_res.output = nullptr;
}
// Class to handle retrieving the state timestamp file from learning service
class StateTimestampRetriever : public ClientRest
{
public:
StateTimestampRetriever() {}
offset += chunk_size;
chunk_count++;
YIELD_IF_POSSIBLE();
dbgTrace(D_WAAP_SERIALIZE) << "Processed compression chunk " << chunk_count
<< ", progress: " << offset << "/" << data.size() << " bytes ("
<< (offset * 100 / data.size()) << "%) - yielded";
Maybe<string> getStateTimestamp() const
{
if (timestamp.get().empty()) {
return genError("State timestamp is empty");
}
finiCompressionStream(compression_stream);
dbgDebug(D_WAAP_SERIALIZE) << "Yielded after finalizing compression stream. "
<< "Total chunks: " << chunk_count << ", Compression ratio: "
<< (data.size() > 0 ? (float)compressed_data.size() / data.size() : 0) << "x";
if (!ok) {
dbgWarning(D_WAAP_SERIALIZE) << "Failed to gzip data";
return genError("Failed to compress data");
return timestamp.get();
}
private:
S2C_PARAM(string, timestamp);
};
// Create string from compressed data
string compressed_str(reinterpret_cast<const char*>(compressed_data.data()), compressed_data.size());
json = compressed_str;
}
return json;
}
SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, string filePath) :
SerializeToFilePeriodically::SerializeToFilePeriodically(ch::seconds pollingIntervals, const string &filePath) :
SerializeToFileBase(filePath),
m_lastSerialization(0),
m_interval(pollingIntervals)
@@ -210,7 +191,7 @@ void SerializeToFilePeriodically::setInterval(ch::seconds newInterval)
}
}
SerializeToFileBase::SerializeToFileBase(string fileName) : m_filePath(fileName)
SerializeToFileBase::SerializeToFileBase(const string &fileName) : m_filePath(fileName)
{
dbgTrace(D_WAAP_SERIALIZE) << "SerializeToFileBase::SerializeToFileBase() fname='" << m_filePath
<< "'";
@@ -243,7 +224,7 @@ void SerializeToFileBase::saveData()
if (maybe_routine.ok()) {
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->yield(false);
}
string data = ss.str();
const string &data = ss.str(); // Use const reference to avoid copying
dbgDebug(D_WAAP_SERIALIZE) << "Serialized data size: " << data.size() << " bytes";
// Get chunk size from profile settings, with default of 16 MiB for compression chunks
@@ -255,13 +236,13 @@ void SerializeToFileBase::saveData()
auto compression_stream = initCompressionStream();
size_t offset = 0;
std::vector<unsigned char> compressed_data;
vector<unsigned char> compressed_data;
bool ok = true;
size_t chunk_count = 0;
// Process data in chunks for compression
while (offset < data.size()) {
size_t chunk_size = std::min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
size_t chunk_size = min(COMPRESSED_CHUNK_SIZE, data.size() - offset);
bool is_last = (offset + chunk_size >= data.size());
CompressionResult chunk_res = compressData(
compression_stream,
@@ -319,7 +300,7 @@ void SerializeToFileBase::saveData()
size_t write_chunks = 0;
while (offset < data_to_write.size()) {
size_t current_chunk_size = std::min(CHUNK_SIZE, data_to_write.size() - offset);
size_t current_chunk_size = min(CHUNK_SIZE, data_to_write.size() - offset);
filestream.write(data_to_write.c_str() + offset, current_chunk_size);
offset += current_chunk_size;
write_chunks++;
@@ -334,7 +315,7 @@ void SerializeToFileBase::saveData()
<< " (" << data_to_write.size() << " bytes in " << write_chunks << " chunks)";
}
string decompress(string fileContent) {
string decompress(const string &fileContent) {
if (!Waap::Util::isGzipped(fileContent)) {
dbgTrace(D_WAAP_SERIALIZE) << "file note zipped";
return fileContent;
@@ -360,7 +341,7 @@ string decompress(string fileContent) {
return fileContent;
}
void SerializeToFileBase::loadFromFile(string filePath)
void SerializeToFileBase::loadFromFile(const string &filePath)
{
dbgTrace(D_WAAP_SERIALIZE) << "loadFromFile() file: " << filePath;
fstream filestream;
@@ -368,7 +349,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
filestream.open(filePath, fstream::in);
if (filestream.is_open() == false) {
dbgTrace(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " <<
dbgWarning(D_WAAP_SERIALIZE) << "failed to open file: " << filePath << " Error: " <<
strerror(errno);
if (!Singleton::exists<I_InstanceAwareness>() || errno != ENOENT)
{
@@ -387,34 +368,48 @@ void SerializeToFileBase::loadFromFile(string filePath)
size_t idPosition = filePath.find(idStr);
if (idPosition != string::npos)
{
filePath.erase(idPosition, idStr.length() - 1);
dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << filePath;
loadFromFile(filePath);
string modifiedFilePath = filePath; // Create a mutable copy
modifiedFilePath.erase(idPosition, idStr.length() - 1);
dbgDebug(D_WAAP_SERIALIZE) << "retry to load file from : " << modifiedFilePath;
loadFromFile(modifiedFilePath);
}
return;
}
dbgTrace(D_WAAP_SERIALIZE) << "loading from file: " << filePath;
int length;
filestream.seekg(0, ios::end); // go to the end
length = filestream.tellg(); // report location (this is the length)
try {
dbgTrace(D_WAAP_SERIALIZE) << "Attempting to load file using streaming approach";
BufferedCompressedInputStream decompressed_stream(filestream);
deserialize(decompressed_stream);
filestream.close();
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using streaming approach";
return;
}
catch (const exception &e) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to load file using streaming approach: " << e.what()
<< ". Falling back to legacy approach.";
// Fall back to the legacy approach for backward compatibility
filestream.clear();
filestream.seekg(0, ios::beg);
}
// Legacy approach: manual file reading, decryption, and decompression
filestream.seekg(0, ios::end);
int length = filestream.tellg();
dbgTrace(D_WAAP_SERIALIZE) << "file length: " << length;
assert(length >= 0); // length -1 really happens if filePath is a directory (!)
char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
vector<char> buffer(length); // Use vector instead of raw pointer for safety
filestream.seekg(0, ios::beg); // go back to the beginning
if (!filestream.read(buffer, length)) // read the whole file into the buffer
if (!filestream.read(buffer.data(), length)) // read the whole file into the buffer
{
filestream.close();
delete[] buffer;
dbgWarning(D_WAAP_SERIALIZE) << "Failed to read file, file: " << filePath;
return;
}
filestream.close();
string dataObfuscated(buffer, length);
delete[] buffer;
string dataObfuscated(buffer.begin(), buffer.end());
stringstream ss;
ss << decompress(dataObfuscated);
@@ -422,6 +417,7 @@ void SerializeToFileBase::loadFromFile(string filePath)
try
{
deserialize(ss);
dbgTrace(D_WAAP_SERIALIZE) << "Successfully loaded file using legacy approach";
}
catch (runtime_error & e) {
dbgWarning(D_WAAP_SERIALIZE) << "failed to deserialize file: " << m_filePath << ", error: " <<
@@ -497,8 +493,8 @@ bool RemoteFilesList::loadJson(const string& xml)
}
contents_node = contents_node->next;
}
files.get().push_back(FileMetaData{ file, lastModified });
filesPathsList.push_back(file);
files.get().push_back(FileMetaData{ move(file), move(lastModified) });
filesPathsList.push_back(files.get().back().filename); // Use the moved string to avoid extra copy
}
node = node->next;
}
@@ -547,6 +543,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
{
dbgInfo(D_WAAP_SERIALIZE) << "Create SerializeToLocalAndRemoteSyncBase. assetId='" << assetId <<
"', owner='" << m_owner << "'";
m_pMainLoop = Singleton::Consume<I_MainLoop>::by<WaapComponent>();
if (Singleton::exists<I_AgentDetails>() &&
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() ==
@@ -586,11 +583,10 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_type = type;
}
}
m_pMainLoop = Singleton::Consume<I_MainLoop>::by<WaapComponent>();
setInterval(interval);
}
bool SerializeToLocalAndRemoteSyncBase::isBase()
bool SerializeToLocalAndRemoteSyncBase::isBase() const
{
return m_remotePath == "";
}
@@ -656,13 +652,12 @@ void SerializeToLocalAndRemoteSyncBase::setRemoteSyncEnabled(bool enabled)
void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
{
dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " <<
newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'";
if (newInterval == m_interval)
{
return;
}
dbgDebug(D_WAAP_SERIALIZE) << "setInterval: from " << m_interval.count() << " to " <<
newInterval.count() << " seconds. assetId='" << m_assetId << "', owner='" << m_owner << "'";
m_interval = newInterval;
@@ -730,7 +725,7 @@ void SerializeToLocalAndRemoteSyncBase::setInterval(ch::seconds newInterval)
m_pMainLoop->yield(remainingTime);
timeBeforeSyncWorker = timer->getWalltime();
m_pMainLoop->addOneTimeRoutine(I_MainLoop::RoutineType::System, [this]() {syncWorker();}, "Sync worker");
syncWorker();
timeAfterSyncWorker = timer->getWalltime();
}
};
@@ -780,32 +775,117 @@ ch::seconds SerializeToLocalAndRemoteSyncBase::getIntervalDuration() const
return m_interval;
}
void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService()
Maybe<string> SerializeToLocalAndRemoteSyncBase::getStateTimestampByListing()
{
for (int i = 0; i < remoteSyncMaxPollingAttempts; i++)
{
m_pMainLoop->yield(ch::seconds(60));
RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
if (remoteFiles.getFilesMetadataList().empty())
{
dbgWarning(D_WAAP_SERIALIZE) << "no files generated by the remote service were found";
return genError("No remote processed files available");
}
dbgDebug(D_WAAP_SERIALIZE) << "State timestamp by listing: "
<< remoteFiles.getFilesMetadataList()[0].modified;
return remoteFiles.getFilesMetadataList()[0].modified;
}
bool SerializeToLocalAndRemoteSyncBase::checkAndUpdateStateTimestamp(const string& currentStateTimestamp)
{
// Check if the state has been updated since last check
if (currentStateTimestamp != m_lastProcessedModified)
{
m_lastProcessedModified = currentStateTimestamp;
dbgDebug(D_WAAP_SERIALIZE) << "State timestamp updated: " << m_lastProcessedModified;
return true; // State was updated
}
return false; // State unchanged
}
void SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteService()
{
bool useFallbackMethod = false;
for (int i = 0; i < remoteSyncMaxPollingAttempts; i++)
{
m_pMainLoop->yield(ch::seconds(60));
// Try the dedicated timestamp file first
Maybe<string> timestampResult(genError("Failed to get state timestamp"));
if (!useFallbackMethod) {
timestampResult = getStateTimestamp();
if (!timestampResult.ok()) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp from file: "
<< timestampResult.getErr() << ", trying listing method";
useFallbackMethod = true; // Switch to listing method on first failure
}
}
else
{
dbgDebug(D_WAAP_SERIALIZE) << "trying listing method";
timestampResult = getStateTimestampByListing();
}
if (!timestampResult.ok())
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp using any method: "
<< timestampResult.getErr();
continue;
}
string lastModified = remoteFiles.getFilesMetadataList().begin()->modified;
if (lastModified != m_lastProcessedModified)
string currentStateTimestamp = timestampResult.unpack();
if (checkAndUpdateStateTimestamp(currentStateTimestamp))
{
m_lastProcessedModified = lastModified;
updateState(remoteFiles.getFilesList());
dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner <<
". updated state generated by remote at " << m_lastProcessedModified;
// Update state directly from the known remote file path
updateStateFromRemoteFile();
dbgInfo(D_WAAP_SERIALIZE) << "Owner: " << m_owner
<< ". updated state using " << (useFallbackMethod ? "file listing (fallback)" : "timestamp file")
<< ": " << m_lastProcessedModified;
return;
}
else
{
dbgWarning(D_WAAP_SERIALIZE) << "State timestamp unchanged ("
<< (useFallbackMethod ? "file listing (fallback)" : "timestamp file") << "): "
<< currentStateTimestamp;
}
dbgWarning(D_WAAP_SERIALIZE) << "polling for update state timeout. for assetId='"
}
// All polling attempts failed - fall back to local sync
dbgWarning(D_WAAP_SERIALIZE) << "Polling for update state timeout, falling back to local sync. for assetId='"
<< m_assetId << "', owner='" << m_owner;
localSyncAndProcess();
}
Maybe<void> SerializeToLocalAndRemoteSyncBase::updateStateFromRemoteFile()
{
auto maybeRemoteFilePath = getRemoteStateFilePath();
if (!maybeRemoteFilePath.ok())
{
string error = "Owner: " + m_owner + ", no remote state file path defined: " + maybeRemoteFilePath.getErr();
dbgWarning(D_WAAP_SERIALIZE) << error;
return genError(error);
}
string remoteFilePath = maybeRemoteFilePath.unpack();
vector<string> files = {remoteFilePath};
updateState(files);
dbgDebug(D_WAAP_SERIALIZE) << "updated state from remote file: " << remoteFilePath;
return Maybe<void>();
}
bool SerializeToLocalAndRemoteSyncBase::shouldNotSync() const
{
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
return mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase();
}
bool SerializeToLocalAndRemoteSyncBase::shouldSendSyncNotification() const
{
return getSettingWithDefault<bool>(true, "features", "learningLeader") &&
((m_type == "CentralizedData") ==
(getProfileAgentSettingWithDefault<bool>(false, "agent.learning.centralLogging")));
}
void SerializeToLocalAndRemoteSyncBase::syncWorker()
{
dbgInfo(D_WAAP_SERIALIZE) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
@@ -814,7 +894,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() || !postData()) {
if (shouldNotSync() || !postData()) {
dbgDebug(D_WAAP_SERIALIZE)
<< "Did not synchronize the data. for asset: "
<< m_assetId
@@ -834,12 +914,31 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
if (m_lastProcessedModified == "")
{
dbgTrace(D_WAAP_SERIALIZE) << "check if remote service is operational";
RemoteFilesList remoteFiles = getRemoteProcessedFilesList();
if (!remoteFiles.getFilesMetadataList().empty())
Maybe<string> maybeTimestamp = getStateTimestamp();
if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty())
{
m_lastProcessedModified = remoteFiles.getFilesMetadataList()[0].modified;
m_lastProcessedModified = maybeTimestamp.unpack();
dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service: " << m_lastProcessedModified;
}
else
{
dbgWarning(D_WAAP_SERIALIZE) << "Failed to get state timestamp from remote service: "
<< maybeTimestamp.getErr();
maybeTimestamp = getStateTimestampByListing();
if (maybeTimestamp.ok() && !maybeTimestamp.unpack().empty())
{
m_lastProcessedModified = maybeTimestamp.unpack();
dbgInfo(D_WAAP_SERIALIZE) << "First sync by remote service using listing: " << m_lastProcessedModified;
}
else
{
dbgWarning(D_WAAP_SERIALIZE)
<< "Failed to get state timestamp from remote service by listing: "
<< maybeTimestamp.getErr()
<< " skipping syncWorker for assetId='"
<< m_assetId << "', owner='" << m_owner << "'";
}
}
}
// check if learning service is enabled
@@ -854,6 +953,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
return;
}
// TODO: add should send sync notification function (e.g. do not sync if not leader)
if (mode == OrchestrationMode::HYBRID) {
dbgDebug(D_WAAP_SERIALIZE) << "detected running in standalone mode";
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
@@ -876,7 +977,8 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
if (!ok) {
dbgWarning(D_WAAP_SERIALIZE) << "failed to send learning notification";
}
} else {
} else if (shouldSendSyncNotification())
{
SyncLearningNotificationObject syncNotification(m_assetId, m_type, getWindowId());
dbgDebug(D_WAAP_SERIALIZE) << "sending sync notification: " << syncNotification;
@@ -1014,8 +1116,40 @@ void SerializeToLocalAndRemoteSyncBase::mergeProcessedFromRemote()
I_MainLoop::RoutineType::Offline,
[&]()
{
RemoteFilesList processedFiles = getProcessedFilesList();
pullProcessedData(processedFiles.getFilesList());
// Instrumentation breadcrumbs to help diagnose startup crash inside this routine
dbgTrace(D_WAAP_SERIALIZE) << "start routine for assetId='" << m_assetId
<< "', owner='" << m_owner << "'";
try {
auto success = updateStateFromRemoteFile();
if (!success.ok()) {
dbgInfo(D_WAAP_SERIALIZE) << "direct state file unavailable: "
<< success.getErr() << ". Falling back to listing.";
RemoteFilesList remoteFiles = getProcessedFilesList();
if (remoteFiles.getFilesList().empty()) {
dbgWarning(D_WAAP_SERIALIZE) << "no remote processed files";
return;
}
const auto &md_list = remoteFiles.getFilesMetadataList();
if (!md_list.empty()) {
m_lastProcessedModified = md_list[0].modified;
} else {
dbgWarning(D_WAAP_SERIALIZE) << "metadata list empty while files list not empty";
}
updateState(remoteFiles.getFilesList());
dbgInfo(D_WAAP_SERIALIZE) << "updated state from remote files. Last modified: "
<< m_lastProcessedModified;
} else {
dbgTrace(D_WAAP_SERIALIZE) << "updated state via direct remote file";
}
} catch (const JsonError &j) {
dbgError(D_WAAP_SERIALIZE) << "JsonError caught: '" << j.getMsg()
<< "' assetId='" << m_assetId << "' owner='" << m_owner << "'";
throw std::runtime_error(std::string("mergeProcessedFromRemote JsonError: ") + j.getMsg());
} catch (const std::exception &e) {
dbgError(D_WAAP_SERIALIZE) << "std::exception caught: " << e.what()
<< " assetId='" << m_assetId << "' owner='" << m_owner << "'";
throw; // Let mainloop handle termination with detailed message
}
},
"Merge processed data from remote for asset Id: " + m_assetId + ", owner:" + m_owner
);
@@ -1052,3 +1186,32 @@ SerializeToLocalAndRemoteSyncBase::getSharedStorageHost()
}
return defaultSharedStorageHost;
}
string SerializeToLocalAndRemoteSyncBase::getStateTimestampPath()
{
return m_remotePath + "/internal/lastModified.data";
}
Maybe<string> SerializeToLocalAndRemoteSyncBase::getStateTimestamp()
{
string timestampPath = getStateTimestampPath();
if (timestampPath.empty()) {
dbgWarning(D_WAAP_SERIALIZE) << "Cannot get state timestamp - invalid path";
return genError("Invalid timestamp path");
}
StateTimestampRetriever timestampRetriever;
bool isSuccessful = sendObject(
timestampRetriever,
HTTPMethod::GET,
getUri() + "/" + timestampPath);
if (!isSuccessful) {
dbgDebug(D_WAAP_SERIALIZE) << "Failed to get state timestamp file from: " << timestampPath;
return genError("Failed to retrieve timestamp file from: " + timestampPath);
}
dbgDebug(D_WAAP_SERIALIZE) << "Retrieved state timestamp: " << timestampRetriever.getStateTimestamp().unpack()
<< " from path: " << timestampPath;
return timestampRetriever.getStateTimestamp().unpack();
}

444
components/security_apps/waap/waap_clib/Signatures.cc
View File

@@ -12,17 +12,23 @@
// limitations under the License.
#include "Signatures.h"
#include "AssertionRegexes.h"
#include "agent_core_utilities.h"
#include "debug.h"
#include "waap.h"
#include <cstdlib> // for getenv
#include <cstring> // for strcmp
#include <fstream>
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN);
USE_DEBUG_FLAG(D_WAAP_HYPERSCAN);
typedef picojson::value::object JsObj;
typedef picojson::value JsVal;
typedef picojson::value::array JsArr;
typedef std::map<std::string, std::vector<std::string>> filtered_parameters_t;
static std::vector<std::string> to_strvec(const picojson::value::array &jsV)
{
std::vector<std::string> r;
@@ -76,8 +82,7 @@ static std::map<std::string, Regex*> to_regexmap(const picojson::value::object&
static filtered_parameters_t to_filtermap(const picojson::value::object &JsObj)
{
filtered_parameters_t result;
for (auto it = JsObj.begin(); it != JsObj.end(); ++it)
{
for (auto it = JsObj.begin(); it != JsObj.end(); ++it) {
const std::string parameter = it->first;
const picojson::value::array &arr = it->second.get<picojson::value::array>();
result[parameter] = to_strvec(arr);
@@ -215,9 +220,14 @@ Signatures::Signatures(const std::string& filepath) :
to_strset(sigsSource["remove_keywords_always"].get<JsArr>())),
user_agent_prefix_re(sigsSource["user_agent_prefix_re"].get<std::string>()),
binary_data_kw_filter(sigsSource["binary_data_kw_filter"].get<std::string>()),
wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get<std::string>())
wbxml_data_kw_filter(sigsSource["wbxml_data_kw_filter"].get<std::string>()),
m_hyperscanInitialized(false)
{
// Only preprocess hyperscan patterns if hyperscan is enabled
bool should_use_hyperscan = Signatures::shouldUseHyperscan();
if (should_use_hyperscan) {
preprocessHyperscanPatterns();
}
}
Signatures::~Signatures()
@@ -229,6 +239,284 @@ bool Signatures::fail()
return error;
}
// Static helper to process assertion flags for a pattern (for testing and internal use)
std::string
Signatures::processAssertions(const std::string &groupName, const std::string &pattern, AssertionFlags &flags)
{
std::string processed = pattern;
// Use regexes from AssertionRegexes namespace to detect assertions at start/end of the pattern string
using namespace Waap::AssertionRegexes;
boost::smatch match;
// Start assertions - only a single '(' can precede
if (boost::regex_search(processed, match, reStartNonWordBehind) && match.position() >= 0) {
flags.setFlag(AssertionFlag::START_NON_WORD_BEHIND);
processed = boost::regex_replace(processed, reStartNonWordBehind, std::string(""));
}
// Path traversal start assertion
if (boost::regex_search(processed, match, rePathTraversalStart) && match.position() >= 0) {
flags.setFlag(AssertionFlag::PATH_TRAVERSAL_START);
processed = boost::regex_replace(processed, rePathTraversalStart, std::string(""));
}
// End assertions - only a single ')' can follow
if (boost::regex_search(processed, match, reEndNonWordAhead) && match.position() >= 0) {
flags.setFlag(AssertionFlag::END_NON_WORD_AHEAD);
processed = boost::regex_replace(processed, reEndNonWordAhead, std::string(""));
} else if (boost::regex_search(processed, match, reEndNonWordSpecial) && match.position() >= 0) {
flags.setFlag(AssertionFlag::END_NON_WORD_SPECIAL);
processed = boost::regex_replace(processed, reEndNonWordSpecial, std::string(""));
}
// Path traversal end assertion
if (boost::regex_search(processed, match, rePathTraversalEnd) && match.position() >= 0) {
flags.setFlag(AssertionFlag::PATH_TRAVERSAL_END);
processed = boost::regex_replace(processed, rePathTraversalEnd, std::string(""));
}
// wildcard evasion regex group name starts with evasion_wildcard_regex
if (groupName.find("evasion_wildcard_regex") == 0) {
flags.setFlag(AssertionFlag::WILDCARD_EVASION);
}
return processed;
}
// Extracts the group name from a regex pattern string (e.g., (?P<groupName>...))
std::string Signatures::extractGroupName(const std::string &pattern) {
boost::regex namedGroupRegex(R"(\(\?P<([^>]+)>)");
boost::smatch match;
if (boost::regex_search(pattern, match, namedGroupRegex)) {
return match[1].str();
}
return "";
}
void Signatures::preprocessHyperscanPatterns()
{
std::map<std::string, size_t> categoryCount;
// Helper function to check if a pattern is hyperscan compatible
auto isHyperscanCompatible = [&categoryCount](const std::string &pattern) -> bool {
// Hyperscan doesn't support certain regex features that we can't easily convert
static const std::vector<std::string> incompatibleFeatures = {
R"((?!\w)", R"((?<!\w)", R"((?=\w)", R"((?<=\w)", // Lookahead/lookbehind assertions for \w
R"((?!)", R"((?<!)", R"((?=)", R"((?<=)", // Lookahead/lookbehind assertions
R"((?>)", R"((?&)", R"((?|)", R"((?P<)", // Atomic groups, named groups, and branching
R"((?R)" // Recursion
};
for (const auto &feature : incompatibleFeatures) {
if (pattern.find(feature) != std::string::npos) {
dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible feature found: " << feature << " in pattern: " << pattern;
categoryCount[feature]++;
return false;
}
}
boost::regex backrefRegex(R"(\(\\\d+\))");
if (boost::regex_search(pattern, backrefRegex)) {
dbgInfo(D_WAAP_HYPERSCAN) << "Incompatible backreference found: " << pattern;
categoryCount["backreference"]++;
return false;
}
return true;
};
// Helper function to convert regex pattern to hyperscan-compatible format
auto convertToHyperscanPattern = [](const std::string &originalPattern) -> std::string {
std::string converted = originalPattern;
// Remove named group syntax - convert (?P<name>...) to ...
boost::regex namedGroupRegex(R"(\(\?P<[^>]+>)");
if (boost::regex_search(converted, namedGroupRegex)) {
std::string end_str = ")";
if (converted.back() == ')') {
converted.pop_back(); // Remove the trailing ')'
end_str = "";
}
converted = boost::regex_replace(converted, namedGroupRegex, end_str);
}
// Handle atomic groups first (before removing word boundaries)
// Hyperscan doesn't support atomic groups, so we need to convert them
// Convert (?>\b) to nothing (remove word boundary atomic groups)
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\b\))"), std::string(""));
// Convert (?>\B) to nothing (remove non-word boundary atomic groups)
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\\B\))"), std::string(""));
// Convert empty atomic groups (?>) to nothing
converted = boost::regex_replace(converted, boost::regex(R"(\(\?\>\))"), std::string(""));
// // Now remove remaining word boundaries (not supported by Hyperscan)
// // At this point, any \b or \B that was inside atomic groups has been handled above
// converted = boost::regex_replace(converted, boost::regex(R"(\\b)"), std::string(""));
// converted = boost::regex_replace(converted, boost::regex(R"(\\B)"), std::string(""));
return converted;
};
// Helper function to get patterns from sigsSource for each category
auto getCommonPatternsForCategory = [this](const std::string &category,
const std::string &regexSource) -> std::vector<std::string> {
std::vector<std::string> patterns;
// Map regexSource/category to the JSON key in sigsSource
std::string key;
if (regexSource == "specific_acuracy_keywords_regex" || category == "specific_accuracy") {
key = "specific_acuracy_keywords_regex_list";
} else if (regexSource == "words_regex" || category == "keywords") {
key = "words_regex_list";
} else if (regexSource == "pattern_regex" || category == "patterns") {
key = "pattern_regex_list";
} else {
// Fallback: allow passing the exact key name
key = regexSource;
dbgDebug(D_WAAP_HYPERSCAN) << "Unknown category/regexSource: " << category << "/" << regexSource
<< ". Using regexSource as key.";
}
// Fetch patterns directly from sigsSource if available
auto it = sigsSource.find(key);
if (it != sigsSource.end()) {
try {
patterns = to_strvec(it->second.get<JsArr>());
} catch (...) {
// If the type is unexpected, return empty and continue gracefully
patterns.clear();
dbgWarning(D_WAAP_HYPERSCAN) << "Unexpected type for key: " << key;
}
}
return patterns;
};
// Process specific_acuracy_keywords_regex patterns
std::vector<std::string> incompatiblePatterns;
{
auto patterns = getCommonPatternsForCategory("specific_accuracy", "specific_acuracy_keywords_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "specific_accuracy";
hsPattern.regexSource = "specific_acuracy_keywords_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "specific_accuracy_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_keywordHyperscanPatterns.push_back(hsPattern);
m_keywordAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
// Process words_regex patterns
{
auto patterns = getCommonPatternsForCategory("keywords", "words_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "keywords";
hsPattern.regexSource = "words_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "keywords_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_keywordHyperscanPatterns.push_back(hsPattern);
m_keywordAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
// Process pattern_regex patterns
{
auto patterns = getCommonPatternsForCategory("patterns", "pattern_regex");
for (const auto &pattern : patterns) {
AssertionFlags flags;
std::string groupName = extractGroupName(pattern);
std::string processedPattern = convertToHyperscanPattern(pattern);
std::string hyperscanPattern = processAssertions(groupName, processedPattern, flags);
if (hyperscanPattern != pattern) {
dbgTrace(D_WAAP_HYPERSCAN) << pattern << " -> " << hyperscanPattern;
}
if (isHyperscanCompatible(hyperscanPattern)) {
HyperscanPattern hsPattern;
hsPattern.originalPattern = pattern;
hsPattern.hyperscanPattern = hyperscanPattern;
hsPattern.category = "patterns";
hsPattern.regexSource = "pattern_regex";
hsPattern.groupName = groupName;
if (hsPattern.groupName.empty()) {
hsPattern.groupName = "patterns_match";
}
hsPattern.isFastReg = (hsPattern.groupName.find("fast_reg") != std::string::npos);
hsPattern.isEvasion = (hsPattern.groupName.find("evasion") != std::string::npos);
m_patternHyperscanPatterns.push_back(hsPattern);
m_patternAssertionFlags.push_back(flags);
} else {
incompatiblePatterns.push_back(pattern);
}
}
}
dbgInfo(D_WAAP_HYPERSCAN) << "Preprocessed Hyperscan patterns: "
<< "keywords=" << m_keywordHyperscanPatterns.size()
<< ", patterns=" << m_patternHyperscanPatterns.size()
<< ", incompatible=" << incompatiblePatterns.size();
for (const auto &it : categoryCount) {
dbgInfo(D_WAAP_HYPERSCAN) << "Feature: " << it.first << ", Count: " << it.second;
}
// Convert incompatible patterns to PmWordSet for traditional regex processing
if (m_regexPreconditions && !incompatiblePatterns.empty()) {
for (const auto &pattern : incompatiblePatterns) {
Waap::RegexPreconditions::WordIndex wordIndex = m_regexPreconditions->getWordByRegex(pattern);
if (wordIndex != Waap::RegexPreconditions::emptyWordIndex) {
m_incompatiblePatternsPmWordSet.insert(wordIndex);
}
}
dbgInfo(D_WAAP_HYPERSCAN) << "Created PmWordSet for " << m_incompatiblePatternsPmWordSet.size()
<< " incompatible patterns (from " << incompatiblePatterns.size() << " total)";
}
}
picojson::value::object Signatures::loadSource(const std::string &waapDataFileName)
{
picojson::value doc;
@@ -258,11 +546,153 @@ picojson::value::object Signatures::loadSource(const std::string& waapDataFileNa
ss >> doc;
if (!picojson::get_last_error().empty()) {
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" <<
picojson::get_last_error() << "').";
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '"
<< picojson::get_last_error() << "').";
error = true; // flag an error
return picojson::value::object();
}
return doc.get<picojson::value::object>()["waap_signatures"].get<picojson::value::object>();
}
const std::vector<Signatures::HyperscanPattern> &Signatures::getKeywordHyperscanPatterns() const
{
return m_keywordHyperscanPatterns;
}
const std::vector<Signatures::HyperscanPattern> &Signatures::getPatternHyperscanPatterns() const
{
return m_patternHyperscanPatterns;
}
const std::vector<Signatures::AssertionFlags> &Signatures::getKeywordAssertionFlags() const
{
return m_keywordAssertionFlags;
}
const std::vector<Signatures::AssertionFlags> &Signatures::getPatternAssertionFlags() const
{
return m_patternAssertionFlags;
}
const Waap::RegexPreconditions::PmWordSet &Signatures::getIncompatiblePatternsPmWordSet() const
{
return m_incompatiblePatternsPmWordSet;
}
void Signatures::processRegexMatch(const std::string &groupName, const std::string &groupValue, std::string &word,
std::vector<std::string> &keyword_matches,
Waap::Util::map_of_stringlists_t &found_patterns, bool longTextFound,
bool binaryDataFound) const
{
std::string group = groupName;
if (group == "") {
return; // skip unnamed group
}
const std::string &value = groupValue;
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: group name='" << group << "' value='" << value << "', word='" << word
<< "':";
if (group.find("fast_reg") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found '*fast_reg*' in group name";
if (group.find("evasion") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found both 'fast_reg' and 'evasion' in group name.";
word = "encoded_" + repr_uniq(value);
if (word == "encoded_") {
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "checkRegex: empty word after repr_uniq: resetting word to 'character_encoding'"
" and group to 'evasion'.";
word = "character_encoding";
} else if (Waap::Util::str_isalnum(word)) {
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "checkRegex: isalnum word after repr_uniq: resetting group to 'evasion'.";
// If the found match is alphanumeric (we've seen strings like "640x480" match)
// we still should assume evasion but it doesn't need to include "fast_reg",
// which would cause unconditional report to stage2 and hit performance...
// This is why we remove the word "fast_reg" from the group name.
group = "evasion";
}
if (longTextFound) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: longTextFound so resetting group name to 'longtext'";
group = "longtext";
}
} else {
word = group;
}
}
// In sequences detected as "longTextFound" or "longBinaryFound", do not add words in the
// "keyword_matches" list that:
// - starts with "encoded_"
// - or startswith("\")
// - or equal to "character_encoding"
if ((longTextFound || binaryDataFound) &&
(word == "character_encoding" || word.substr(0, 1) == "\\" || word.substr(0, 8) == "encoded_")) {
// For now, do not skip
// TODO - check if skipping improves detection
dbgTrace(D_WAAP_SAMPLE_SCAN) << "longText/binaryData found with character_encoding";
} else if (binaryDataFound && (isShortWord(word) || isShortHtmlTag(word) ||
NGEN::Regex::regexMatch(__FILE__, __LINE__, group, binary_data_kw_filter))) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding group='" << group << "', word='" << word
<< "' - due to binary data";
return;
} else if ((std::find(keyword_matches.begin(), keyword_matches.end(), word) == keyword_matches.end())) {
// python: if (word not in current_matches): current_matches.append(word)
keyword_matches.push_back(word);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "added keyword match for group='" << group << "', value='" << value
<< "', word='" << word << "'";
}
// python:
// if group not in found_patterns:
// found_patterns[group]=[]
if (found_patterns.find(group) == found_patterns.end()) {
found_patterns[group] = std::vector<std::string>();
}
// python:
// if value not in found_patterns[group]:
// found_patterns[group].append(value)
if (std::find(found_patterns[group].begin(), found_patterns[group].end(), value) == found_patterns[group].end()) {
found_patterns[group].push_back(value);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "added pattern match for group='" << group << "', value='" << value
<< "', word='" << word << "'";
}
}
bool Signatures::isHyperscanInitialized() const
{
return m_hyperscanInitialized;
}
void Signatures::setHyperscanInitialized(bool initialized)
{
m_hyperscanInitialized = initialized;
}
bool Signatures::shouldUseHyperscan(bool force)
{
// This can be controlled by environment variable or configuration
static bool useHyperscan = false;
#ifdef USE_HYPERSCAN
static bool checked = false;
if (!checked || force) {
// Check environment variable first
const char *env = getenv("WAAP_USE_HYPERSCAN");
if (env) {
useHyperscan = (strcmp(env, "1") == 0 || strcasecmp(env, "true") == 0);
dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage set by environment: " << useHyperscan;
} else {
// Default to false to maintain backward compatibility - Hyperscan is opt-in
useHyperscan = false;
dbgDebug(D_WAAP_SAMPLE_SCAN) << "Hyperscan usage default (disabled): " << useHyperscan;
}
checked = true;
}
#endif // USE_HYPERSCAN
return useHyperscan;
}

138
components/security_apps/waap/waap_clib/Signatures.h
View File

@@ -16,19 +16,40 @@
#include "Waf2Regex.h"
#include "picojson.h"
#include "flags.h"
#include <boost/regex.hpp>
class Signatures {
private:
// json parsed sources (not really needed once data is loaded)
picojson::value::object sigsSource;
bool error;
public:
// Enum for zero-length assertion flags
enum class AssertionFlag {
START_WORD_BEHIND = 0, // (?<=\w)
START_NON_WORD_BEHIND, // (?<!\w)
END_WORD_AHEAD, // (?=\w)
END_NON_WORD_AHEAD, // (?!\w)
END_NON_WORD_SPECIAL, // (?=[^\w?<>:=]|$)
PATH_TRAVERSAL_START, // (?<![\.,:])
PATH_TRAVERSAL_END, // (?![\.,:])
WILDCARD_EVASION, // (slashes and question mark must be present)
COUNT // Must be last for Flags template
};
// Use the Flags template from utilities/flags.h for assertion flags
using AssertionFlags = Flags<AssertionFlag>;
static std::string extractGroupName(const std::string &pattern);
static std::string processAssertions(const std::string &groupName,
const std::string &pattern,
AssertionFlags &flags);
Signatures(const std::string& filepath);
~Signatures();
bool fail();
picojson::value::object sigsSource;
bool error;
std::shared_ptr<Waap::RegexPreconditions> m_regexPreconditions;
// Regexes loaded from compiled signatures
@@ -81,8 +102,113 @@ public:
const boost::regex binary_data_kw_filter;
const boost::regex wbxml_data_kw_filter;
private:
picojson::value::object loadSource(const std::string& waapDataFileName);
// Pre-compiled Hyperscan patterns and metadata for performance optimization
struct HyperscanPattern {
std::string originalPattern;
std::string hyperscanPattern;
std::string groupName;
std::string category;
std::string regexSource;
bool isFastReg;
bool isEvasion;
HyperscanPattern() : isFastReg(false), isEvasion(false) {}
};
// Pre-processed hyperscan patterns for each regex category
std::vector<HyperscanPattern> m_keywordHyperscanPatterns;
std::vector<HyperscanPattern> m_patternHyperscanPatterns;
// Assertion flags corresponding to each pattern (same indices as above vectors)
std::vector<AssertionFlags> m_keywordAssertionFlags;
std::vector<AssertionFlags> m_patternAssertionFlags;
// Getter methods for precompiled patterns
const std::vector<HyperscanPattern>& getKeywordHyperscanPatterns() const;
const std::vector<HyperscanPattern>& getPatternHyperscanPatterns() const;
// Getter methods for assertion flags
const std::vector<AssertionFlags>& getKeywordAssertionFlags() const;
const std::vector<AssertionFlags>& getPatternAssertionFlags() const;
// PmWordSet for incompatible patterns that need to use traditional regex scanning
Waap::RegexPreconditions::PmWordSet m_incompatiblePatternsPmWordSet;
// Getter method for incompatible patterns PmWordSet
const Waap::RegexPreconditions::PmWordSet& getIncompatiblePatternsPmWordSet() const;
// Hyperscan initialization state management
bool isHyperscanInitialized() const;
void setHyperscanInitialized(bool initialized);
// Check if Hyperscan should be used (based on configuration)
static bool shouldUseHyperscan(bool force = false);
void processRegexMatch(
const std::string &groupName,
const std::string &groupValue,
std::string &word,
std::vector<std::string> &keyword_matches,
Waap::Util::map_of_stringlists_t &found_patterns,
bool longTextFound,
bool binaryDataFound
) const;
private:
picojson::value::object loadSource(const std::string& waapDataFileName);
void preprocessHyperscanPatterns();
bool m_hyperscanInitialized;
};
inline std::string repr_uniq(const std::string & value) {
std::string result;
char hist[256];
memset(&hist, 0, sizeof(hist));
for (std::string::const_iterator pC = value.begin(); pC != value.end(); ++pC) {
unsigned char ch = (unsigned char)(*pC);
// Only take ASCII characters that are not alphanumeric, and each character only once
if (ch <= 127 && !isalnum(ch) && hist[ch] == 0) {
// Convert low ASCII characters to their C/C++ printable equivalent
// (used for easier viewing. Also, binary data causes issues with ElasticSearch)
switch (ch) {
case 0x07: result += "\\a"; break;
case 0x08: result += "\\b"; break;
case 0x09: result += "\\t"; break;
case 0x0A: result += "\\n"; break;
case 0x0B: result += "\\v"; break;
case 0x0C: result += "\\f"; break;
case 0x0D: result += "\\r"; break;
case 0x5C: result += "\\\\"; break;
case 0x27: result += "\\\'"; break;
case 0x22: result += "\\\""; break;
case 0x3F: result += "\\\?"; break;
default: {
if (ch >= 32) {
result += (char)ch;
}
else {
char buf[16];
sprintf(buf, "\\" "x%02X", ch);
result += buf;
}
}
}
hist[ch] = 1;
}
}
return result;
}
inline bool isShortWord(const std::string& word) {
return word.size() <= 2;
}
inline bool isShortHtmlTag(const std::string& word) {
return !word.empty() && word.size() <= 4 && word[0] == '<' && word[word.size() - 1] == '>';
}
#endif

2
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@@ -225,7 +225,7 @@ void
WaapAttackTypesMetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
if (data.blockType == FORCE_EXCEPTION) {
dbgInfo(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed";
dbgDebug(D_WAAP) << "Data block type is FORCE_EXCEPTION, no update needed";
return;
}

183
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc
View File

@@ -28,22 +28,52 @@ TrustedSourcesConfidenceCalculator::TrustedSourcesConfidenceCalculator(
path,
(remotePath == "") ? remotePath : remotePath + "/Trust",
assetId,
"TrustedSourcesConfidenceCalculator")
"TrustedSourcesConfidenceCalculator"),
m_persistent_state(),
m_incremental_logger(std::make_shared<KeyValSourceLogger>())
{
restore();
}
bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t minSources) const
{
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state first (accumulated data from previous syncs)
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
auto sourceSetItr = sourceCtrItr->second.find(value);
if (sourceSetItr != sourceCtrItr->second.end())
{
size_t persistent_sources = sourceSetItr->second.size();
if (persistent_sources >= minSources) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << sourceSetItr->second.size();
return sourceSetItr->second.size() >= minSources;
<< " : " << value << " is " << persistent_sources << " (persistent only)";
return true;
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr->second.find(src) == sourceSetItr->second.end()) {
incremental_sources++;
}
}
}
}
}
size_t total_sources = persistent_sources + incremental_sources;
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << total_sources << " (persistent: " << persistent_sources
<< ", incremental: " << incremental_sources << ")";
return total_sources >= minSources;
}
else
{
@@ -52,6 +82,18 @@ bool TrustedSourcesConfidenceCalculator::is_confident(Key key, Val value, size_t
}
else
{
// Check if data exists only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(value);
if (incr_set_itr != incr_ctr_itr->second.end()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "The number of trusted sources for " << key
<< " : " << value << " is " << incr_set_itr->second.size() << " (incremental only)";
return incr_set_itr->second.size() >= minSources;
}
}
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return false;
@@ -76,25 +118,36 @@ private:
S2C_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger)
};
class TrsutedSourcesLogger : public RestGetFile
class TrustedSourcesLogger : public RestGetFile
{
public:
TrsutedSourcesLogger(const TrustedSourcesConfidenceCalculator::KeyValSourceLogger& _logger)
: logger(_logger)
TrustedSourcesLogger(std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> _logger_ptr)
: logger_ptr(_logger_ptr)
{
logger = move(*logger_ptr);
}
private:
std::shared_ptr<TrustedSourcesConfidenceCalculator::KeyValSourceLogger> logger_ptr;
C2S_PARAM(TrustedSourcesConfidenceCalculator::KeyValSourceLogger, logger);
};
bool TrustedSourcesConfidenceCalculator::postData()
{
if (m_incremental_logger->empty())
{
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "No data to post, skipping";
return true; // Nothing to post
}
std::string url = getPostDataUrl();
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the data to: " << url;
mergeIncrementalToPersistent();
TrustedSourcesLogger logger(m_incremental_logger);
// Clear and reset incremental logger for next cycle
m_incremental_logger = std::make_shared<KeyValSourceLogger>();
TrsutedSourcesLogger logger(m_logger);
bool ok = sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
@@ -109,7 +162,7 @@ void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the window data for trusted sources";
std::string url = getPostDataUrl();
std::string sentFile = url.erase(0, url.find_first_of('/') + 1);
for (auto file : files)
for (const auto &file : files) // Use const reference to avoid copying
{
if (file == sentFile)
{
@@ -135,19 +188,22 @@ void TrustedSourcesConfidenceCalculator::processData()
{
}
void TrustedSourcesConfidenceCalculator::updateState(const std::vector<std::string> &files)
{
m_logger.clear();
pullProcessedData(files);
}
Maybe<std::string> TrustedSourcesConfidenceCalculator::getRemoteStateFilePath() const
{
return m_remotePath + "/remote/data.data";
}
void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std::string> &files) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Fetching the logger object for trusted sources";
bool pull_ok = false;
for (auto file: files) {
for (const auto &file : files) { // Use const reference
GetTrustedFile getTrustFile;
bool res = sendObjectWithRetry(getTrustFile,
bool res = sendObject(getTrustFile,
HTTPMethod::GET,
getUri() + "/" + file);
pull_ok |= res;
@@ -165,7 +221,9 @@ void TrustedSourcesConfidenceCalculator::postProcessedData()
std::string url = getUri() + "/" + m_remotePath + "/processed/data.data";
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Sending the processed data to: " << url;
TrsutedSourcesLogger logger(m_logger);
// Send persistent state as processed data
auto logger_ptr = std::make_shared<TrustedSourcesConfidenceCalculator::KeyValSourceLogger>(m_persistent_state);
TrustedSourcesLogger logger(logger_ptr);
sendNoReplyObjectWithRetry(logger,
HTTPMethod::PUT,
url);
@@ -176,19 +234,63 @@ TrustedSourcesConfidenceCalculator::ValuesSet TrustedSourcesConfidenceCalculator
size_t minSources) const
{
ValuesSet values;
auto sourceCtrItr = m_logger.find(key);
if (sourceCtrItr != m_logger.end())
// Check persistent state
auto sourceCtrItr = m_persistent_state.find(key);
if (sourceCtrItr != m_persistent_state.end())
{
for (auto sourceSetItr : sourceCtrItr->second)
{
if (sourceSetItr.second.size() >= minSources)
size_t persistent_sources = sourceSetItr.second.size();
if (persistent_sources >= minSources)
{
values.insert(sourceSetItr.first);
continue; // No need to check incremental logger if we already have enough sources
}
// Also check incremental logger for recent data
size_t incremental_sources = 0;
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
auto incr_set_itr = incr_ctr_itr->second.find(sourceSetItr.first);
if (incr_set_itr != incr_ctr_itr->second.end()) {
// Count unique sources (avoid double counting)
for (const auto &src : incr_set_itr->second) {
if (sourceSetItr.second.find(src) == sourceSetItr.second.end()) {
incremental_sources++;
}
}
}
}
}
if (persistent_sources + incremental_sources >= minSources)
{
values.insert(sourceSetItr.first);
}
}
}
else
{
// Also check values that exist only in incremental logger
if (m_incremental_logger) {
auto incr_ctr_itr = m_incremental_logger->find(key);
if (incr_ctr_itr != m_incremental_logger->end()) {
for (auto incr_set_itr : incr_ctr_itr->second) {
// Skip if already processed in persistent state
if (sourceCtrItr != m_persistent_state.end() &&
sourceCtrItr->second.find(incr_set_itr.first) != sourceCtrItr->second.end()) {
continue;
}
if (incr_set_itr.second.size() >= minSources) {
values.insert(incr_set_itr.first);
}
}
}
}
if (values.empty()) {
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to find the key(" << key << ")";
}
return values;
@@ -198,7 +300,7 @@ void TrustedSourcesConfidenceCalculator::serialize(std::ostream& stream)
{
cereal::JSONOutputArchive archive(stream);
archive(cereal::make_nvp("version", 2), cereal::make_nvp("logger", m_logger));
archive(cereal::make_nvp("version", 3), cereal::make_nvp("persistent_state", m_persistent_state));
}
void TrustedSourcesConfidenceCalculator::deserialize(std::istream &stream)
@@ -218,9 +320,15 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
switch (version)
{
case 3:
{
archive(cereal::make_nvp("persistent_state", m_persistent_state));
break;
}
case 2:
{
archive(cereal::make_nvp("logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("logger", m_persistent_state));
break;
}
case 1:
@@ -229,13 +337,14 @@ void TrustedSourcesConfidenceCalculator::deserialize(std::istream& stream)
archive(cereal::make_nvp("logger", logger));
for (auto &log : logger)
{
m_logger[normalize_param(log.first)] = log.second;
m_persistent_state[normalize_param(log.first)] = log.second;
}
break;
}
case 0:
{
archive(cereal::make_nvp("m_logger", m_logger));
// Legacy: load into persistent state
archive(cereal::make_nvp("m_logger", m_persistent_state));
break;
}
default:
@@ -254,7 +363,7 @@ void TrustedSourcesConfidenceCalculator::mergeFromRemote(const KeyValSourceLogge
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Registering the source: " << src
<< " for the value: " << sourcesItr.first << " and the key: " << srcCounterItr.first;
m_logger[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
m_persistent_state[normalize_param(srcCounterItr.first)][sourcesItr.first].insert(src);
}
}
}
@@ -269,10 +378,28 @@ void TrustedSourcesConfidenceCalculator::log(Key key, Val value, Source source)
<< key
<< " from the source: "
<< source;
m_logger[key][value].insert(source);
(*m_incremental_logger)[key][value].insert(source);
}
void TrustedSourcesConfidenceCalculator::reset()
{
m_logger.clear();
m_persistent_state.clear();
m_incremental_logger->clear();
}
void TrustedSourcesConfidenceCalculator::mergeIncrementalToPersistent()
{
// Merge incremental data into persistent state (same logic as in postData but without network operations)
for (const auto &keyEntry : *m_incremental_logger) {
const std::string &key = keyEntry.first;
for (const auto &valueEntry : keyEntry.second) {
const std::string &value = valueEntry.first;
for (const std::string &source : valueEntry.second) {
m_persistent_state[key][value].insert(source);
}
}
}
// Clear incremental logger after merging
m_incremental_logger->clear();
}

12
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.h
View File

@@ -19,6 +19,8 @@
USE_DEBUG_FLAG(D_WAAP);
// TODO PHASE3: remove inheritance from SerializeToLocalAndRemoteSyncBase
// this class is responsible for logging trusted sources indicators matches (without validation)
class TrustedSourcesConfidenceCalculator : public SerializeToLocalAndRemoteSyncBase
{
@@ -41,6 +43,7 @@ public:
virtual void postProcessedData();
virtual void pullProcessedData(const std::vector<std::string> &files);
virtual void updateState(const std::vector<std::string> &files);
virtual Maybe<std::string> getRemoteStateFilePath() const override;
ValuesSet getConfidenceValues(const Key &key, size_t minSources) const;
@@ -52,6 +55,13 @@ public:
void log(Key key, Val value, Source source);
void reset();
// Helper method for testing - merges incremental data to persistent state without network operations
void mergeIncrementalToPersistent();
private:
KeyValSourceLogger m_logger;
// Persistent state - accumulated trusted sources data across sync cycles
KeyValSourceLogger m_persistent_state;
// Incremental logger - only new data since last sync (gets moved during postData)
std::shared_ptr<KeyValSourceLogger> m_incremental_logger;
};

19
components/security_apps/waap/waap_clib/TuningDecision.cc
View File

@@ -26,7 +26,7 @@ USE_DEBUG_FLAG(D_WAAP);
TuningDecision::TuningDecision(const string& remotePath)
:
m_remotePath(replaceAllCopy(remotePath + "/tuning", "//", "/")),
m_remotePath(replaceAllCopy(remotePath + "/tuning/decisions.data", "//", "/")),
m_baseUri()
{
if (remotePath == "")
@@ -35,7 +35,7 @@ TuningDecision::TuningDecision(const string& remotePath)
}
Singleton::Consume<I_MainLoop>::by<WaapComponent>()->addRecurringRoutine(
I_MainLoop::RoutineType::System,
chrono::minutes(10),
chrono::minutes(30),
[&]() { updateDecisions(); },
"Get tuning updates"
);
@@ -118,27 +118,16 @@ TuningDecisionType TuningDecision::convertDecisionType(string decisionTypeStr)
void TuningDecision::updateDecisions()
{
TuningEvents tuningEvents;
RemoteFilesList tuningDecisionFiles;
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) {
m_baseUri = "/api/";
} else {
m_baseUri = "/storage/waap/";
}
dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri;
bool isSuccessful = sendObject(tuningDecisionFiles,
HTTPMethod::GET,
m_baseUri + "?list-type=2&prefix=" + m_remotePath);
if (!isSuccessful || tuningDecisionFiles.getFilesList().empty())
{
dbgDebug(D_WAAP) << "Failed to get the list of files";
return;
}
if (!sendObject(tuningEvents,
HTTPMethod::GET,
m_baseUri + tuningDecisionFiles.getFilesList()[0]))
m_baseUri + m_remotePath))
{
return;
}
@@ -181,7 +170,7 @@ TuningDecision::getSharedStorageHost()
char* sharedStorageHost = getenv(SHARED_STORAGE_HOST_ENV_NAME);
if (sharedStorageHost != NULL) {
shared_storage_host = string(sharedStorageHost);
dbgInfo(D_WAAP) << "shared storage host is set to " << shared_storage_host;
dbgDebug(D_WAAP) << "shared storage host is set to " << shared_storage_host;
return shared_storage_host;
}
dbgWarning(D_WAAP) << "shared storage host is not set. using default: " << defaultSharedStorageHost;

25
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc
View File

@@ -91,10 +91,16 @@ void TypeIndicatorFilter::registerKeywords(const std::string& key, const std::st
{
std::set<std::string> types = m_pWaapAssetState->getSampleType(sample);
std::string source = pTransaction->getSourceIdentifier();
std::string trusted_source = getTrustedSource(pTransaction);
auto trusted_source_maybe = getTrustedSource(pTransaction);
std::string trusted_source = trusted_source_maybe.ok() ? trusted_source_maybe.unpack() : "";
for (const std::string &type : types)
{
if (type == "long_random_text")
{
// Skip long random text types, as they are not useful for filtering
continue;
}
if (type == "local_file_path")
{
std::string location = IndicatorsFiltersManager::getLocationFromKey(key, pTransaction);
@@ -153,3 +159,20 @@ std::set<std::string> TypeIndicatorFilter::getParamTypes(const std::string& cano
}
return types;
}
bool TypeIndicatorFilter::shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction) {
// Retrieve the sample from the transaction
std::string sample = pTransaction->getLastScanSample();
// Get the types associated with the sample
std::set<std::string> sampleTypes = m_pWaapAssetState->getSampleType(sample);
// Check if any of the sample types should be tracked
for (const auto& type : sampleTypes) {
if (m_confidence_calc.shouldTrackParameter(key, type)) {
return true;
}
}
return false;
}

2
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.h
View File

@@ -41,7 +41,7 @@ public:
virtual void registerKeywords(const std::string& key, Waap::Keywords::KeywordsSet& keyword,
IWaf2Transaction* pTransaction);
bool shouldTrack(const std::string& key, const IWaf2Transaction* pTransaction);
void registerKeywords(const std::string& key, const std::string& sample, IWaf2Transaction* pTransaction);
void loadParams(std::shared_ptr<Waap::Parameters::WaapParameters> pParams);

583
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -13,6 +13,8 @@
// #define WAF2_LOGGING_ENABLE (does performance impact)
#include "WaapAssetState.h"
#include "WaapHyperscanEngine.h"
#include "Signatures.h"
#include "Waf2Regex.h"
#include "debug.h"
#include "Waf2Util.h"
@@ -53,8 +55,7 @@ print_filtered(std::string title, const std::set<std::string>& ignored_set, cons
if (ignored_set.find(word) == ignored_set.end()) {
// not in ignored_set
dbgTrace(D_WAAP_SAMPLE_SCAN) << "+'" << word << "'";
}
else {
} else {
// in ignored set
dbgTrace(D_WAAP_SAMPLE_SCAN) << "-'" << word << "'";
}
@@ -102,11 +103,13 @@ static const boost::regex utf_evasion_for_dot_regex(utf_evasion_for_dot_helper);
static const std::string sqli_comma_evasion_regex_helper = "\"\\s*,\\s*\"";
static const boost::regex sqli_comma_evasion_regex(sqli_comma_evasion_regex_helper);
WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState,
const std::string& waapDataFileName,
const std::string& id) :
WaapAssetState(pWaapAssetState->m_Signatures,
WaapAssetState::WaapAssetState(
const std::shared_ptr<WaapAssetState> &pWaapAssetState, const std::string &waapDataFileName, const std::string &id
) :
WaapAssetState(
pWaapAssetState->m_Signatures,
waapDataFileName,
pWaapAssetState->m_hyperscanEngine,
pWaapAssetState->m_cleanValuesCache.capacity(),
pWaapAssetState->m_suspiciousValuesCache.capacity(),
pWaapAssetState->m_sampleTypeCache.capacity(),
@@ -122,17 +125,19 @@ WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAsset
clearRateLimitingState();
clearSecurityHeadersState();
clearErrorLimitingState();
}
);
});
}
WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
WaapAssetState::WaapAssetState(
std::shared_ptr<Signatures> signatures,
const std::string &waapDataFileName,
std::shared_ptr<WaapHyperscanEngine> hyperscanEngine,
size_t cleanValuesCacheCapacity,
size_t suspiciousValuesCacheCapacity,
size_t sampleTypeCacheCapacity,
const std::string& assetId) :
m_Signatures(signatures),
m_hyperscanEngine(hyperscanEngine),
m_waapDataFileName(waapDataFileName),
m_assetId(assetId),
m_requestsMonitor(nullptr),
@@ -536,59 +541,6 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
return text;
}
inline std::string repr_uniq(const std::string & value) {
std::string result;
char hist[256];
memset(&hist, 0, sizeof(hist));
for (std::string::const_iterator pC = value.begin(); pC != value.end(); ++pC) {
unsigned char ch = (unsigned char)(*pC);
// Only take ASCII characters that are not alphanumeric, and each character only once
if (ch <= 127 && !isalnum(ch) && hist[ch] == 0) {
// Convert low ASCII characters to their C/C++ printable equivalent
// (used for easier viewing. Also, binary data causes issues with ElasticSearch)
switch (ch) {
case 0x07: result += "\\a"; break;
case 0x08: result += "\\b"; break;
case 0x09: result += "\\t"; break;
case 0x0A: result += "\\n"; break;
case 0x0B: result += "\\v"; break;
case 0x0C: result += "\\f"; break;
case 0x0D: result += "\\r"; break;
case 0x5C: result += "\\\\"; break;
case 0x27: result += "\\\'"; break;
case 0x22: result += "\\\""; break;
case 0x3F: result += "\\\?"; break;
default: {
if (ch >= 32) {
result += (char)ch;
}
else {
char buf[16];
sprintf(buf, "\\" "x%02X", ch);
result += buf;
}
}
}
hist[ch] = 1;
}
}
return result;
}
static bool isShortWord(const std::string &word)
{
return word.size() <= 2;
}
static bool isShortHtmlTag(const std::string &word)
{
return !word.empty() && word.size() <= 3 && word[0] == '<';
}
void
WaapAssetState::checkRegex(
const SampleValue &sample,
@@ -621,105 +573,69 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
}
for (std::vector<RegexMatch::MatchGroup>::const_iterator pGroup = match.groups.begin() + 1;
pGroup != match.groups.end();
++pGroup) {
std::string group = pGroup->name;
if (group == "") {
continue; // skip unnamed group
pGroup != match.groups.end(); ++pGroup) {
m_Signatures->processRegexMatch(pGroup->name, pGroup->value, word, keyword_matches, found_patterns,
longTextFound, binaryDataFound);
}
const std::string& value = pGroup->value;
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: group name='" << group <<
"' value='" << value << "', word='" << word << "':";
// python:
// if 'fast_reg' in group:
// if 'evasion' in group:
// word = repr(str(''.join(set(value))))
// else:
// word =group
if (group.find("fast_reg") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "checkRegex: found '*fast_reg*' in group name";
if (group.find("evasion") != std::string::npos) {
dbgTrace(D_WAAP_SAMPLE_SCAN) <<
"checkRegex: found both 'fast_reg' and 'evasion' in group name.";
word = "encoded_" + repr_uniq(value);
// Normally, the word added to the keyword_matches list contain the character sequence.
// However, sometimes (for example in case the sequence contained only unicode characters),
// after running repr_uniq() the word will remain empty string. In this case leave
// something meaningful/readable there.
if (word == "encoded_") {
dbgTrace(D_WAAP_SAMPLE_SCAN) <<
"checkRegex: empty word after repr_uniq: resetting word to 'character_encoding'"
" and group to 'evasion'.";
word = "character_encoding";
}
else if (Waap::Util::str_isalnum(word)) {
dbgTrace(D_WAAP_SAMPLE_SCAN) <<
"checkRegex: isalnum word after repr_uniq: resetting group to 'evasion'.";
// If the found match is alphanumeric (we've seen strings like "640x480" match)
// we still should assume evasion but it doesn't need to include "fast_reg",
// which would cause unconditional report to stage2 and hit performance...
// This is why we remove the word "fast_reg" from the group name.
group = "evasion";
}
if (longTextFound) {
dbgTrace(D_WAAP_SAMPLE_SCAN) <<
"checkRegex: longTextFound so resetting group name to 'longtext'";
group = "longtext";
}
}
else {
word = group;
}
}
// In sequences detected as "longTextFound" or "longBinaryFound", do not add words in the
// "keyword_matches" list that:
// - starts with "encoded_"
// - or startswith("\")
// - or equal to "character_encoding"
if ((longTextFound || binaryDataFound) &&
(word == "character_encoding" || word.substr(0, 1) == "\\" || word.substr(0, 8) == "encoded_")) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding keyword '" << word << "' because longtext was found";
}
else if (binaryDataFound && (isShortWord(word) || isShortHtmlTag(word) ||
NGEN::Regex::regexMatch(__FILE__, __LINE__, group, m_Signatures->binary_data_kw_filter))) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "Not adding group='" << group << "', word='" << word <<
"' - due to binary data";
continue;
}
else if ((std::find(
keyword_matches.begin(),
keyword_matches.end(),
word) == keyword_matches.end())) {
// python: if (word not in current_matches): current_matches.append(word)
keyword_matches.push_back(word);
void WaapAssetState::performStandardRegexChecks(
const SampleValue &sample,
Waf2ScanResult &res,
bool longTextFound,
bool binaryDataFound,
bool includePatternRegex) const
{
// Check if Hyperscan should be used based on configuration and availability
if (m_hyperscanEngine && m_hyperscanEngine->isInitialized()) {
// Use Hyperscan implementation for compatible patterns
dbgTrace(D_WAAP_SAMPLE_SCAN) << "using Hyperscan engine, includePatternRegex=" << includePatternRegex;
m_hyperscanEngine->scanSample(sample, res, longTextFound, binaryDataFound, true, includePatternRegex);
}
// python:
// if group not in found_patterns:
// found_patterns[group]=[]
if (found_patterns.find(group) == found_patterns.end()) {
found_patterns[group] = std::vector<std::string>();
// Always run regular regex checks for patterns incompatible with Hyperscan
// When Hyperscan is used, pmSet contains only the regex patterns that couldn't be compiled with Hyperscan
// When Hyperscan is not used, pmSet contains all patterns found by Aho-Corasick precondition scan
checkRegex(sample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(sample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, longTextFound,
binaryDataFound);
if (includePatternRegex) {
checkRegex(sample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, longTextFound,
binaryDataFound);
}
}
// python:
// if value not in found_patterns[group]:
// found_patterns[group].append(value)
if (std::find(
found_patterns[group].begin(),
found_patterns[group].end(),
value
) == found_patterns[group].end()) {
found_patterns[group].push_back(value);
}
void WaapAssetState::performResponseRegexChecks(
const SampleValue &sample,
Waf2ScanResult &res,
bool longTextFound,
bool binaryDataFound) const
{
checkRegex(sample, m_Signatures->resp_body_words_regex_list, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(sample, m_Signatures->resp_body_pattern_regex_list, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
void WaapAssetState::performPatternRegexChecks(
const SampleValue &sample,
Waf2ScanResult &res,
bool longTextFound,
bool binaryDataFound) const
{
// Check if Hyperscan should be used based on configuration and availability
if (m_hyperscanEngine && m_hyperscanEngine->isInitialized()) {
// Use Hyperscan implementation for compatible patterns
dbgTrace(D_WAAP_SAMPLE_SCAN) << "using Hyperscan engine";
m_hyperscanEngine->scanSample(sample, res, longTextFound, binaryDataFound, false, true);
}
// Always run regular regex checks for patterns incompatible with Hyperscan
// When Hyperscan is used, pmSet contains only the regex patterns that couldn't be compiled with Hyperscan
// When Hyperscan is not used, pmSet contains all patterns found by Aho-Corasick precondition scan
checkRegex(sample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, longTextFound,
binaryDataFound);
}
// TODO:: implement onload mechanism.
@@ -747,7 +663,8 @@ static void calcRepeatAndWordsCount(const std::string &line, unsigned int &repea
}
static void calcRepetitionAndProbing(Waf2ScanResult &res, const std::set<std::string> *ignored_keywords,
const std::string &line, bool &detectedRepetition, bool &detectedProbing, unsigned int &wordsCount)
const std::string &line, bool &detectedRepetition, bool &detectedProbing,
unsigned int &wordsCount)
{
unsigned int repeat;
calcRepeatAndWordsCount(line, repeat, wordsCount);
@@ -763,22 +680,21 @@ static void calcRepetitionAndProbing(Waf2ScanResult &res, const std::set<std::st
size_t keywords_num = countNotInSet(res.keyword_matches, *ignored_keywords);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "wordsCount: " << wordsCount << ", repeat=" << repeat
<< ", keyword_matches(num=" << keywords_num << ", size=" << res.keyword_matches.size() << ")";
<< ", keyword_matches(num=" << keywords_num << ", size=" << res.keyword_matches.size()
<< ")";
if (!detectedProbing // res.keyword_matches.size()
&& keywords_num + 2 > wordsCount
// res.keyword_matches.size()
&& keywords_num != 0)
{
dbgTrace(D_WAAP_SAMPLE_SCAN) << "probing detected: keywords_num=" << keywords_num <<
", wordsCount=" << wordsCount;
&& keywords_num != 0) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "probing detected: keywords_num=" << keywords_num
<< ", wordsCount=" << wordsCount;
detectedProbing = true;
res.keyword_matches.push_back("probing");
}
}
void
WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
void WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
{
// Test for long value without spaces (these can often cause false alarms)
if (m_Signatures->nospaces_long_value_re.hasMatch(res.unescaped_line)) {
@@ -790,12 +706,9 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
if (m_Signatures->ignored_for_nospace_long_value.find(word) !=
m_Signatures->ignored_for_nospace_long_value.end()) {
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "Removing keyword '"
<< word
<< "' because nospaces_long_value was found";
<< "Removing keyword '" << word << "' because nospaces_long_value was found";
it = v.erase(it);
}
else {
} else {
++it;
}
}
@@ -813,8 +726,7 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
// into the object easier.
std::string
WaapAssetState::nicePrint(Waf2ScanResult &res) const
std::string WaapAssetState::nicePrint(Waf2ScanResult &res) const
{
std::string result = "Waf2ScanResult:\n";
result += "keyword_matches:\n";
@@ -907,18 +819,7 @@ WaapAssetState::apply(
if (scanStage == "resp_body") {
res.clear();
SampleValue sample(line, nullptr);
checkRegex(sample,
m_Signatures->resp_body_words_regex_list,
res.keyword_matches,
res.found_patterns,
false,
false);
checkRegex(sample,
m_Signatures->resp_body_pattern_regex_list,
res.keyword_matches,
res.found_patterns,
false,
false);
performResponseRegexChecks(sample, res, false, false);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply(): response body " <<
(res.keyword_matches.empty() ? "is not" : "is") << " suspicious";
return !res.keyword_matches.empty();
@@ -927,18 +828,7 @@ WaapAssetState::apply(
if (scanStage == "resp_header") {
res.clear();
SampleValue sample(line, nullptr);
checkRegex(sample,
m_Signatures->resp_body_words_regex_list,
res.keyword_matches,
res.found_patterns,
false,
false);
checkRegex(sample,
m_Signatures->resp_body_pattern_regex_list,
res.keyword_matches,
res.found_patterns,
false,
false);
performResponseRegexChecks(sample, res, false, false);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply(): response header " <<
(res.keyword_matches.empty() ? "is not" : "is") << " suspicious";
return !res.keyword_matches.empty();
@@ -992,9 +882,7 @@ WaapAssetState::apply(
ignored_keywords = &m_Signatures->url_ignored_keywords;
ignored_patterns = &m_Signatures->url_ignored_patterns;
isUrlScanStage = true;
}
else if ((scanStage.size() == 6 && scanStage == "header") ||
(scanStage.size() == 6 && scanStage == "cookie")) {
} else if ((scanStage.size() == 6 && scanStage == "header") || (scanStage.size() == 6 && scanStage == "cookie")) {
if (m_Signatures->header_ignored_re.hasMatch(line)) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line << "'): ignored for header.";
@@ -1033,8 +921,8 @@ WaapAssetState::apply(
if (isOnLoad) {
// Skip values that are too short
if (line.length() < 3) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line <<
"'): skipping: did not pass the length check.";
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "WaapAssetState::apply('" << line << "'): skipping: did not pass the length check.";
if (shouldCache) {
m_cleanValuesCache.insert(CacheKey(line, scanStage, isBinaryData, splitType.ok() ? *splitType : ""));
@@ -1068,8 +956,8 @@ WaapAssetState::apply(
// Skip values that are longer than 10 characters, and match allowed_text_re regex
if (line.length() > 10) {
if (m_Signatures->allowed_text_re.hasMatch(line) > 0) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "WaapAssetState::apply('" << line <<
"'): matched on allowed_text - ignoring.";
dbgTrace(D_WAAP_SAMPLE_SCAN)
<< "WaapAssetState::apply('" << line << "'): matched on allowed_text - ignoring.";
if (shouldCache) {
m_cleanValuesCache.insert(
@@ -1110,18 +998,10 @@ WaapAssetState::apply(
// Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
// This is done to improve performance of regex matching.
SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
SampleValue unescapedLineSample(res.unescaped_line, m_Signatures.get());
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
checkRegex(
unescapedLineSample,
m_Signatures->specific_acuracy_keywords_regex,
res.keyword_matches,
res.found_patterns,
longTextFound,
binaryDataFound
);
checkRegex(unescapedLineSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, longTextFound,
binaryDataFound);
performStandardRegexChecks(unescapedLineSample, res, longTextFound, binaryDataFound, false);
filterKeywordsDueToLongText(res);
@@ -1160,13 +1040,8 @@ WaapAssetState::apply(
unescaped = "|" + res.unescaped_line;
}
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
filterKeywordsDueToLongText(res);
@@ -1220,21 +1095,15 @@ WaapAssetState::apply(
unescaped += res.unescaped_line.substr(pos); // add tail
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("os_cmd_ev");
os_cmd_ev = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1256,21 +1125,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("quotes_ev");
quotes_ev = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1291,13 +1154,8 @@ WaapAssetState::apply(
unescaped = unescape(unescaped);
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1321,13 +1179,8 @@ WaapAssetState::apply(
unescaped = unescape(unescaped);
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1351,21 +1204,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("comment_ev");
comment_ev = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1390,21 +1237,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("path_traversal");
path_traversal_ev = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1430,13 +1271,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1464,21 +1300,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("quotes_space_evasion");
quoutes_space_evasion = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
@@ -1503,13 +1333,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1533,13 +1358,8 @@ WaapAssetState::apply(
unescaped = unescape(unescaped);
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
evasion_detected = true;
@@ -1549,7 +1369,6 @@ WaapAssetState::apply(
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
if ((res.unescaped_line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(res.unescaped_line)) {
@@ -1563,13 +1382,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, false, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
false, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
false, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, false, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1591,7 +1405,6 @@ WaapAssetState::apply(
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
if ((line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(line)) {
@@ -1604,13 +1417,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, false, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
false, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
false, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, false, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1646,13 +1454,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1664,7 +1467,6 @@ WaapAssetState::apply(
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
if ((line.find("%") != std::string::npos) && evasion_bad_hex_regex.hasMatch(line)) {
@@ -1678,13 +1480,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1710,13 +1507,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1730,7 +1522,6 @@ WaapAssetState::apply(
}
}
if ((line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(line)) {
dbgTrace(D_WAAP_EVASIONS) << "UTF evasion for dot found (%c0%*e) in raw line";
std::string unescaped = line;
@@ -1742,13 +1533,8 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
@@ -1785,13 +1571,20 @@ WaapAssetState::apply(
bool evBinaryDataFound = binaryDataFound;
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, evKeywordMatches,
res.found_patterns, evLongTextFound, evBinaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, evKeywordMatches, evFoundPatterns,
evLongTextFound, evBinaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, evRegexMatches, evFoundPatterns,
evLongTextFound, evBinaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
// Create a temporary result structure to use with our helper function
Waf2ScanResult evRes;
evRes.keyword_matches = evKeywordMatches;
evRes.regex_matches = evRegexMatches;
evRes.found_patterns = evFoundPatterns;
performStandardRegexChecks(unescapedSample, evRes, evLongTextFound, evBinaryDataFound);
// Extract results back to individual variables
evKeywordMatches = evRes.keyword_matches;
evRegexMatches = evRes.regex_matches;
evFoundPatterns = evRes.found_patterns;
}
if (evKeywordMatches.size() != res.keyword_matches.size()) {
@@ -1857,21 +1650,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("evasion");
escape = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
evasion_detected = true;
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
@@ -1934,21 +1721,15 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("evasion");
escape = false;
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
evasion_detected = true;
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
@@ -1970,20 +1751,14 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
SampleValue unescapedSample(unescaped, m_Signatures.get());
performStandardRegexChecks(unescapedSample, res, longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("evasion");
}
else if (!binaryDataFound) {
} else if (!binaryDataFound) {
evasion_detected = true;
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
@@ -2043,8 +1818,7 @@ WaapAssetState::apply(
detectedProbing) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "pre-suspicion found.";
// apply regex signatures
checkRegex(unescapedLineSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
performPatternRegexChecks(unescapedLineSample, res, longTextFound, binaryDataFound);
// python:
// if len(regex_matches) and 'probing' not in keyword_matches:
@@ -2069,13 +1843,11 @@ WaapAssetState::apply(
// python:
// if 'acuracy' in patterns and not url:
if (Waap::Util::find_in_map_of_stringlists_keys("acur", res.found_patterns))
{
if (Waap::Util::find_in_map_of_stringlists_keys("acur", res.found_patterns)) {
acuracy = 1;
// search for "high_acuracy" or "hi_acur" signature names
if (Waap::Util::find_in_map_of_stringlists_keys("high", res.found_patterns) ||
Waap::Util::find_in_map_of_stringlists_keys("hi_acur", res.found_patterns))
{
Waap::Util::find_in_map_of_stringlists_keys("hi_acur", res.found_patterns)) {
acuracy = 2;
}
}
@@ -2090,20 +1862,19 @@ WaapAssetState::apply(
print_filtered("patterns", *ignored_patterns, res.regex_matches);
print_found_patterns(res.found_patterns);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "before decision: keywords(num=" << keywords_num << ", size=" <<
res.keyword_matches.size() << "); regex(num=" << regex_num << ", size=" << res.regex_matches.size() <<
"; acuracy=" << acuracy << "; score=" << score << "; forceReport=" << forceReport << "; probing=" <<
detectedProbing << "; repetition=" << detectedRepetition << "; 'fast_reg' in found_patterns: " <<
Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "before decision: keywords(num=" << keywords_num
<< ", size=" << res.keyword_matches.size() << "); regex(num=" << regex_num
<< ", size=" << res.regex_matches.size() << "; acuracy=" << acuracy
<< "; score=" << score << "; forceReport=" << forceReport
<< "; probing=" << detectedProbing << "; repetition=" << detectedRepetition
<< "; 'fast_reg' in found_patterns: "
<< Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns);
#endif
// python:
// if (keywords_num+acuracy+2*regex_num)>2 or special_patten in patterns or
// 'fast_reg' in patterns or 'probing' in keyword_matches or 'repetition' in keyword_matches:
if (score > 2 ||
forceReport ||
detectedProbing ||
detectedRepetition ||
if (score > 2 || forceReport || detectedProbing || detectedRepetition ||
Waap::Util::find_in_map_of_stringlists_keys("fast_reg", res.found_patterns)) {
dbgTrace(D_WAAP_SAMPLE_SCAN) << "apply(): suspicion found (score=" << score << ").";
@@ -2134,7 +1905,8 @@ void WaapAssetState::updateScores()
scoreBuilder.snap();
}
std::string WaapAssetState::getWaapDataFileName() const {
std::string WaapAssetState::getWaapDataFileName() const
{
return m_waapDataFileName;
}
@@ -2143,7 +1915,8 @@ std::map<std::string, std::vector<std::string>>& WaapAssetState::getFilterVerbos
return m_filtered_keywords_verbose;
}
std::string WaapAssetState::getWaapDataDir() const {
std::string WaapAssetState::getWaapDataDir() const
{
size_t lastSlash = m_waapDataFileName.find_last_of('/');
std::string sigsFilterDir = ((lastSlash == std::string::npos) ?
m_waapDataFileName : m_waapDataFileName.substr(0, lastSlash));
@@ -2266,8 +2039,7 @@ std::set<std::string> WaapAssetState::getSampleType(const std::string & sample)
types.insert("binary_input");
}
if (types.empty())
{
if (types.empty()) {
types.insert("unknown");
m_sampleTypeCache.insert(sample);
}
@@ -2313,33 +2085,26 @@ void WaapAssetState::filterKeywordsByParameters(
{
dbgTrace(D_WAAP_ASSET_STATE) << "filter keywords based on parameter name: " << parameter_name;
auto filter_parameters_itr = m_Signatures->filter_parameters.find(parameter_name);
if (filter_parameters_itr != m_Signatures->filter_parameters.end())
{
if (filter_parameters_itr != m_Signatures->filter_parameters.end()) {
dbgTrace(D_WAAP_ASSET_STATE) << "Found keywords to filter based on parameter name";
const auto &vec = filter_parameters_itr->second;
for (auto keyword_to_filter : vec)
{
for (auto keyword_to_filter : vec) {
auto keywords_set_itr = keywords_set.find(keyword_to_filter);
if (keywords_set_itr != keywords_set.end())
{
if (keywords_set_itr != keywords_set.end()) {
dbgTrace(D_WAAP_ASSET_STATE) << "Filtering keyword: " << keyword_to_filter;
keywords_set.erase(keyword_to_filter);
}
}
}
else
{
} else {
dbgTrace(D_WAAP_ASSET_STATE) << "No keywords need to be filtered for this parameter";
}
}
void WaapAssetState::removeKeywords(Waap::Keywords::KeywordsSet &keywords_set)
{
for (auto &keyword_to_remove : m_Signatures->remove_keywords_always)
{
for (auto &keyword_to_remove : m_Signatures->remove_keywords_always) {
auto keyword_set_itr = keywords_set.find(keyword_to_remove);
if (keyword_set_itr != keywords_set.end())
{
if (keyword_set_itr != keywords_set.end()) {
dbgTrace(D_WAAP_ASSET_STATE) << "Removing keyword: " << keyword_to_remove << " from keyword set";
keywords_set.erase(keyword_set_itr);
}
@@ -2354,8 +2119,7 @@ void WaapAssetState::removeWBXMLKeywords(Waap::Keywords::KeywordsSet &keywords_s
dbgTrace(D_WAAP_ASSET_STATE) << "Filtering keyword due to wbxml: '" << *it << "'";
filtered_keywords.push_back(*it);
it = keywords_set.erase(it);
}
else {
} else {
++it;
}
}
@@ -2407,4 +2171,3 @@ void WaapAssetState::clearSecurityHeadersState()
{
m_securityHeadersState.reset();
}

22
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -34,6 +34,7 @@
#include "ScanResult.h"
#include "WaapSampleValue.h"
#include "RequestsMonitor.h"
#include "WaapHyperscanEngine.h"
enum space_stage {SPACE_SYNBOL, BR_SYMBOL, BN_SYMBOL, BRN_SEQUENCE, BNR_SEQUENCE, NO_SPACES};
@@ -43,18 +44,34 @@ class WaapAssetState : public boost::noncopyable, public I_WaapAssetState
{
private: //ugly but needed for build
std::shared_ptr<Signatures> m_Signatures;
std::shared_ptr<WaapHyperscanEngine> m_hyperscanEngine;
std::string m_waapDataFileName;
std::map<std::string, std::vector<std::string>> m_filtered_keywords_verbose;
void checkRegex(const SampleValue &sample, const Regex & pattern, std::vector<std::string>& keyword_matches,
Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
// Helper function to perform the common pattern of checkRegex calls (2 or 3 calls based on parameter)
void performStandardRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound, bool includePatternRegex = true) const;
// Helper function to perform response body/header pattern of two checkRegex calls
void performResponseRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound) const;
// Helper function to perform pattern subset match only
void performPatternRegexChecks(const SampleValue &sample, Waf2ScanResult &res,
bool longTextFound, bool binaryDataFound) const;
void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
std::string nicePrint(Waf2ScanResult &res) const;
public:
// Load and compile signatures from file
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& waapDataFileName,
explicit WaapAssetState(
std::shared_ptr<Signatures> signatures,
const std::string &waapDataFileName,
std::shared_ptr<WaapHyperscanEngine> hyperscanEngine = nullptr,
size_t cleanCacheCapacity = SIGS_APPLY_CLEAN_CACHE_CAPACITY,
size_t suspiciousCacheCapacity = SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
size_t sampleTypeCacheCapacity = SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
@@ -111,7 +128,6 @@ public:
void clearErrorLimitingState();
void clearSecurityHeadersState();
std::shared_ptr<Waap::RateLimiting::State>& getRateLimitingState();
std::shared_ptr<Waap::RateLimiting::State>& getErrorLimitingState();
std::shared_ptr<Waap::SecurityHeaders::State>& getSecurityHeadersState();

22
components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc
View File

@@ -55,6 +55,7 @@ void WaapAssetStatesManager::setAssetDirectoryPath(const std::string &assetDirec
WaapAssetStatesManager::Impl::Impl() :
m_signatures(nullptr),
m_hyperscanEngine(nullptr),
m_basicWaapSigs(nullptr),
m_AssetBasedWaapSigs(),
m_assetDirectoryPath(BACKUP_DIRECTORY_PATH)
@@ -67,18 +68,33 @@ WaapAssetStatesManager::Impl::~Impl()
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapDataFileName)
{
if (m_signatures && !m_signatures->fail() && m_basicWaapSigs)
if (m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs)
{
// already initialized successfully.
return true;
}
try {
m_signatures = std::make_shared<Signatures>(waapDataFileName);
// Initialize Hyperscan engine
m_hyperscanEngine = std::make_shared<WaapHyperscanEngine>();
if (!Signatures::shouldUseHyperscan()) {
dbgTrace(D_WAAP) << "Hyperscan disabled by configuration, will use PCRE2";
} else if (!m_hyperscanEngine->initialize(m_signatures)) {
dbgTrace(D_WAAP) << "Hyperscan initialization failed, will use PCRE2";
} else {
m_signatures->setHyperscanInitialized(true);
dbgTrace(D_WAAP) << "Hyperscan initialized successfully";
}
m_basicWaapSigs = std::make_shared<WaapAssetState>(
m_signatures,
waapDataFileName,
m_hyperscanEngine,
SIGS_APPLY_CLEAN_CACHE_CAPACITY,
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY);
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
"");
}
catch (std::runtime_error & e) {
// TODO:: properly handle component initialization failure
@@ -89,7 +105,7 @@ bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapData
return false;
}
return m_signatures && !m_signatures->fail() && m_basicWaapSigs;
return m_signatures && !m_signatures->fail() && m_hyperscanEngine && m_basicWaapSigs;
}
std::shared_ptr<WaapAssetState> WaapAssetStatesManager::Impl::getWaapAssetStateGlobal()

2
components/security_apps/waap/waap_clib/WaapAssetStatesManager.h
View File

@@ -15,6 +15,7 @@
#include "singleton.h"
#include "Signatures.h"
#include "WaapHyperscanEngine.h"
#include <string>
#include <memory>
#include <unordered_map>
@@ -65,6 +66,7 @@ private:
const std::string& instanceId);
std::shared_ptr<Signatures> m_signatures;
std::shared_ptr<WaapHyperscanEngine> m_hyperscanEngine;
std::shared_ptr<WaapAssetState> m_basicWaapSigs;
std::unordered_map<std::string, std::shared_ptr<WaapAssetState>> m_AssetBasedWaapSigs;
std::string m_assetDirectoryPath;

2
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@@ -26,7 +26,7 @@ set<string> WaapConfigAPI::assets_ids_aggregation{};
bool
WaapConfigAPI::getWaapAPIConfig(WaapConfigAPI& ngenAPIConfig) {
auto &maybe_ngen_config = getConfiguration<WaapConfigAPI>(
auto &maybe_ngen_config = getConfigurationWithCache<WaapConfigAPI>(
"WAAP",
"WebAPISecurity"
);

1
components/security_apps/waap/waap_clib/WaapConfigApi.h
View File

@@ -34,6 +34,7 @@ public:
static bool getWaapAPIConfig(WaapConfigAPI &ngenAPIConfig);
static void notifyAssetsCount();
static void clearAssetsCount();
virtual WaapConfigType getType() const override { return WaapConfigType::API; }
private:

5
components/security_apps/waap/waap_clib/WaapConfigApplication.cc
View File

@@ -23,6 +23,8 @@ set<string> WaapConfigApplication::assets_ids{};
set<string> WaapConfigApplication::assets_ids_aggregation{};
bool WaapConfigApplication::getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig) {
// Only get tenant/profile id if debug is active
if (isDebugRequired(TRACE, D_WAAP)) {
auto maybe_tenant_id = Singleton::Consume<I_Environment>::by<WaapConfigApplication>()->get<string>(
"ActiveTenantId"
);
@@ -32,7 +34,8 @@ bool WaapConfigApplication::getWaapSiteConfig(WaapConfigApplication& ngenSiteCon
string tenant_id = (maybe_tenant_id.ok() ? *maybe_tenant_id : "not found");
string profile_id = (maybe_profile_id.ok() ? *maybe_profile_id : "not found");
dbgTrace(D_WAAP) << "Tenant ID: " << tenant_id << ", Profile ID: " << profile_id;
auto &maybe_ngen_config = getConfiguration<WaapConfigApplication>(
}
auto &maybe_ngen_config = getConfigurationWithCache<WaapConfigApplication>(
"WAAP",
"WebApplicationSecurity"
);

1
components/security_apps/waap/waap_clib/WaapConfigApplication.h
View File

@@ -39,6 +39,7 @@ public:
static bool getWaapSiteConfig(WaapConfigApplication& ngenSiteConfig);
static void notifyAssetsCount();
static void clearAssetsCount();
virtual WaapConfigType getType() const override { return WaapConfigType::Application; }
private:
static const std::string s_PracticeSubType;

2
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@@ -103,7 +103,7 @@ const std::vector<std::string>& ExceptionsByPractice::getExceptionsOfPractice(De
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_web_app_ids;
default:
dbgError(D_WAAP) <<
dbgDebug(D_WAAP) <<
"Can't find practice type for exceptions by practice: " <<
practiceType <<
", return web app exceptions";

18
components/security_apps/waap/waap_clib/WaapRegexPreconditions.cc
View File

@@ -26,21 +26,24 @@ namespace Waap {
// Register empty string work under known index
registerWord("");
std::string pm_list_name = "preconditions";
std::string pm_keys_name = "precondition_keys";
// The key should always be there unless data file is corrupted (but there's a unit test that tests exactly
// that!)
if (jsObj.find("preconditions") == jsObj.end()) {
if (jsObj.find(pm_list_name) == jsObj.end()) {
dbgError(D_WAAP_REGEX) << "Error loading regex preconditions (signatures data file corrupt?)...";
error = true;
return;
}
if (jsObj.find("precondition_keys") == jsObj.end()) {
if (jsObj.find(pm_keys_name) == jsObj.end()) {
dbgError(D_WAAP_REGEX) << "Error loading regex precondition sets (signatures data file corrupt?)...";
error = true;
return;
}
auto preconditions = jsObj.at("preconditions").get<picojson::value::object>();
auto preconditions = jsObj.at(pm_list_name).get<picojson::value::object>();
// Loop over pre-conditions (rules) and load them
dbgTrace(D_WAAP_REGEX) << "Loading regex preconditions...";
@@ -142,7 +145,7 @@ namespace Waap {
// Build full list of words to load into aho-corasick pattern matcher
dbgTrace(D_WAAP_REGEX) << "Loading regex precondition_keys into Aho-Corasick pattern matcher...";
auto preconditionKeys = jsObj.at("precondition_keys").get<picojson::value::array>();
auto preconditionKeys = jsObj.at(pm_keys_name).get<picojson::value::array>();
std::set<PMPattern> pmPatterns;
for (const auto &preconditionKey : preconditionKeys) {
@@ -167,18 +170,19 @@ namespace Waap {
// Initialize the aho-corasick pattern matcher with the patterns
Maybe<void> pmHookStatus = m_pmHook.prepare(pmPatterns);
if (!pmHookStatus.ok()) {
dbgError(D_WAAP_REGEX) << "Aho-Corasick engine failed to load!";
error = true;
return;
}
dbgTrace(D_WAAP_REGEX) << "Aho-Corasick engine loaded.";
dbgTrace(D_WAAP_REGEX) << "Aho-corasick pattern matching engine initialized!";
}
RegexPreconditions::~RegexPreconditions() {
// No Hyperscan resource management here; handled by HyperscanHook
}
bool Waap::RegexPreconditions::isNoRegexPattern(const std::string &pattern) const
{
return m_noRegexPatterns.find(pattern) != m_noRegexPatterns.end();

45
components/security_apps/waap/waap_clib/WaapRegexPreconditions.h
View File

@@ -22,9 +22,13 @@
#include <stdint.h>
#include <string>
// Forward declaration for test friend class
class TestableRegexPreconditions;
namespace Waap {
class RegexPreconditions
{
friend class TestableRegexPreconditions; // Allow test access to private methods
public:
typedef size_t WordIndex;
static const WordIndex emptyWordIndex; // special word index used to index the "impossible" empty word
@@ -37,8 +41,29 @@ namespace Waap {
public:
typedef std::unordered_set<WordIndex> PmWordSet;
struct WordInfo {
WordIndex napostNapreWordIndex;
WordIndex napostWordIndex;
WordIndex napreWordIndex;
WordIndex baseWordIndex;
std::string wordStr;
bool noRegex;
WordInfo()
:
napostNapreWordIndex(emptyWordIndex),
napostWordIndex(emptyWordIndex),
napreWordIndex(emptyWordIndex),
baseWordIndex(0),
wordStr(),
noRegex(false)
{
}
};
// The constructor builds internal data from Json object. Once built - the object becomes read-only.
RegexPreconditions(const picojson::value::object &jsObj, bool &error);
~RegexPreconditions();
bool isNoRegexPattern(const std::string &pattern) const;
const std::string &getWordStrByWordIndex(WordIndex wordIndex) const;
Waap::RegexPreconditions::WordIndex getWordByRegex(const std::string &pattern) const;
@@ -61,26 +86,6 @@ namespace Waap {
// Aho-Corasick pattern matcher object
PMHook m_pmHook;
struct WordInfo {
WordIndex napostNapreWordIndex;
WordIndex napostWordIndex;
WordIndex napreWordIndex;
WordIndex baseWordIndex;
std::string wordStr;
bool noRegex;
WordInfo()
:
napostNapreWordIndex(emptyWordIndex),
napostWordIndex(emptyWordIndex),
napreWordIndex(emptyWordIndex),
baseWordIndex(0),
wordStr(),
noRegex(false)
{
}
};
WordIndex registerWord(const std::string &wordStr);
std::vector<WordInfo> m_pmWordInfo;
std::map<std::string, WordIndex> m_wordStrToIndex; // TODO:: remove this into throwaway object, no need to keep

29
components/security_apps/waap/waap_clib/WaapSampleValue.cc
View File

@@ -12,19 +12,31 @@
// limitations under the License.
#include "WaapSampleValue.h"
#include "Signatures.h"
#include "debug.h"
SampleValue::SampleValue(const std::string &sample,
const std::shared_ptr<Waap::RegexPreconditions> &regexPreconditions)
USE_DEBUG_FLAG(D_WAAP_SAMPLE_SCAN);
SampleValue::SampleValue(const std::string &sample, const Signatures* signatures)
:
m_sample(sample),
m_regexPreconditions(regexPreconditions),
m_signatures(signatures),
m_pmWordSet()
{
if (m_regexPreconditions) {
if (m_signatures) {
if (m_signatures->m_regexPreconditions) {
if (!m_signatures->isHyperscanInitialized()) {
// Run aho-corasick and related rules once the sample value is known.
// The result pmWordSet is reused later for multiple calls to findMatches on the same sample.
regexPreconditions->pmScan(
m_signatures->m_regexPreconditions->pmScan(
Buffer(m_sample.data(), m_sample.size(), Buffer::MemoryType::STATIC), m_pmWordSet);
} else {
// Add incompatible patterns from Signatures to the PmWordSet so they will be processed
// by traditional regex matching in Regex::findAllMatches
const auto& incompatiblePmWordSet = m_signatures->getIncompatiblePatternsPmWordSet();
m_pmWordSet.insert(incompatiblePmWordSet.begin(), incompatiblePmWordSet.end());
}
}
}
}
@@ -38,5 +50,10 @@ void
SampleValue::findMatches(const Regex &pattern, std::vector<RegexMatch> &matches) const
{
static const size_t maxMatchesPerSignature = 5;
pattern.findAllMatches(m_sample, matches, m_regexPreconditions ? &m_pmWordSet : nullptr, maxMatchesPerSignature);
pattern.findAllMatches(
m_sample,
matches,
(m_signatures && m_signatures->m_regexPreconditions) ? &m_pmWordSet : nullptr,
maxMatchesPerSignature
);
}

6
components/security_apps/waap/waap_clib/WaapSampleValue.h
View File

@@ -21,16 +21,18 @@
#include "WaapRegexPreconditions.h"
#include "buffer.h"
class Signatures;
class SampleValue
{
public:
SampleValue(const std::string &sample, const std::shared_ptr<Waap::RegexPreconditions> &regexPreconditions);
SampleValue(const std::string &sample, const Signatures* signatures);
const std::string &getSampleString() const;
void findMatches(const Regex &pattern, std::vector<RegexMatch> &matches) const;
private:
std::string m_sample;
const std::shared_ptr<Waap::RegexPreconditions> m_regexPreconditions;
const Signatures* m_signatures;
Waap::RegexPreconditions::PmWordSet m_pmWordSet;
};

5
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -287,8 +287,11 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
m_transaction->getAssetState()->logParamHit(res, m_transaction);
std::set<std::string> paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes(
std::set<std::string> paramTypes;
if (m_transaction->getAssetState()->m_filtersMngr != nullptr) {
paramTypes = m_transaction->getAssetState()->m_filtersMngr->getParameterTypes(
IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction));
}
if (paramTypes.size() == 1 && paramTypes.find("local_file_path") != paramTypes.end())
{

2
components/security_apps/waap/waap_clib/WaapTrigger.cc
View File

@@ -89,7 +89,7 @@ const std::vector<std::string>& TriggersByPractice::getTriggersByPractice(Decisi
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_web_app_ids;
default:
dbgError(D_WAAP) <<
dbgDebug(D_WAAP) <<
"Can't find practice type for triggers by practice: " <<
practiceType <<
", return web app triggers";

4
components/security_apps/waap/waap_clib/WaapTrigger.h
View File

@@ -201,7 +201,7 @@ struct Policy {
}
catch (std::runtime_error &e) {
ar.setNextName(nullptr);
dbgInfo(D_WAAP) << "Failed to load triggers per practice, error: " << e.what();
dbgDebug(D_WAAP) << "Failed to load triggers per practice, error: " << e.what();
triggersByPractice = TriggersByPractice();
}
try {
@@ -211,7 +211,7 @@ struct Policy {
}
catch (std::runtime_error &e) {
ar.setNextName(nullptr);
dbgInfo(D_WAAP) << "Failed to load web user response per practice, error: " << e.what();
dbgDebug(D_WAAP) << "Failed to load web user response per practice, error: " << e.what();
responseByPractice = WebUserResponseByPractice();
}
ar(

4
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc
View File

@@ -246,7 +246,8 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
}
// Detect URLEncode value
isUrlEncoded = checkUrlEncoded(cur_val.data(), cur_val.size());
if (isDebugRequired(TRACE, D_WAAP))
{
textual.clear();
textual.append("hasCharSlash = ");
textual +=(hasCharSlash ? "true" : "false");
@@ -279,3 +280,4 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
textual.append("\nhasPercent = ");
textual +=(hasPercent ? "true" : "false");
}
}

71
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -371,6 +371,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
Waf2Transaction::~Waf2Transaction() {
dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser";
delete m_requestBodyParser;
@@ -606,6 +607,7 @@ void Waf2Transaction::set_method(const char* method) {
bool Waf2Transaction::checkIsScanningRequired()
{
bool result = false;
if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
m_siteConfig = &m_ngenAPIConfig;
auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
@@ -639,7 +641,6 @@ bool Waf2Transaction::checkIsScanningRequired()
result = true;
}
}
return result;
}
@@ -652,7 +653,7 @@ bool Waf2Transaction::setCurrentAssetContext()
result |= checkIsScanningRequired();
if (!m_siteConfig) {
dbgWarning(D_WAAP) << "[transaction:" << this << "] "
dbgDebug(D_WAAP) << "[transaction:" << this << "] "
"Failed to set sitePolicy for asset... using the original signatures";
return result;
}
@@ -1573,7 +1574,6 @@ Waf2Transaction::decideFinal(
// decision of (either) API or Application module
bool shouldBlock = false;
// TODO:: base class for both, with common inteface
WaapConfigAPI ngenAPIConfig;
WaapConfigApplication ngenSiteConfig;
@@ -1586,7 +1586,7 @@ Waf2Transaction::decideFinal(
m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION] = getOverrideState(sitePolicy);
// User limits
shouldBlock = (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
shouldBlock = (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP);
}
else if (WaapConfigApplication::getWaapSiteConfig(ngenSiteConfig)) {
dbgTrace(D_WAAP) << "Waf2Transaction::decideFinal(): got relevant Application configuration from the I/S";
@@ -1608,7 +1608,7 @@ Waf2Transaction::decideFinal(
shouldBlock |= m_csrfState.decide(m_methodStr, m_waapDecision, csrfPolicy);
}
// User limits
shouldBlock |= (getUserLimitVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP);
shouldBlock |= (getUserLimitVerdict() == ServiceVerdict::TRAFFIC_VERDICT_DROP);
}
if (mode == 2) {
@@ -1641,7 +1641,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
waapLog << LogField("sourceIP", m_remote_addr);
waapLog << LogField("httpSourceId", m_source_identifier);
if (getProfileAgentSettingWithDefault<bool>(false, "agent.saasProfile.ignoreSourceIP")){
dbgTrace(D_WAAP) << "ignoring remote port in nexus log";
} else {
waapLog << LogField("sourcePort", m_remote_port);
}
waapLog << LogField("httpHostName", m_hostStr);
waapLog << LogField("httpMethod", m_methodStr);
if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
@@ -1757,7 +1761,9 @@ Waf2Transaction::sendLog()
dbgTrace(D_WAAP) << "send log got decision type: " << decision_type;
auto final_decision = m_waapDecision.getDecision(decision_type);
if (!final_decision) {
dbgTrace(D_WAAP) << "send log no decision found, using AUTONOMOUS_SECURITY_DECISION";
final_decision = m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION);
decision_type = AUTONOMOUS_SECURITY_DECISION;
}
std::string attackTypes = buildAttackTypes();
@@ -1853,6 +1859,14 @@ Waf2Transaction::sendLog()
return;
}
auto triggerLog = maybeTriggerLog.unpack();
if(final_decision->shouldForceLog() &&
triggerLog.shouldIgnoreExceptionLog(LogTriggerConf::SecurityType::ThreatPrevention)) {
// If we should ignore exception log, we need to handle it
dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: ignoring exception log";
return;
}
bool send_extended_log = shouldSendExtendedLog(triggerLog, decision_type);
shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision();
// Do not send Detect log if trigger disallows it
@@ -1898,6 +1912,16 @@ Waf2Transaction::sendLog()
return;
}
ReportIS::Severity severity =
decision_type == USER_LIMITS_DECISION ? ReportIS::Severity::HIGH : ReportIS::Severity::CRITICAL;
if (final_decision->shouldForceLog()) {
if (logOverride == OVERRIDE_DROP) {
severity = ReportIS::Severity::MEDIUM;
} else if (logOverride == OVERRIDE_ACCEPT) {
severity = ReportIS::Severity::INFO;
}
}
switch (decision_type)
{
case USER_LIMITS_DECISION: {
@@ -1923,7 +1947,7 @@ Waf2Transaction::sendLog()
"Web Request",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::HIGH,
severity,
Priority::HIGH,
shouldBlock);
@@ -1942,7 +1966,7 @@ Waf2Transaction::sendLog()
"API Request",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::CRITICAL,
severity,
Priority::HIGH,
shouldBlock);
@@ -1972,7 +1996,7 @@ Waf2Transaction::sendLog()
"CSRF Protection",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
Severity::CRITICAL,
severity,
Priority::HIGH,
shouldBlock);
@@ -2277,18 +2301,27 @@ bool Waf2Transaction::decideResponse()
}
auto triggerLog = maybeTriggerLog.unpack();
auto env = Singleton::Consume<I_Environment>::by<Waf2Transaction>();
auto http_chunk_type = env->get<ngx_http_chunk_type_e>("HTTP Chunk type");
auto http_chunk_type = env->get<AttachmentDataType>("HTTP Chunk type");
bool should_send_extended_log = shouldSendExtendedLog(triggerLog) && http_chunk_type.ok();
if (should_send_extended_log &&
*http_chunk_type == ngx_http_chunk_type_e::RESPONSE_CODE &&
*http_chunk_type == AttachmentDataType::RESPONSE_CODE &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
) {
dbgTrace(D_WAAP) << "response body is not active. Disabling extended logging";
should_send_extended_log = false;
} else if (should_send_extended_log &&
*http_chunk_type == ngx_http_chunk_type_e::REQUEST_END &&
*http_chunk_type == AttachmentDataType::REQUEST_END &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseCode) &&
!triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody)
) {
dbgTrace(D_WAAP) << "response code is not active. Disabling extended logging";
should_send_extended_log = false;
} else if (should_send_extended_log &&
*http_chunk_type == AttachmentDataType::RESPONSE_BODY &&
triggerLog.isWebLogFieldActive(LogTriggerConf::WebLogFields::responseBody) &&
m_response_body.length() >= MAX_RESPONSE_BODY_SIZE)
{
dbgTrace(D_WAAP) << "response body collected (" << m_response_body.length() << " bytes). Disabling extended logging";
should_send_extended_log = false;
}
@@ -2347,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// collect sourceip, sourceIdentifier, url
exceptions_dict["sourceIP"].insert(m_remote_addr);
exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
exceptions_dict["url"].insert(getUriStr());
exceptions_dict["url"].insert(m_uriPath);
exceptions_dict["hostName"].insert(m_hostStr);
exceptions_dict["method"].insert(m_methodStr);
@@ -2359,15 +2392,21 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
}
bool isConfigExist = false;
if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap api config found";
m_siteConfig = &m_ngenAPIConfig;
if (m_siteConfig &&
(m_siteConfig->getType() == WaapConfigType::API || m_siteConfig->getType() == WaapConfigType::Application)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap config already exists";
isConfigExist = true;
} else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap web application config found";
m_siteConfig = &m_ngenSiteConfig;
isConfigExist = true;
}
else if (WaapConfigAPI::getWaapAPIConfig(m_ngenAPIConfig)) {
dbgTrace(D_WAAP_OVERRIDE) << "waap api config found";
m_siteConfig = &m_ngenAPIConfig;
isConfigExist = true;
}
std::vector<std::string> site_exceptions;
if (isConfigExist) {
dbgTrace(D_WAAP_OVERRIDE) << "config exists, get override policy";
@@ -2387,7 +2426,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
behaviors.insert(params.begin(), params.end());
}
} else {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
auto exceptions = getConfigurationWithCache<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
return false;

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -63,6 +63,8 @@ typedef void(*subtransaction_cb_t)(Waf2Transaction* subTransaction, void *ctx);
#define OVERRIDE_ACCEPT "Accept"
#define OVERRIDE_DROP "Drop"
#define NEXUS_PORT 99999
class Waf2Transaction :
public IWaf2Transaction,
public TableOpaqueSerialize<Waf2Transaction>,
@@ -134,7 +136,7 @@ public:
virtual std::shared_ptr<WaapAssetState> getAssetState();
virtual const std::string getLocation() const;
ngx_http_cp_verdict_e getUserLimitVerdict();
ServiceVerdict getUserLimitVerdict();
const std::string getUserLimitVerdictStr() const;
const std::string getViolatedUserLimitTypeStr() const;
const std::string getCurrentWebUserResponse();

124
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -22,6 +22,8 @@
#include <memory>
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
#define LOW_REPUTATION_THRESHOLD 4
#define NORMAL_REPUTATION_THRESHOLD 6
@@ -363,7 +365,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
const ReportIS::Priority priority =
Waap::Util::computePriorityFromThreatLevel(autonomousSecurityDecision->getThreatLevel());
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
auto maybeLogTriggerConf = getConfigurationWithCache<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"Web Request",
@@ -445,12 +447,12 @@ void Waf2Transaction::createUserLimitsState()
}
}
ngx_http_cp_verdict_e
ServiceVerdict
Waf2Transaction::getUserLimitVerdict()
{
if (!isUserLimitReached()) {
// Either limit not reached or attack mitigation mode is DISABLED
return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
return ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
std::string msg;
@@ -460,7 +462,7 @@ Waf2Transaction::getUserLimitVerdict()
std::string reason;
reason = " reason: " + getViolatedUserLimitTypeStr();
ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
ServiceVerdict verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
const AttackMitigationMode mode = WaapConfigBase::get_WebAttackMitigationMode(*m_siteConfig);
auto decision = m_waapDecision.getDecision(USER_LIMITS_DECISION);
if (mode == AttackMitigationMode::LEARNING) {
@@ -469,19 +471,19 @@ Waf2Transaction::getUserLimitVerdict()
// detect mode and no active drop exception
decision->setBlock(false);
if (isIllegalMethodViolation()) {
dbgInfo(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT;
dbgDebug(D_WAAP_ULIMITS) << msg << "INSPECT" << reason << " in detect mode";
verdict = ServiceVerdict::TRAFFIC_VERDICT_INSPECT;
}
else {
dbgInfo(D_WAAP_ULIMITS) << msg << "PASS" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
dbgDebug(D_WAAP_ULIMITS) << msg << "PASS" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
} else {
// detect mode and active drop exception
decision->setBlock(true);
decision->setForceBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Block" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
dbgDebug(D_WAAP_ULIMITS) << msg << "Override Block" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
}
}
else if (mode == AttackMitigationMode::PREVENT) {
@@ -489,14 +491,14 @@ Waf2Transaction::getUserLimitVerdict()
if (!m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION].bForceException) {
// prevent mode and no active accept exception
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
dbgDebug(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_DROP;
} else {
// prevent mode and active accept exception
decision->setBlock(false);
decision->setForceAllow(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
dbgDebug(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ServiceVerdict::TRAFFIC_VERDICT_ACCEPT;
}
}
@@ -662,25 +664,79 @@ Waf2Transaction::getBehaviors(
std::set<ParameterBehavior> all_params;
I_GenericRulebase *i_rulebase = Singleton::Consume<I_GenericRulebase>::by<Waf2Transaction>();
for (const auto &id : exceptions) {
dbgTrace(D_WAAP) << "get parameter exception for: " << id;
auto params = i_rulebase->getParameterException(id).getBehavior(exceptions_dict);
dbgTrace(D_WAAP_OVERRIDE) << "get parameter exception for: " << id;
const auto &paramException = i_rulebase->getParameterException(id);
auto params = paramException.getBehavior(exceptions_dict);
bool hasKVPair = paramException.isContainingKVPair();
// If isContainingKVPair() is false, continue as usual
if (!hasKVPair) {
dbgTrace(D_WAAP_OVERRIDE) << "Using traditional matching for exception " << id << " (no KV pairs)";
if (checkResponse && !getResponseBody().empty()) {
std::unordered_map<std::string, std::set<std::string>> response_dict = {
{"responseBody", {getResponseBody()}}
};
auto params = i_rulebase->getParameterException(id).getBehavior(response_dict);
if (params.size() > 0) {
dbgTrace(D_WAAP) << "got responseBody behavior, setApplyOverride(true)";
auto responseParams = paramException.getBehavior(response_dict);
if (responseParams.size() > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "got responseBody behavior, setApplyOverride(true)";
m_responseInspectReasons.setApplyOverride(true);
all_params.insert(params.begin(), params.end());
all_params.insert(responseParams.begin(), responseParams.end());
// once found, no need to check again
checkResponse = false;
}
}
dbgTrace(D_WAAP) << "got "<< params.size() << " behaviors";
dbgTrace(D_WAAP_OVERRIDE) << "got "<< params.size() << " behaviors (non-KV pair)";
all_params.insert(params.begin(), params.end());
} else {
if (params.size() > 0) {
bool anyKVMatched = false;
std::set<ParameterBehavior> kvParams;
dbgTrace(D_WAAP_OVERRIDE) << "Using KV pair matching for exception " << id;
// Check parameter name/value pairs
for (const DeepParser::KeywordInfo& keywordInfo : getKeywordInfo()) {
std::unordered_map<std::string, std::set<std::string>> kv_dict = {
{"paramName", {keywordInfo.getName()}},
{"paramValue", {keywordInfo.getValue()}}
};
auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true);
if (kvBehaviors.size() > 0) {
anyKVMatched = true;
kvParams.insert(kvBehaviors.begin(), kvBehaviors.end());
}
}
// Check header name/value pairs
for (const auto& hdr_pair : getHdrPairs()) {
std::string name = hdr_pair.first;
std::transform(name.begin(), name.end(), name.begin(), ::tolower);
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
std::unordered_map<std::string, std::set<std::string>> kv_dict = {
{"headerName", {name}},
{"headerValue", {value}}
};
auto kvBehaviors = i_rulebase->getParameterException(id).getBehavior(kv_dict, true);
if (kvBehaviors.size() > 0) {
anyKVMatched = true;
kvParams.insert(kvBehaviors.begin(), kvBehaviors.end());
}
}
// Consider matches only if at least one of the key/value pairs have been successfully matched
if (anyKVMatched) {
dbgTrace(D_WAAP_OVERRIDE) << "got "<< kvParams.size() << " behaviors (KV pair matched)";
all_params.insert(kvParams.begin(), kvParams.end());
} else {
dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no specific pairs matched";
}
} else {
dbgTrace(D_WAAP_OVERRIDE) << "KV pair exception found but no initial match";
}
}
}
return all_params;
}
@@ -696,26 +752,26 @@ bool Waf2Transaction::shouldEnforceByPracticeExceptions(DecisionType practiceTyp
auto exceptions = overridePolicy->getExceptionsByPractice().getExceptionsOfPractice(practiceType);
if (!exceptions.empty()) {
dbgTrace(D_WAAP) << "get behaviors for practice: " << practiceType;
dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for practice: " << practiceType;
auto behaviors = getBehaviors(getExceptionsDict(practiceType), exceptions);
if (behaviors.size() > 0)
{
auto it = m_overrideStateByPractice.find(practiceType);
if (it == m_overrideStateByPractice.end()) {
dbgWarning(D_WAAP) << "no override state for practice: " << practiceType;
dbgDebug(D_WAAP_OVERRIDE) << "no override state for practice: " << practiceType;
return false;
}
setOverrideState(behaviors, it->second);
if (it->second.bForceBlock) {
dbgTrace(D_WAAP)
dbgTrace(D_WAAP_OVERRIDE)
<< "should block by exceptions for practice: " << practiceType;
decision->setBlock(true);
decision->setForceBlock(true);
shouldEnforce = true;
}
if (it->second.bForceException) {
dbgTrace(D_WAAP)
dbgTrace(D_WAAP_OVERRIDE)
<< "should not block by exceptions for practice: " << practiceType;
decision->setBlock(false);
decision->setForceAllow(true);
@@ -748,16 +804,16 @@ void Waf2Transaction::setOverrideState(const std::set<ParameterBehavior>& behavi
m_matchedOverrideIds.insert(behavior.getId());
if (behavior.getKey() == BehaviorKey::ACTION) {
if (behavior.getValue() == BehaviorValue::ACCEPT) {
dbgTrace(D_WAAP) << "setting bForceException due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bForceException due to override behavior: " << behavior.getId();
state.bForceException = true;
state.forceExceptionIds.insert(behavior.getId());
} else if (behavior.getValue() == BehaviorValue::REJECT) {
dbgTrace(D_WAAP) << "setting bForceBlock due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bForceBlock due to override behavior: " << behavior.getId();
state.bForceBlock = true;
state.forceBlockIds.insert(behavior.getId());
}
} else if(behavior.getKey() == BehaviorKey::LOG && behavior.getValue() == BehaviorValue::IGNORE) {
dbgTrace(D_WAAP) << "setting bSupressLog due to override behavior: " << behavior.getId();
dbgTrace(D_WAAP_OVERRIDE) << "setting bSupressLog due to override behavior: " << behavior.getId();
state.bSupressLog = true;
}
}
@@ -771,11 +827,11 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
auto exceptions = overridePolicy->getExceptionsByPractice().
getExceptionsOfPractice(AUTONOMOUS_SECURITY_DECISION);
if (!exceptions.empty()) {
dbgTrace(D_WAAP) << "get behaviors for override state";
dbgTrace(D_WAAP_OVERRIDE) << "get behaviors for override state";
m_responseInspectReasons.setApplyOverride(false);
auto behaviors = getBehaviors(getExceptionsDict(AUTONOMOUS_SECURITY_DECISION), exceptions, true);
if (behaviors.size() > 0) {
dbgTrace(D_WAAP) << "set override state by practice found behaviors";
dbgTrace(D_WAAP_OVERRIDE) << "set override state by practice found behaviors";
setOverrideState(behaviors, m_overrideStateByPractice[AUTONOMOUS_SECURITY_DECISION]);
}
m_isHeaderOverrideScanRequired = false;
@@ -819,10 +875,10 @@ const Maybe<LogTriggerConf, Config::Errors> Waf2Transaction::getTriggerLog(const
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto trigger_config = getConfiguration<LogTriggerConf>("rulebase", "log");
auto trigger_config = getConfigurationWithCache<LogTriggerConf>("rulebase", "log");
if (!trigger_config.ok()) {
dbgError(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr();
dbgDebug(D_WAAP) << "Failed to get log trigger configuration, err: " << trigger_config.getErr();
}
return trigger_config;
}
@@ -851,9 +907,9 @@ bool Waf2Transaction::isTriggerReportExists(const std::shared_ptr<
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto trigger_config = getConfiguration<ReportTriggerConf>("rulebase", "report");
auto trigger_config = getConfigurationWithCache<ReportTriggerConf>("rulebase", "report");
if (!trigger_config.ok()) {
dbgWarning(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr();
dbgDebug(D_WAAP) << "Failed to get report trigger configuration, err: " << trigger_config.getErr();
m_triggerReport = false;
return false;
}

31
components/security_apps/waap/waap_clib/Waf2Regex.cc
View File

@@ -117,11 +117,14 @@ SingleRegex::~SingleRegex() {
}
}
bool SingleRegex::hasMatch(const std::string& s) const {
bool SingleRegex::hasMatch(const std::string& s, size_t start, size_t end) const {
PCRE2_SIZE startOffset = static_cast<PCRE2_SIZE>(start);
size_t data_size = std::min(s.size(), end);
int rc = pcre2_match(
m_re, // code
reinterpret_cast<PCRE2_SPTR>(s.data()), s.size(), // subject/subject length
0, // start offset
reinterpret_cast<PCRE2_SPTR>(s.data()), data_size, // subject/subject length
startOffset, // start offset
0, // options
m_matchData,
NULL // match_context
@@ -244,13 +247,23 @@ const std::string &SingleRegex::getName() const
return m_regexName;
}
size_t SingleRegex::findMatchRanges(const std::string& s, std::vector<RegexMatchRange>& matchRanges) const {
PCRE2_SIZE startOffset = 0;
size_t
SingleRegex::findMatchRanges(
const std::string &s,
std::vector<RegexMatchRange> &matchRanges,
size_t maxMatches,
size_t start,
size_t end
) const
{
PCRE2_SIZE startOffset = static_cast<PCRE2_SIZE>(start);
size_t matchCount = 0;
size_t data_size = std::min(s.size(), end);
do {
int rc = pcre2_match(
m_re, // code
reinterpret_cast<PCRE2_SPTR>(s.data()), s.size(), // subject/subject length
reinterpret_cast<PCRE2_SPTR>(s.data()), data_size, // subject/subject length
startOffset, // start offset
0, // options
m_matchData,
@@ -277,6 +290,12 @@ size_t SingleRegex::findMatchRanges(const std::string& s, std::vector<RegexMatch
startOffset = ov[1];
matchRanges.push_back(RegexMatchRange(ov[0], ov[1]));
matchCount++;
// Check if we've reached the maximum number of matches (if maxMatches is not 0)
if (maxMatches > 0 && matchCount >= maxMatches) {
break;
}
} while (true);
return matchRanges.size();

10
components/security_apps/waap/waap_clib/Waf2Regex.h
View File

@@ -54,10 +54,16 @@ public:
SingleRegex(const std::string &pattern, bool &error, const std::string &regexName, bool bNoRegex=false,
const std::string &regexMatchName="", const std::string &regexMatchValue="");
~SingleRegex();
bool hasMatch(const std::string &s) const;
bool hasMatch(const std::string &s, size_t start = 0, size_t end = SIZE_MAX) const;
size_t findAllMatches(const std::string &s, std::vector<RegexMatch> &matches,
size_t max_matches = std::string::npos) const;
size_t findMatchRanges(const std::string &s, std::vector<RegexMatchRange> &matchRanges) const;
size_t findMatchRanges(
const std::string &s,
std::vector<RegexMatchRange> &matchRanges,
size_t maxMatches = 0,
size_t start = 0,
size_t end = SIZE_MAX
) const;
const std::string &getName() const;
private:
pcre2_code *m_re;

119
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -48,6 +48,49 @@ USE_DEBUG_FLAG(D_WAAP_JSON);
#define MIN_HEX_LENGTH 6
#define charToDigit(c) (c - '0')
// Base64 decoding lookup table constants
#define BASE64_INVALID -1
#define BASE64_PADDING -2
// Base64 lookup table for optimized decoding
static int base64_table[256];
static bool base64_table_initialized = false;
// Initialize the base64 lookup table for optimized decoding
static void initialize_base64_table()
{
if (base64_table_initialized) {
return;
}
// Initialize all entries to invalid
for (int i = 0; i < 256; i++) {
base64_table[i] = BASE64_INVALID;
}
// Set valid base64 characters (A-Z = 0-25)
for (int i = 0; i < 26; i++) {
base64_table['A' + i] = i;
}
// Set valid base64 characters (a-z = 26-51)
for (int i = 0; i < 26; i++) {
base64_table['a' + i] = 26 + i;
}
// Set valid base64 characters (0-9 = 52-61)
for (int i = 0; i < 10; i++) {
base64_table['0' + i] = 52 + i;
}
// Set special base64 characters
base64_table['+'] = 62;
base64_table['/'] = 63;
base64_table['='] = BASE64_PADDING;
base64_table_initialized = true;
}
// See https://dev.w3.org/html5/html-author/charref
const struct HtmlEntity g_htmlEntities[] =
{
@@ -1086,6 +1129,19 @@ base64_decode_status decideStatusBase64Decoded(
return B64_DECODE_INVALID;
}
//
// Base64 decoding with direct array lookup
// Performance optimization notes:
// - Replaced unordered_map lookups with direct array indexing
// - Improved cache locality with a simple 256-element array
// - Eliminated hash computation and collision handling overhead
// - One-time initialization of the lookup table
//
// The table maps each ASCII character to its base64 value:
// - For valid base64 characters (A-Z, a-z, 0-9, +, /): returns 0-63 value
// - For padding character ('='): returns BASE64_PADDING (-2)
// - For invalid characters: returns BASE64_INVALID (-1)
//
// Attempts to validate and decode base64-encoded chunk.
// Value is the full value inside which potential base64-encoded chunk was found,
@@ -1107,6 +1163,9 @@ base64_decode_status decodeBase64Chunk(
bool clear_on_error,
bool called_with_prefix)
{
// Initialize the base64 lookup table on first call
initialize_base64_table();
decoded.clear();
uint32_t acc = 0;
int acc_bits = 0; // how many bits are filled in acc
@@ -1135,8 +1194,9 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID;
}
std::unordered_map<char, double> original_occurences_counter;
std::unordered_map<char, double> decoded_occurences_counter;
// Fixed arrays for character counting - Proposal 1 optimization
uint32_t original_counts[256] = {0}; // Count for each ASCII character in original
uint32_t decoded_counts[256] = {0}; // Count for each byte value in decoded
while (it != end) {
unsigned char c = *it;
@@ -1159,35 +1219,21 @@ base64_decode_status decodeBase64Chunk(
// allow for more terminator characters
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
continue;
}
unsigned char val = 0;
// Use lookup table for faster decoding
int lookup_value = base64_table[static_cast<unsigned char>(c)];
if (c >= 'A' && c <= 'Z') {
val = c - 'A';
}
else if (c >= 'a' && c <= 'z') {
val = c - 'a' + 26;
}
else if (isdigit(c)) {
val = c - '0' + 52;
}
else if (c == '+') {
val = 62;
}
else if (c == '/') {
val = 63;
}
else if (c == '=') {
// Start tracking terminator characters
if (lookup_value == BASE64_PADDING) {
// Start tracking terminator characters ('=')
terminatorCharsSeen++;
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
continue;
}
else {
else if (lookup_value == BASE64_INVALID) {
dbgTrace(D_WAAP_BASE64)
<< "(leave as-is) because of non-base64 character ('"
<< c
@@ -1199,6 +1245,8 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID; // non-base64 character
}
unsigned char val = static_cast<unsigned char>(lookup_value);
acc = (acc << 6) | val;
acc_bits += 6;
@@ -1218,11 +1266,11 @@ base64_decode_status decodeBase64Chunk(
}
decoded += (char)code;
decoded_occurences_counter[(char)code]++;
decoded_counts[static_cast<unsigned char>(code)]++;
}
it++;
original_occurences_counter[c]++;
original_counts[static_cast<unsigned char>(c)]++;
}
// end of encoded sequence decoded.
@@ -1242,14 +1290,22 @@ base64_decode_status decodeBase64Chunk(
double entropy = 0;
double p = 0;
double decoded_entropy = 0;
for (const auto& pair : original_occurences_counter) {
p = pair.second / length;
// Calculate entropy from original character counts
for (int i = 0; i < 256; i++) {
if (original_counts[i] > 0) {
p = static_cast<double>(original_counts[i]) / length;
entropy -= p * std::log2(p);
}
for (const auto &pair : decoded_occurences_counter) {
p = pair.second / decoded.size();
}
// Calculate entropy from decoded character counts
for (int i = 0; i < 256; i++) {
if (decoded_counts[i] > 0) {
p = static_cast<double>(decoded_counts[i]) / decoded.size();
decoded_entropy -= p * std::log2(p);
}
}
dbgTrace(D_WAAP_BASE64)
<< "Base entropy = "
<< entropy
@@ -1917,9 +1973,6 @@ isGzipped(const string &stream)
{
if (stream.size() < 2) return false;
auto unsinged_stream = reinterpret_cast<const u_char *>(stream.data());
dbgTrace(D_WAAP) << "isGzipped: first two bytes: "
<< std::hex << static_cast<int>(unsinged_stream[0]) << " "
<< std::hex << static_cast<int>(unsinged_stream[1]);
return unsinged_stream[0] == 0x1f && unsinged_stream[1] == 0x8b;
}
@@ -2310,7 +2363,7 @@ string extractForwardedIp(const string &x_forwarded_hdr_val)
vector<string> trusted_ips;
string forward_ip;
auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
auto identify_config = getConfigurationWithCache<UsersAllIdentifiersConfig>(
"rulebase",
"usersIdentifiers"
);

Some files were not shown because too many files have changed in this diff Show More