github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-11-28 10:41:59 +00:00
parent 6255e1f30d
commit 1c1f0b7e29
59 changed files with 842 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
components/attachment-intakers/nginx_attachment/nginx_attachment.cc
View File

@@ -1135,7 +1135,11 @@ private:
"webUserResponse"
);
bool remove_event_id_param =
getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
string uuid;
string redirectUrl;
if (i_transaction_table->hasState<NginxAttachmentOpaque>()) {
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
uuid = opaque.getSessionUUID();
@@ -1145,7 +1149,12 @@ private:
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
web_response_data.response_data.redirect_data.redirect_location_size =
web_trigger_conf.getRedirectURL().size();
web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0;
bool add_event = web_trigger_conf.getAddEventId();
if (add_event && !remove_event_id_param) {
web_response_data.response_data.redirect_data.redirect_location_size +=
strlen("?event_id=") + uuid.size();
}
web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
} else {
web_response_data.response_data.custom_response_data.title_size =
@@ -1159,8 +1168,13 @@ private:
verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));
if (web_trigger_conf.getDetailsLevel() == "Redirect") {
verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getRedirectURL().data()));
verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size());
redirectUrl = web_trigger_conf.getRedirectURL();
if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
redirectUrl += "?event-id=" + uuid;
}
verdict_data.push_back(reinterpret_cast<const char *>(redirectUrl.data()));
verdict_data_sizes.push_back(redirectUrl.size());
} else {
verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getResponseTitle().data()));
verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size());

30
components/attachment-intakers/nginx_attachment/user_identifiers_config.cc
View File

@@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector<CIDRSData> &cidr_values)
}
Maybe<string>
UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const
{
vector<string> header_values = split(str);
@@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
vector<string> xff_values = getHeaderValuesFromConfig("x-forwarded-for");
vector<CIDRSData> cidr_values(xff_values.begin(), xff_values.end());
for (const string &value : header_values) {
if (!IPAddr::createIPAddr(value).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value;
for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) {
if (!IPAddr::createIPAddr(*it).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it;
return genError("Invalid IP address");
}
if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found");
if (type == ExtractType::PROXYIP) continue;
if (!isIpTrusted(*it, cidr_values)) {
dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it;
return *it;
}
}
if (!IPAddr::createIPAddr(header_values[0]).ok()) {
dbgWarning(D_NGINX_ATTACHMENT_PARSER)
<< "Invalid IP address found in the xff header IPs list: "
<< header_values[0];
return genError("Invalid IP address");
}
return header_values[0];
@@ -312,7 +323,7 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
return;
}
NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
auto value = parseXForwardedFor(header.getValue());
auto value = parseXForwardedFor(header.getValue(), type);
if (!value.ok()) {
dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
return;
@@ -321,12 +332,13 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext
if (type == ExtractType::SOURCEIDENTIFIER) {
opaque.setSourceIdentifier(header.getKey(), value.unpack());
dbgDebug(D_NGINX_ATTACHMENT_PARSER)
<< "Added source identifir to XFF "
<< "Added source identifier from XFF header"
<< value.unpack();
opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack());
dbgTrace(D_NGINX_ATTACHMENT_PARSER)
<< "XFF found, set ctx with value from header: "
<< static_cast<string>(header.getValue());
<< "XFF found, set ctx with value from header: "
<< static_cast<string>(header.getValue());
} else {
opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack());
}

2
components/include/reverse_proxy_defaults.h
View File

@@ -28,7 +28,7 @@ static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/ngi
static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf";
static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
static const std::string default_nginx_config_include_file =
"/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf";
"/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf";
static const std::string default_global_conf_include_template =
"/etc/cp/conf/rpmanager/nginx-conf-include-template";
static const std::string default_global_conf_include_template_no_responses =

2
components/include/user_identifiers_config.h
View File

@@ -58,7 +58,7 @@ private:
const std::string::const_iterator &end,
const std::string &key) const;
Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const;
Maybe<std::string> parseXForwardedFor(const std::string &str) const;
Maybe<std::string> parseXForwardedFor(const std::string &str, ExtractType type) const;
std::vector<UsersIdentifiersConfig> user_identifiers;
};

4
components/include/waap.h
View File

@@ -33,7 +33,6 @@ class I_WaapAssetStatesManager;
class I_Messaging;
class I_AgentDetails;
class I_Encryptor;
class I_WaapModelResultLogger;
const std::string WAAP_APPLICATION_NAME = "waap application";
@@ -51,8 +50,7 @@ class WaapComponent
Singleton::Consume<I_AgentDetails>,
Singleton::Consume<I_Messaging>,
Singleton::Consume<I_Encryptor>,
Singleton::Consume<I_Environment>,
Singleton::Consume<I_WaapModelResultLogger>
Singleton::Consume<I_Environment>
{
public:
WaapComponent();

2
components/security_apps/orchestration/details_resolver/details_resolver.cc
View File

@@ -350,7 +350,7 @@ DetailsResolver::Impl::readCloudMetadata()
}
if (!cloud_metadata.ok()) {
dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr();
dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
return genError("Failed to fetch cloud metadata");
}

47
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@@ -18,6 +18,8 @@
#include <regex>
#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include <cereal/external/rapidjson/document.h>
#include <cereal/external/rapidjson/filereadstream.h>
#if defined(gaia)
@@ -100,6 +102,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
return command_output;
}
Maybe<string>
getOtlpAgentGaiaOsRole(const string &command_output)
{
if (command_output == "" ) return string("-1");
return command_output;
}
Maybe<string>
getQUID(const string &command_output)
{
@@ -111,6 +121,13 @@ getQUID(const string &command_output)
return command_output;
}
Maybe<string>
getIsAiopsRunning(const string &command_output)
{
if (command_output == "" ) return string("false");
return command_output;
}
Maybe<string>
checkHasSDWan(const string &command_output)
@@ -186,6 +203,24 @@ getMgmtObjAttr(shared_ptr<istream> file_stream, const string &attr)
return genError("Object attribute was not found. Attr: " + attr);
}
Maybe<string>
getAttrFromCpsdwanGetDataJson(const string &attr)
{
static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json";
std::ifstream ifs(get_data_json_path);
if (ifs.is_open()) {
rapidjson::IStreamWrapper isw(ifs);
rapidjson::Document document;
document.ParseStream(isw);
if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) {
return string(document[attr.c_str()].GetString());
}
}
return genError("Attribute " + attr + " was not found in " + get_data_json_path);
}
Maybe<string>
getMgmtObjUid(const string &command_output)
{
@@ -193,6 +228,11 @@ getMgmtObjUid(const string &command_output)
return command_output;
}
Maybe<string> obj_uuid = getAttrFromCpsdwanGetDataJson("uuid");
if (obj_uuid.ok()) {
return obj_uuid.unpack();
}
static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
auto file_stream = std::make_shared<std::ifstream>(obj_path);
if (!file_stream->is_open()) {
@@ -310,7 +350,12 @@ getSmbObjectName(const string &command_output)
if (command_output.empty() || command_output[0] != centrally_managed_comd_output) {
return genError("Object name was not found");
}
Maybe<string> obj_name = getAttrFromCpsdwanGetDataJson("name");
if (obj_name.ok()) {
return obj_name.unpack();
}
static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
auto ifs = std::make_shared<std::ifstream>(obj_path);
if (!ifs->is_open()) {

15
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@@ -73,6 +73,15 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
"&& python3 /opt/CPquid/Quid_Api.py -i "
"/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
getQUID)
SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
"&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
getOtlpAgentGaiaOsRole)
SHELL_CMD_HANDLER(
"IS_AIOPS_RUNNING",
"FS_PATH=<FILESYSTEM-PREFIX>; "
"PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
"[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
getIsAiopsRunning)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",
@@ -180,8 +189,7 @@ SHELL_CMD_HANDLER(
)
SHELL_CMD_HANDLER(
"managements",
"sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |"
" sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
"echo 1",
extractManagements
)
#endif //gaia
@@ -237,8 +245,7 @@ SHELL_CMD_HANDLER(
SHELL_CMD_HANDLER(
"managements",
"sed -n '/:masters (/,$p' /tmp/local.cfg |"
" sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
"echo 1",
extractManagements
)
#endif//smb

4
components/security_apps/orchestration/hybrid_mode_telemetry.cc
View File

@@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &)
{
auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
auto maybe_cmd_output = shell_cmd->getExecOutput(
getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count"
getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count",
1000,
false
);
// get wd process restart count

6
components/security_apps/orchestration/include/declarative_policy_utils.h
View File

@@ -79,8 +79,8 @@ public:
) override;
std::string getUpdate(CheckUpdateRequest &request) override;
bool shouldApplyPolicy() override;
void turnOffApplyPolicyFlag() override;
void turnOnApplyPolicyFlag() override;
void turnOffApplyLocalPolicyFlag() override;
void turnOnApplyLocalPolicyFlag() override;
std::string getCurrPolicy() override { return curr_policy; }
@@ -94,7 +94,7 @@ private:
std::string curr_version;
std::string curr_policy;
std::string curr_checksum;
bool should_apply_policy;
bool should_apply_local_policy;
};
#endif // __DECLARATIVE_POLICY_UTILS_H__

4
components/security_apps/orchestration/include/i_declarative_policy.h
View File

@@ -22,8 +22,8 @@ public:
virtual std::string getCurrPolicy() = 0;
virtual void turnOffApplyPolicyFlag() = 0;
virtual void turnOnApplyPolicyFlag() = 0;
virtual void turnOffApplyLocalPolicyFlag() = 0;
virtual void turnOnApplyLocalPolicyFlag() = 0;
protected:
virtual ~I_DeclarativePolicy() {}

2
components/security_apps/orchestration/orchestration_comp.cc
View File

@@ -2033,7 +2033,7 @@ private:
}
auto policy_mgmt_mode = getSettingWithDefault<string>("management", "profileManagedMode");
if (getOrchestrationMode() == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative") {
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOnApplyLocalPolicyFlag();
}
auto policy_version = i_service_controller->getPolicyVersion();

6
components/security_apps/orchestration/service_controller/service_controller.cc
View File

@@ -793,7 +793,7 @@ ServiceController::Impl::updateServiceConfiguration(
<< "Policy file was not updated. Sending reload command regarding settings and data";
auto signal_services = sendSignalForServices(nano_services_to_update, "");
if (!signal_services.ok()) return signal_services.passErr();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}
@@ -940,7 +940,7 @@ ServiceController::Impl::updateServiceConfiguration(
if (new_policy_path.compare(config_file_path) == 0) {
dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file";
policy_version = version_value;
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}
@@ -959,7 +959,7 @@ ServiceController::Impl::updateServiceConfiguration(
}
if (!was_policy_updated && !send_signal_for_services_err.empty()) return genError(send_signal_for_services_err);
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyPolicyFlag();
Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>()->turnOffApplyLocalPolicyFlag();
return Maybe<void>();
}

21
components/security_apps/orchestration/update_communication/declarative_policy_utils.cc
View File

@@ -17,7 +17,7 @@ void
DeclarativePolicyUtils::init()
{
local_policy_path = getFilesystemPathConfig() + "/conf/local_policy.yaml";
should_apply_policy = true;
should_apply_local_policy = true;
Singleton::Consume<I_RestApi>::by<DeclarativePolicyUtils>()->addRestCall<ApplyPolicyRest>(
RestAction::SET, "apply-policy"
);
@@ -40,7 +40,7 @@ DeclarativePolicyUtils::upon(const ApplyPolicyEvent &event)
{
dbgTrace(D_ORCHESTRATOR) << "Apply policy event";
local_policy_path = event.getPolicyPath();
should_apply_policy = true;
should_apply_local_policy = true;
}
// LCOV_EXCL_STOP
@@ -48,19 +48,24 @@ bool
DeclarativePolicyUtils::shouldApplyPolicy()
{
auto env_type = Singleton::Consume<I_EnvDetails>::by<DeclarativePolicyUtils>()->getEnvType();
return env_type == EnvType::K8S ? true : should_apply_policy;
if (env_type == EnvType::K8S) {
I_OrchestrationTools *orch_tools = Singleton::Consume<I_OrchestrationTools>::by<DeclarativePolicyUtils>();
auto maybe_new_version = orch_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger");
return maybe_new_version != curr_version;
}
return should_apply_local_policy;
}
void
DeclarativePolicyUtils::turnOffApplyPolicyFlag()
DeclarativePolicyUtils::turnOffApplyLocalPolicyFlag()
{
should_apply_policy = false;
should_apply_local_policy = false;
}
void
DeclarativePolicyUtils::turnOnApplyPolicyFlag()
DeclarativePolicyUtils::turnOnApplyLocalPolicyFlag()
{
should_apply_policy = true;
should_apply_local_policy = true;
}
Maybe<string>
@@ -211,6 +216,6 @@ DeclarativePolicyUtils::periodicPolicyLoad()
if (*new_checksum == curr_checksum) return;
should_apply_policy = true;
should_apply_local_policy = true;
curr_checksum = *new_checksum;
}

2
components/security_apps/orchestration/update_communication/fog_communication.cc
View File

@@ -74,7 +74,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request)
<< " to: "
<< policy_mgmt_mode;
profile_mode = policy_mgmt_mode;
i_declarative_policy->turnOnApplyPolicyFlag();
i_declarative_policy->turnOnApplyLocalPolicyFlag();
}
if (i_declarative_policy->shouldApplyPolicy()) {

9
components/security_apps/rate_limit/rate_limit.cc
View File

@@ -293,8 +293,13 @@ public:
return ACCEPT;
}
burst = rule.getRateLimit();
limit = calcRuleLimit(rule);
auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1;
if (replicas == 0) {
dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1";
replicas = 1;
}
burst = static_cast<float>(rule.getRateLimit()) / replicas;
limit = static_cast<float>(calcRuleLimit(rule)) / replicas;
dbgTrace(D_RATE_LIMIT)
<< "found rate limit rule with: "

37
components/security_apps/waap/include/i_waap_model_result_logger.h
View File

@@ -1,37 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
class IWaf2Transaction;
struct Waf2ScanResult;
namespace Waap {
namespace Scores {
struct ModelLoggingSettings;
}
}
class I_WaapModelResultLogger {
public:
virtual ~I_WaapModelResultLogger() {}
virtual void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double newScore,
double baseScore) = 0;
};

3
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -87,9 +87,10 @@ add_library(waap_clib
ParserPairs.cc
Waf2Util2.cc
ParserPDF.cc
ParserKnownBenignSkipper.cc
ParserScreenedJson.cc
ParserBinaryFile.cc
RegexComparator.cc
WaapModelResultLogger.cc
)
add_definitions("-Wno-unused-function")

71
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -28,6 +28,8 @@
#include "ParserDelimiter.h"
#include "ParserPDF.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
#include "ParserScreenedJson.h"
#include "WaapAssetState.h"
#include "Waf2Regex.h"
#include "Waf2Util.h"
@@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64ParamFound,
@@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,
@@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
b64FileType
@@ -918,6 +925,7 @@ bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
) {
@@ -959,6 +967,7 @@ DeepParser::createInternalParser(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@@ -978,7 +987,19 @@ DeepParser::createInternalParser(
<< "\n\tflags: "
<< flags
<< "\n\tparser_depth: "
<< parser_depth;
<< parser_depth
<< "\n\tisBodyPayload: "
<< isBodyPayload
<< "\n\tisRefererPayload: "
<< isRefererPayload
<< "\n\tisRefererParamPayload: "
<< isRefererParamPayload
<< "\n\tisUrlPayload: "
<< isUrlPayload
<< "\n\tisUrlParamPayload: "
<< isUrlParamPayload
<< "\n\tisCookiePayload: "
<< isCookiePayload;
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false,
isAmperType = false;
bool isKeyValDelimited = false;
@@ -1045,6 +1066,53 @@ DeepParser::createInternalParser(
}
}
if (Waap::Util::isScreenedJson(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserScreenedJson>>(*this, parser_depth + 1));
offset = 0;
return offset;
}
dbgTrace(D_WAAP_DEEP_PARSER)
<< "Offset = "
<< offset
<< " depth = "
<< m_depth
<< " isBodyPayload = "
<< isBodyPayload;
//Detect sensor_data format in body and just use dedicated filter for it
if (m_depth == 1
&& isBodyPayload
&& Waap::Util::detectKnownSource(cur_val) == Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserKnownBenignSkipper>>(
*this,
parser_depth + 1,
Waap::Util::SOURCE_TYPE_SENSOR_DATA
)
);
offset = 0;
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it";
return offset;
}
// Detect cookie parameter sensorsdata2015jssdkcross
// and causes false positives due to malformed JSON. Make preprocessing to parse it correctly
if (m_depth == 2
&& isCookiePayload) {
offset = Waap::Util::definePrefixedJson(cur_val);
if (offset >= 0) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserJson>>(
*this,
parser_depth + 1,
m_pTransaction
)
);
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data";
return offset;
}
}
// Detect wbxml (binary XML) data type
if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) {
m_is_wbxml = true;
@@ -1374,6 +1442,7 @@ DeepParser::createInternalParser(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth
);

3
components/security_apps/waap/waap_clib/DeepParser.h
View File

@@ -129,6 +129,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@@ -144,6 +145,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
);
@@ -160,6 +162,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,

96
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
#endif
}
// std::string nicePrint() - is a function used to create std::string that will represent all data that is
// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
// into the object easier.
std::string
WaapAssetState::nicePrint(Waf2ScanResult &res) const
{
std::string result = "Waf2ScanResult:\n";
result += "keyword_matches:\n";
for (const auto &keyword : res.keyword_matches) {
result += keyword + "\n";
}
result += "regex_matches:\n";
for (const auto &regex : res.regex_matches) {
result += regex + "\n";
}
result += "filtered_keywords:\n";
for (const auto &filtered : res.filtered_keywords) {
result += filtered + "\n";
}
result += "found_patterns:\n";
for (const auto &pattern : res.found_patterns) {
result += pattern.first + ":\n";
for (const auto &value : pattern.second) {
result += value + "\n";
}
}
result += "unescaped_line: " + res.unescaped_line + "\n";
result += "param_name: " + res.param_name + "\n";
result += "location: " + res.location + "\n";
result += "score: " + std::to_string(res.score) + "\n";
result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n";
result += "scoreArray:\n";
for (const auto &score : res.scoreArray) {
result += std::to_string(score) + "\n";
}
result += "keywordCombinations:\n";
for (const auto &combination : res.keywordCombinations) {
result += combination + "\n";
}
result += "attack_types:\n";
for (const auto &attack : res.attack_types) {
result += attack + "\n";
}
result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n";
return result;
}
bool
checkBinaryData(const std::string &line, bool binaryDataFound)
{
@@ -1033,7 +1082,7 @@ WaapAssetState::apply(
// Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
// This is done to improve performance of regex matching.
SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
checkRegex(
unescapedLineSample,
m_Signatures->specific_acuracy_keywords_regex,
@@ -1111,7 +1160,7 @@ WaapAssetState::apply(
}
bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res);
if (os_cmd_ev) {
dbgTrace(D_WAAP_EVASIONS) << "os command evasion found";
@@ -1295,6 +1344,47 @@ WaapAssetState::apply(
}
}
bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns);
dbgTrace(D_WAAP_EVASIONS)
<< "path_traversal_ev = " << path_traversal_ev
<< " sample = " << res.unescaped_line
<< " res.unescaped_line.find(2f) = " << res.unescaped_line.find("2f");
if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) {
// Possible path traversal evasion .2f. detected: - clean up and scan with regexes again.
dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line
<< "Status beroe evasion checking " << nicePrint(res);
std::string unescaped = line;
replaceAll(unescaped, "2f", "/");
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("path_traversal");
path_traversal_ev = false;
}
else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
}
bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
"quotes_space_ev_fast_reg",
res.found_patterns
@@ -1726,7 +1816,7 @@ WaapAssetState::apply(
wordsCount = std::min(wordsCount, newWordsCount);
}
}
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
// Remove evasion keywords that should not be reported because there's no real evasion found
if (!keywordsToRemove.empty()) {
dbgTrace(D_WAAP_SAMPLE_SCAN)

1
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -49,6 +49,7 @@ private: //ugly but needed for build
Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
std::string nicePrint(Waf2ScanResult &res) const;
public:
// Load and compile signatures from file

250
components/security_apps/waap/waap_clib/WaapModelResultLogger.cc
View File

@@ -1,250 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "WaapModelResultLogger.h"
#include "Waf2Engine.h"
#include "i_time_get.h"
#include "i_messaging.h"
#include "i_instance_awareness.h"
#include "http_manager.h"
#include "LogGenWrapper.h"
#include "rest.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
using namespace std;
static const unsigned int MAX_FILES_PER_WINDOW = 5;
static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
class WaapModelReport : public RestGetFile
{
public:
WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
private:
C2S_PARAM(vector<WaapModelResult>, data);
};
class WaapModelResultLogger::Impl
:
Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
{
public:
Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
last_sent_s3(std::chrono::minutes::zero()),
last_kusto_log_window(std::chrono::minutes::zero()) {}
virtual ~Impl();
void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore) override;
private:
void logToStream(WaapModelResult &result, chrono::minutes now);
void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
bool shouldSendLogsToS3(chrono::minutes now);
void sendLogsToS3();
size_t max_logs;
unsigned int sent_files_count;
unsigned int sent_logs_count;
std::chrono::minutes last_sent_s3;
std::chrono::minutes last_kusto_log_window;
std::map<std::string, vector<WaapModelResult>> logs;
};
WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
{
}
WaapModelResultLogger::~WaapModelResultLogger()
{
}
void
WaapModelResultLogger::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
)
{
pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
}
void
WaapModelResultLogger::Impl::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore)
{
if (transaction == NULL) return;
if (!Singleton::exists<I_Messaging>()) {
dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
return;
}
double score_diff = score - otherScore;
if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
(score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
return;
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
WaapModelResult result = WaapModelResult(
*transaction,
res,
modelName,
otherModelName,
score,
otherScore,
now.count()
);
if (settings.logToStream) logToStream(result, now);
if (settings.logToS3) logToS3(result, transaction, now);
}
void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
{
auto asset_state = transaction->getAssetState();
string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
auto asset_logs = logs.find(asset_id);
if (asset_logs == logs.end()) {
logs.emplace(asset_id, vector<WaapModelResult>());
}
logs.at(asset_id).push_back(result);
if (shouldSendLogsToS3(now)) {
sendLogsToS3();
}
}
void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
{
if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
last_kusto_log_window = now;
sent_logs_count = 0;
}
else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
return;
}
sent_logs_count++;
dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"WAAP Model Telemetry",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::HIGH,
false);
LogGen& waap_log = logGenWrapper.getLogGen();
waap_log.addMarkerSuffix(result.location);
waap_log << LogField("httpuripath", result.uri);
waap_log << LogField("matchedlocation", result.location);
waap_log << LogField("matchedparameter", result.param);
waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
waap_log << LogField("waapfinalscore", (int)(result.score * 100));
waap_log << LogField("indicatorssource", result.modelName);
waap_log << LogField("indicatorsversion", result.otherModelName);
}
bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
{
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
for (const auto &asset_logs : logs) {
if (asset_logs.second.size() >= max_logs) return true;
}
return false;
}
void WaapModelResultLogger::Impl::sendLogsToS3()
{
dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
for (auto &asset_logs : logs) {
if (asset_logs.second.empty()) {
continue;
}
if (sent_files_count >= MAX_FILES_PER_WINDOW) {
dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
asset_logs.second.clear();
continue;
}
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
string tenant_id = agentDetails->getTenantId();
string agent_id = agentDetails->getAgentId();
string asset_id = asset_logs.first;
if (Singleton::exists<I_InstanceAwareness>()) {
I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
Maybe<string> uniqueId = instance->getUniqueID();
if (uniqueId.ok())
{
agent_id += "/" + uniqueId.unpack();
}
}
string uri = "/storage/waap/" +
tenant_id + "/" + asset_id + "/waap_model_results/window_" +
to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
"/" + agent_id + "/data.data";
WaapModelReport report = WaapModelReport(asset_logs.second);
dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
", length " << asset_logs.second.size() <<
", uri " << uri;
msg->sendAsyncMessage(
HTTPMethod::PUT,
uri,
report,
MessageCategory::LOG
);
asset_logs.second.clear();
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
last_sent_s3 = now;
sent_files_count = 0;
} else {
sent_files_count++;
}
}
WaapModelResultLogger::Impl::~Impl()
{}

109
components/security_apps/waap/waap_clib/WaapModelResultLogger.h
View File

@@ -1,109 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <map>
#include <chrono>
#include <vector>
#include <ostream>
#include "cereal/archives/json.hpp"
#include "i_waap_model_result_logger.h"
#include "DeepAnalyzer.h"
#include "i_transaction.h"
#include "ScanResult.h"
#include "WaapAssetState.h"
#include "WaapScores.h"
class WaapModelResultLogger
:
Singleton::Provide<I_WaapModelResultLogger>
{
public:
WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
virtual ~WaapModelResultLogger();
virtual void logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
);
class Impl;
protected:
std::unique_ptr<Impl> pimpl;
static const size_t MAX_WAAP_MODEL_LOGS = 20000;
};
class WaapModelResult
{
public:
WaapModelResult(
IWaf2Transaction &transaction,
Waf2ScanResult &res,
const std::string &modelName,
const std::string &otherModelName,
double score,
double otherScore,
uint64_t time
) : uri(transaction.getUri()), location(res.location), param(res.param_name),
modelName(modelName), otherModelName(otherModelName),
score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
{
}
template<class Archive>
void serialize(Archive &ar) const
{
ar(cereal::make_nvp("uri", uri));
ar(cereal::make_nvp("location", location));
ar(cereal::make_nvp("param", param));
ar(cereal::make_nvp("modelName", modelName));
ar(cereal::make_nvp("otherModelName", otherModelName));
ar(cereal::make_nvp("score", score));
ar(cereal::make_nvp("otherScore", otherScore));
ar(cereal::make_nvp("keywords", keywords));
ar(cereal::make_nvp("sample", sample));
ar(cereal::make_nvp("id", id));
ar(cereal::make_nvp("time", time));
}
std::string toString() const
{
std::stringstream message_stream;
{
cereal::JSONOutputArchive ar(message_stream);
serialize(ar);
}
return message_stream.str();
}
std::string uri;
std::string location;
std::string param;
std::string modelName;
std::string otherModelName;
double score;
double otherScore;
std::vector<std::string> keywords;
std::string sample;
uint64_t id;
uint64_t time;
};

5
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@@ -48,8 +48,9 @@ public:
m_tag = to_lower_copy(m_tag);
if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") {
m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" &&
m_tag != "paramlocation" && m_tag != "responsebody" && m_tag != "headername" &&
m_tag != "headervalue" && m_tag != "method") {
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -105,7 +105,7 @@ bool WaapOverrideFunctor::operator()(
}
return false;
}
else if (tagLower == "keyword") {
else if (tagLower == "keyword" || tagLower == "indicator") {
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {

12
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -15,7 +15,6 @@
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@@ -111,10 +110,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
@@ -123,14 +118,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
@@ -142,6 +133,9 @@ double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {

11
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -1358,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
return false;
}
std::string body(data);
std::string body(data, data_len);
if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
{
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
@@ -1661,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("sourcePort", m_remote_port);
waapLog << LogField("httpHostName", m_hostStr);
waapLog << LogField("httpMethod", m_methodStr);
if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
bool send_extended_log = shouldSendExtendedLog(triggerLog);
@@ -2343,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
exceptions_dict["url"].insert(getUriStr());
exceptions_dict["hostName"].insert(m_hostStr);
exceptions_dict["method"].insert(m_methodStr);
for (auto &keyword : res.keyword_matches) {
exceptions_dict["indicator"].insert(keyword);
@@ -2355,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (const auto &behavior : behaviors) {
dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
if (!res.filtered_keywords.empty() || res.score > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
std::string overrideId = behavior.getId();
if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
@@ -2375,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
}
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
std::string overrideId = behavior.getId();
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);

1
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -41,7 +41,6 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"

23
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -35,6 +35,7 @@
#include "user_identifiers_config.h"
#include "Waf2Regex.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
using boost::algorithm::to_lower_copy;
using namespace std;
@@ -1218,21 +1219,21 @@ static const SingleRegex csp_report_policy_re(
"csp_report_policy"
);
static const SingleRegex base64_key_value_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
static const SingleRegex json_key_value_detector_re(
"\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
err,
"json_key_value");
err,
"json_key_value");
static const SingleRegex base64_key_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
static const SingleRegex base64_prefix_detector_re(
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
// looks for combination <param>={<some text>*:<some text>*}
//used to allow parsing param=JSON to reduce false positives

10
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -894,6 +894,16 @@ namespace Util {
bool isValidJson(const std::string &input);
enum KnownSourceType {
SOURCE_TYPE_UNKNOWN = 0,
SOURCE_TYPE_SENSOR_DATA = 1
};
KnownSourceType detectKnownSource(const std::string &input);
bool isScreenedJson(const std::string &input);
int definePrefixedJson(const std::string &input);
bool detectJSONasParameter(const std::string &s,
std::string &key,
std::string &value);

49
components/security_apps/waap/waap_clib/Waf2Util2.cc
View File

@@ -1,5 +1,7 @@
#include "Waf2Util.h"
#include "Waf2Regex.h"
#include <string>
#include "debug.h"
namespace Waap {
namespace Util {
@@ -628,5 +630,52 @@ isValidJson(const std::string &input)
return false;
}
KnownSourceType
detectKnownSource(const std::string &input)
{
static bool err = false;
static const SingleRegex known_source_sensor_data_re(
"^\\{\\\"sensor_data\\\":\\\"",
err,
"known_source_sensor_data"
);
if (known_source_sensor_data_re.hasMatch(input)) {
return SOURCE_TYPE_SENSOR_DATA;
}
return SOURCE_TYPE_UNKNOWN;
}
int
definePrefixedJson(const std::string &input)
{
static const size_t MAX_JSON_PREFIX_LEN = 32;
static const size_t MIN_PARAMETER_LEN = 4;
if (input.size() < MIN_PARAMETER_LEN) {
return -1;
}
for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) {
if (input[i] == '-' && input[i+1] == '{') return i + 1;
}
return -1;
}
bool
isScreenedJson(const std::string &input)
{
static bool err = false;
static const SingleRegex screened_json_re(
R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])",
err,
"screened_json"
);
if (screened_json_re.hasMatch(input)) {
return true;
}
return false;
}
} // namespace Util
} // namespace Waap

13
components/security_apps/waap/waap_component_impl.cc
View File

@@ -50,8 +50,7 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer(),
waapModelResultLogger()
deepAnalyzer()
{
}
@@ -536,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
verdict = drop_response.getVerdict();
}
bool sould_inject_response = waf2Transaction.shouldInjectResponse();
// in Chunked transfer encoding the last chunk is always empty - and we leave it empty
bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0;
dbgTrace(D_WAAP)
<< (sould_inject_response ? "should Inject Response" : "should not Inject Response")
<< (should_stay_empty_chunk ? " empty last chunk will stay empty" : "");
if (verdict == pending_response.getVerdict() &&
waf2Transaction.shouldInjectResponse() &&
!event.isLastChunk()
sould_inject_response &&
!should_stay_empty_chunk
) {
// Inject if needed. Note that this is only reasonable to do if there was no DROP decision above

2
components/security_apps/waap/waap_component_impl.h
View File

@@ -19,7 +19,6 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@@ -81,7 +80,6 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};

14
components/utils/generic_rulebase/match_query.cc
View File

@@ -111,7 +111,7 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in)
is_specific_label = false;
}
}
is_ignore_keyword = (key == "indicator");
is_ignore_keyword = (key == "indicator" || key == "keyword");
if (condition_type != Conditions::Exist) {
archive_in(cereal::make_nvp("value", value));
@@ -244,9 +244,10 @@ MatchQuery::getAllKeys() const
bool
MatchQuery::matchAttributes(
const unordered_map<string, set<string>> &key_value_pairs,
set<string> &matched_override_keywords ) const
set<string> &matched_override_keywords) const
{
dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes";
if (type == MatchType::Condition) {
auto key_value_pair = key_value_pairs.find(key);
if (key_value_pair == key_value_pairs.end()) {
@@ -257,9 +258,11 @@ MatchQuery::matchAttributes(
} else if (type == MatchType::Operator && operator_type == Operators::And) {
for (const MatchQuery &inner_match: items) {
if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) {
dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator";
return false;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator";
return true;
} else if (type == MatchType::Operator && operator_type == Operators::Or) {
// With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched
@@ -272,6 +275,7 @@ MatchQuery::matchAttributes(
res = true;
}
}
dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res;
return res;
} else {
dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type";
@@ -285,6 +289,7 @@ MatchQuery::getMatch( const unordered_map<string, set<string>> &key_value_pairs)
MatchQuery::MatchResult matches;
matches.matched_keywords = make_shared<set<string>>();
matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords);
dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match;
return matches;
}
@@ -306,10 +311,13 @@ MatchQuery::matchAttributes(
if (isKeyTypeIp()) {
match = matchAttributesIp(values);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for IP address: " << match;
} else if (isRegEx()) {
match = matchAttributesRegEx(values, matched_override_keywords);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for regex: " << match;
} else {
match = matchAttributesString(values);
dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match;
}
return negate ? !match : match;
@@ -324,6 +332,8 @@ MatchQuery::matchAttributesRegEx(
boost::cmatch value_matcher;
for (const boost::regex &val_regex : regex_values) {
for (const string &requested_match_value : values) {
dbgTrace(D_RULEBASE_CONFIG) << "Matching value: '" << requested_match_value
<< "' with regex: '" << val_regex << "'";
if (NGEN::Regex::regexMatch(
__FILE__,
__LINE__,

5
components/utils/generic_rulebase/parameters_config.cc
View File

@@ -119,10 +119,13 @@ ParameterException::getBehavior(
for (const MatchBehaviorPair &match_behavior_pair: match_queries) {
MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs);
if (match_res.is_match) {
dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches.";
dbgTrace(D_RULEBASE_CONFIG)
<< "Successfully matched an exception from a list of matches, behavior: "
<< match_behavior_pair.behavior.getId();
// When matching indicators with action=ignore, we expect no behavior override.
// Instead, a matched keywords list should be returned which will be later removed from score calculation
if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) {
dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore";
matched_override_keywords.insert(match_res.matched_keywords->begin(),
match_res.matched_keywords->end());
} else {