github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-11-28 10:41:59 +00:00
parent 6255e1f30d
commit 1c1f0b7e29
59 changed files with 842 additions and 707 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
components/security_apps/waap/include/i_waap_model_result_logger.h
View File

@@ -1,37 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
class IWaf2Transaction;
struct Waf2ScanResult;
namespace Waap {
namespace Scores {
struct ModelLoggingSettings;
}
}
class I_WaapModelResultLogger {
public:
virtual ~I_WaapModelResultLogger() {}
virtual void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double newScore,
double baseScore) = 0;
};

3
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -87,9 +87,10 @@ add_library(waap_clib
ParserPairs.cc
Waf2Util2.cc
ParserPDF.cc
ParserKnownBenignSkipper.cc
ParserScreenedJson.cc
ParserBinaryFile.cc
RegexComparator.cc
WaapModelResultLogger.cc
)
add_definitions("-Wno-unused-function")

71
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -28,6 +28,8 @@
#include "ParserDelimiter.h"
#include "ParserPDF.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
#include "ParserScreenedJson.h"
#include "WaapAssetState.h"
#include "Waf2Regex.h"
#include "Waf2Util.h"
@@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64BinaryFileType
@@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
base64ParamFound,
@@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,
@@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth,
b64FileType
@@ -918,6 +925,7 @@ bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
) {
@@ -959,6 +967,7 @@ DeepParser::createInternalParser(
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@@ -978,7 +987,19 @@ DeepParser::createInternalParser(
<< "\n\tflags: "
<< flags
<< "\n\tparser_depth: "
<< parser_depth;
<< parser_depth
<< "\n\tisBodyPayload: "
<< isBodyPayload
<< "\n\tisRefererPayload: "
<< isRefererPayload
<< "\n\tisRefererParamPayload: "
<< isRefererParamPayload
<< "\n\tisUrlPayload: "
<< isUrlPayload
<< "\n\tisUrlParamPayload: "
<< isUrlParamPayload
<< "\n\tisCookiePayload: "
<< isCookiePayload;
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false,
isAmperType = false;
bool isKeyValDelimited = false;
@@ -1045,6 +1066,53 @@ DeepParser::createInternalParser(
}
}
if (Waap::Util::isScreenedJson(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserScreenedJson>>(*this, parser_depth + 1));
offset = 0;
return offset;
}
dbgTrace(D_WAAP_DEEP_PARSER)
<< "Offset = "
<< offset
<< " depth = "
<< m_depth
<< " isBodyPayload = "
<< isBodyPayload;
//Detect sensor_data format in body and just use dedicated filter for it
if (m_depth == 1
&& isBodyPayload
&& Waap::Util::detectKnownSource(cur_val) == Waap::Util::SOURCE_TYPE_SENSOR_DATA) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserKnownBenignSkipper>>(
*this,
parser_depth + 1,
Waap::Util::SOURCE_TYPE_SENSOR_DATA
)
);
offset = 0;
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it";
return offset;
}
// Detect cookie parameter sensorsdata2015jssdkcross
// and causes false positives due to malformed JSON. Make preprocessing to parse it correctly
if (m_depth == 2
&& isCookiePayload) {
offset = Waap::Util::definePrefixedJson(cur_val);
if (offset >= 0) {
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserJson>>(
*this,
parser_depth + 1,
m_pTransaction
)
);
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data";
return offset;
}
}
// Detect wbxml (binary XML) data type
if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) {
m_is_wbxml = true;
@@ -1374,6 +1442,7 @@ DeepParser::createInternalParser(
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload,
isCookiePayload,
flags,
parser_depth
);

3
components/security_apps/waap/waap_clib/DeepParser.h
View File

@@ -129,6 +129,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
Waap::Util::BinaryFileType b64FileType
@@ -144,6 +145,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth
);
@@ -160,6 +162,7 @@ private:
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
bool isCookiePayload,
int flags,
size_t parser_depth,
bool base64ParamFound,

96
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const
#endif
}
// std::string nicePrint() - is a function used to create std::string that will represent all data that is
// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive
// into the object easier.
std::string
WaapAssetState::nicePrint(Waf2ScanResult &res) const
{
std::string result = "Waf2ScanResult:\n";
result += "keyword_matches:\n";
for (const auto &keyword : res.keyword_matches) {
result += keyword + "\n";
}
result += "regex_matches:\n";
for (const auto &regex : res.regex_matches) {
result += regex + "\n";
}
result += "filtered_keywords:\n";
for (const auto &filtered : res.filtered_keywords) {
result += filtered + "\n";
}
result += "found_patterns:\n";
for (const auto &pattern : res.found_patterns) {
result += pattern.first + ":\n";
for (const auto &value : pattern.second) {
result += value + "\n";
}
}
result += "unescaped_line: " + res.unescaped_line + "\n";
result += "param_name: " + res.param_name + "\n";
result += "location: " + res.location + "\n";
result += "score: " + std::to_string(res.score) + "\n";
result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n";
result += "scoreArray:\n";
for (const auto &score : res.scoreArray) {
result += std::to_string(score) + "\n";
}
result += "keywordCombinations:\n";
for (const auto &combination : res.keywordCombinations) {
result += combination + "\n";
}
result += "attack_types:\n";
for (const auto &attack : res.attack_types) {
result += attack + "\n";
}
result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n";
return result;
}
bool
checkBinaryData(const std::string &line, bool binaryDataFound)
{
@@ -1033,7 +1082,7 @@ WaapAssetState::apply(
// Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below
// This is done to improve performance of regex matching.
SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res);
checkRegex(
unescapedLineSample,
m_Signatures->specific_acuracy_keywords_regex,
@@ -1111,7 +1160,7 @@ WaapAssetState::apply(
}
bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns);
dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res);
if (os_cmd_ev) {
dbgTrace(D_WAAP_EVASIONS) << "os command evasion found";
@@ -1295,6 +1344,47 @@ WaapAssetState::apply(
}
}
bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns);
dbgTrace(D_WAAP_EVASIONS)
<< "path_traversal_ev = " << path_traversal_ev
<< " sample = " << res.unescaped_line
<< " res.unescaped_line.find(2f) = " << res.unescaped_line.find("2f");
if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) {
// Possible path traversal evasion .2f. detected: - clean up and scan with regexes again.
dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line
<< "Status beroe evasion checking " << nicePrint(res);
std::string unescaped = line;
replaceAll(unescaped, "2f", "/");
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount == res.keyword_matches.size()) {
// Remove the evasion keyword if no real evasion found
keywordsToRemove.push_back("path_traversal");
path_traversal_ev = false;
}
else if (!binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res);
}
bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys(
"quotes_space_ev_fast_reg",
res.found_patterns
@@ -1726,7 +1816,7 @@ WaapAssetState::apply(
wordsCount = std::min(wordsCount, newWordsCount);
}
}
dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res);
// Remove evasion keywords that should not be reported because there's no real evasion found
if (!keywordsToRemove.empty()) {
dbgTrace(D_WAAP_SAMPLE_SCAN)

1
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -49,6 +49,7 @@ private: //ugly but needed for build
Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const;
void filterKeywordsDueToLongText(Waf2ScanResult &res) const;
std::string nicePrint(Waf2ScanResult &res) const;
public:
// Load and compile signatures from file

250
components/security_apps/waap/waap_clib/WaapModelResultLogger.cc
View File

@@ -1,250 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "WaapModelResultLogger.h"
#include "Waf2Engine.h"
#include "i_time_get.h"
#include "i_messaging.h"
#include "i_instance_awareness.h"
#include "http_manager.h"
#include "LogGenWrapper.h"
#include "rest.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER);
using namespace std;
static const unsigned int MAX_FILES_PER_WINDOW = 5;
static const unsigned int MAX_LOGS_PER_WINDOW = 1800;
static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30);
class WaapModelReport : public RestGetFile
{
public:
WaapModelReport(const vector<WaapModelResult> &_data) : data(_data) {}
private:
C2S_PARAM(vector<WaapModelResult>, data);
};
class WaapModelResultLogger::Impl
:
Singleton::Provide<I_WaapModelResultLogger>::From<WaapModelResultLogger>
{
public:
Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0),
last_sent_s3(std::chrono::minutes::zero()),
last_kusto_log_window(std::chrono::minutes::zero()) {}
virtual ~Impl();
void
logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore) override;
private:
void logToStream(WaapModelResult &result, chrono::minutes now);
void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now);
bool shouldSendLogsToS3(chrono::minutes now);
void sendLogsToS3();
size_t max_logs;
unsigned int sent_files_count;
unsigned int sent_logs_count;
std::chrono::minutes last_sent_s3;
std::chrono::minutes last_kusto_log_window;
std::map<std::string, vector<WaapModelResult>> logs;
};
WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique<WaapModelResultLogger::Impl>(maxLogs))
{
}
WaapModelResultLogger::~WaapModelResultLogger()
{
}
void
WaapModelResultLogger::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
)
{
pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore);
}
void
WaapModelResultLogger::Impl::logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
string modelName,
string otherModelName,
double score,
double otherScore)
{
if (transaction == NULL) return;
if (!Singleton::exists<I_Messaging>()) {
dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log";
return;
}
double score_diff = score - otherScore;
if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF &&
! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) ||
(score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) {
return;
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
WaapModelResult result = WaapModelResult(
*transaction,
res,
modelName,
otherModelName,
score,
otherScore,
now.count()
);
if (settings.logToStream) logToStream(result, now);
if (settings.logToS3) logToS3(result, transaction, now);
}
void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now)
{
auto asset_state = transaction->getAssetState();
string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : "";
auto asset_logs = logs.find(asset_id);
if (asset_logs == logs.end()) {
logs.emplace(asset_id, vector<WaapModelResult>());
}
logs.at(asset_id).push_back(result);
if (shouldSendLogsToS3(now)) {
sendLogsToS3();
}
}
void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now)
{
if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) {
last_kusto_log_window = now;
sent_logs_count = 0;
}
else if (sent_logs_count > MAX_LOGS_PER_WINDOW) {
return;
}
sent_logs_count++;
dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry";
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
LogGenWrapper logGenWrapper(
maybeLogTriggerConf,
"WAAP Model Telemetry",
ReportIS::Audience::SECURITY,
LogTriggerConf::SecurityType::ThreatPrevention,
ReportIS::Severity::CRITICAL,
ReportIS::Priority::HIGH,
false);
LogGen& waap_log = logGenWrapper.getLogGen();
waap_log.addMarkerSuffix(result.location);
waap_log << LogField("httpuripath", result.uri);
waap_log << LogField("matchedlocation", result.location);
waap_log << LogField("matchedparameter", result.param);
waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64);
waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64);
waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100));
waap_log << LogField("waapfinalscore", (int)(result.score * 100));
waap_log << LogField("indicatorssource", result.modelName);
waap_log << LogField("indicatorsversion", result.otherModelName);
}
bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now)
{
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true;
for (const auto &asset_logs : logs) {
if (asset_logs.second.size() >= max_logs) return true;
}
return false;
}
void WaapModelResultLogger::Impl::sendLogsToS3()
{
dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog";
I_Messaging *msg = Singleton::Consume<I_Messaging>::by<WaapComponent>();
for (auto &asset_logs : logs) {
if (asset_logs.second.empty()) {
continue;
}
if (sent_files_count >= MAX_FILES_PER_WINDOW) {
dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window";
asset_logs.second.clear();
continue;
}
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
string tenant_id = agentDetails->getTenantId();
string agent_id = agentDetails->getAgentId();
string asset_id = asset_logs.first;
if (Singleton::exists<I_InstanceAwareness>()) {
I_InstanceAwareness* instance = Singleton::Consume<I_InstanceAwareness>::by<WaapComponent>();
Maybe<string> uniqueId = instance->getUniqueID();
if (uniqueId.ok())
{
agent_id += "/" + uniqueId.unpack();
}
}
string uri = "/storage/waap/" +
tenant_id + "/" + asset_id + "/waap_model_results/window_" +
to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) +
"/" + agent_id + "/data.data";
WaapModelReport report = WaapModelReport(asset_logs.second);
dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first <<
", length " << asset_logs.second.size() <<
", uri " << uri;
msg->sendAsyncMessage(
HTTPMethod::PUT,
uri,
report,
MessageCategory::LOG
);
asset_logs.second.clear();
}
auto current_time = Singleton::Consume<I_TimeGet>::by<WaapComponent>()->getWalltime();
auto now = chrono::duration_cast<chrono::minutes>(current_time);
if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) {
last_sent_s3 = now;
sent_files_count = 0;
} else {
sent_files_count++;
}
}
WaapModelResultLogger::Impl::~Impl()
{}

109
components/security_apps/waap/waap_clib/WaapModelResultLogger.h
View File

@@ -1,109 +0,0 @@
// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <memory>
#include <string>
#include <map>
#include <chrono>
#include <vector>
#include <ostream>
#include "cereal/archives/json.hpp"
#include "i_waap_model_result_logger.h"
#include "DeepAnalyzer.h"
#include "i_transaction.h"
#include "ScanResult.h"
#include "WaapAssetState.h"
#include "WaapScores.h"
class WaapModelResultLogger
:
Singleton::Provide<I_WaapModelResultLogger>
{
public:
WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS);
virtual ~WaapModelResultLogger();
virtual void logModelResult(
Waap::Scores::ModelLoggingSettings &settings,
IWaf2Transaction* transaction,
Waf2ScanResult &res,
std::string modelName,
std::string otherModelName,
double score,
double otherScore
);
class Impl;
protected:
std::unique_ptr<Impl> pimpl;
static const size_t MAX_WAAP_MODEL_LOGS = 20000;
};
class WaapModelResult
{
public:
WaapModelResult(
IWaf2Transaction &transaction,
Waf2ScanResult &res,
const std::string &modelName,
const std::string &otherModelName,
double score,
double otherScore,
uint64_t time
) : uri(transaction.getUri()), location(res.location), param(res.param_name),
modelName(modelName), otherModelName(otherModelName),
score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter),
sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time)
{
}
template<class Archive>
void serialize(Archive &ar) const
{
ar(cereal::make_nvp("uri", uri));
ar(cereal::make_nvp("location", location));
ar(cereal::make_nvp("param", param));
ar(cereal::make_nvp("modelName", modelName));
ar(cereal::make_nvp("otherModelName", otherModelName));
ar(cereal::make_nvp("score", score));
ar(cereal::make_nvp("otherScore", otherScore));
ar(cereal::make_nvp("keywords", keywords));
ar(cereal::make_nvp("sample", sample));
ar(cereal::make_nvp("id", id));
ar(cereal::make_nvp("time", time));
}
std::string toString() const
{
std::stringstream message_stream;
{
cereal::JSONOutputArchive ar(message_stream);
serialize(ar);
}
return message_stream.str();
}
std::string uri;
std::string location;
std::string param;
std::string modelName;
std::string otherModelName;
double score;
double otherScore;
std::vector<std::string> keywords;
std::string sample;
uint64_t id;
uint64_t time;
};

5
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@@ -48,8 +48,9 @@ public:
m_tag = to_lower_copy(m_tag);
if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") {
m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" &&
m_tag != "paramlocation" && m_tag != "responsebody" && m_tag != "headername" &&
m_tag != "headervalue" && m_tag != "method") {
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -105,7 +105,7 @@ bool WaapOverrideFunctor::operator()(
}
return false;
}
else if (tagLower == "keyword") {
else if (tagLower == "keyword" || tagLower == "indicator") {
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {

12
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -15,7 +15,6 @@
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@@ -111,10 +110,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
std::string other_pool_name = Waap::Scores::getOtherScorePoolName();
@@ -123,14 +118,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
@@ -142,6 +133,9 @@ double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {

11
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -1358,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
return false;
}
std::string body(data);
std::string body(data, data_len);
if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body))
{
dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false";
@@ -1661,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("sourcePort", m_remote_port);
waapLog << LogField("httpHostName", m_hostStr);
waapLog << LogField("httpMethod", m_methodStr);
if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId());
if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName());
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
bool send_extended_log = shouldSendExtendedLog(triggerLog);
@@ -2343,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
exceptions_dict["sourceIdentifier"].insert(m_source_identifier);
exceptions_dict["url"].insert(getUriStr());
exceptions_dict["hostName"].insert(m_hostStr);
exceptions_dict["method"].insert(m_methodStr);
for (auto &keyword : res.keyword_matches) {
exceptions_dict["indicator"].insert(keyword);
@@ -2355,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (const auto &behavior : behaviors) {
dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId();
if (!res.filtered_keywords.empty() || res.score > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators";
std::string overrideId = behavior.getId();
if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
@@ -2375,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
}
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore.";
std::string overrideId = behavior.getId();
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);

1
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -41,7 +41,6 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"

23
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -35,6 +35,7 @@
#include "user_identifiers_config.h"
#include "Waf2Regex.h"
#include "ParserBinaryFile.h"
#include "ParserKnownBenignSkipper.h"
using boost::algorithm::to_lower_copy;
using namespace std;
@@ -1218,21 +1219,21 @@ static const SingleRegex csp_report_policy_re(
"csp_report_policy"
);
static const SingleRegex base64_key_value_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
"^[^<>{};,&\\?|=\\s]+={1}\\s*.+",
err,
"base64_key_value");
static const SingleRegex json_key_value_detector_re(
"\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
err,
"json_key_value");
err,
"json_key_value");
static const SingleRegex base64_key_detector_re(
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
"^[^<>{};,&\\?|=\\s]+={1}",
err,
"base64_key");
static const SingleRegex base64_prefix_detector_re(
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
"data:\\S*;base64,\\S+|base64,\\S+",
err,
"base64_prefix");
// looks for combination <param>={<some text>*:<some text>*}
//used to allow parsing param=JSON to reduce false positives

10
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -894,6 +894,16 @@ namespace Util {
bool isValidJson(const std::string &input);
enum KnownSourceType {
SOURCE_TYPE_UNKNOWN = 0,
SOURCE_TYPE_SENSOR_DATA = 1
};
KnownSourceType detectKnownSource(const std::string &input);
bool isScreenedJson(const std::string &input);
int definePrefixedJson(const std::string &input);
bool detectJSONasParameter(const std::string &s,
std::string &key,
std::string &value);

49
components/security_apps/waap/waap_clib/Waf2Util2.cc
View File

@@ -1,5 +1,7 @@
#include "Waf2Util.h"
#include "Waf2Regex.h"
#include <string>
#include "debug.h"
namespace Waap {
namespace Util {
@@ -628,5 +630,52 @@ isValidJson(const std::string &input)
return false;
}
KnownSourceType
detectKnownSource(const std::string &input)
{
static bool err = false;
static const SingleRegex known_source_sensor_data_re(
"^\\{\\\"sensor_data\\\":\\\"",
err,
"known_source_sensor_data"
);
if (known_source_sensor_data_re.hasMatch(input)) {
return SOURCE_TYPE_SENSOR_DATA;
}
return SOURCE_TYPE_UNKNOWN;
}
int
definePrefixedJson(const std::string &input)
{
static const size_t MAX_JSON_PREFIX_LEN = 32;
static const size_t MIN_PARAMETER_LEN = 4;
if (input.size() < MIN_PARAMETER_LEN) {
return -1;
}
for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) {
if (input[i] == '-' && input[i+1] == '{') return i + 1;
}
return -1;
}
bool
isScreenedJson(const std::string &input)
{
static bool err = false;
static const SingleRegex screened_json_re(
R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])",
err,
"screened_json"
);
if (screened_json_re.hasMatch(input)) {
return true;
}
return false;
}
} // namespace Util
} // namespace Waap

13
components/security_apps/waap/waap_component_impl.cc
View File

@@ -50,8 +50,7 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer(),
waapModelResultLogger()
deepAnalyzer()
{
}
@@ -536,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
verdict = drop_response.getVerdict();
}
bool sould_inject_response = waf2Transaction.shouldInjectResponse();
// in Chunked transfer encoding the last chunk is always empty - and we leave it empty
bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0;
dbgTrace(D_WAAP)
<< (sould_inject_response ? "should Inject Response" : "should not Inject Response")
<< (should_stay_empty_chunk ? " empty last chunk will stay empty" : "");
if (verdict == pending_response.getVerdict() &&
waf2Transaction.shouldInjectResponse() &&
!event.isLastChunk()
sould_inject_response &&
!should_stay_empty_chunk
) {
// Inject if needed. Note that this is only reasonable to do if there was no DROP decision above

2
components/security_apps/waap/waap_component_impl.h
View File

@@ -19,7 +19,6 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@@ -81,7 +80,6 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};