github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 11:16:30 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jan_31_2024-Dev

Browse Source
This commit is contained in:
Ned Wright
2024-01-31 17:34:53 +00:00
parent 752a5785f0
commit 6d67818a94
376 changed files with 8101 additions and 7064 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

@@ -16,7 +16,6 @@
#include "waap.h"
#include "ConfidenceFile.h"
#include "i_agent_details.h"
#include "i_messaging.h"
#include "i_mainloop.h"
#include <math.h>
@@ -138,7 +137,7 @@ bool ConfidenceCalculator::postData()
WindowLogPost currentWindow(m_time_window_logger_backup);
bool ok = sendNoReplyObjectWithRetry(currentWindow,
I_Messaging::Method::PUT,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
@@ -164,7 +163,7 @@ void ConfidenceCalculator::pullData(const std::vector<std::string>& files)
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Pulling the file: " << file;
WindowLogGet getWindow;
bool ok = sendObjectWithRetry(getWindow,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/" + file);
if (!ok) {
@@ -213,7 +212,7 @@ void ConfidenceCalculator::pullProcessedData(const std::vector<std::string>& fil
{
ConfidenceFileDecryptor getConfFile;
bool res = sendObjectWithRetry(getConfFile,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/" + file);
is_ok |= res;
if (res && getConfFile.getConfidenceSet().ok())
@@ -243,7 +242,7 @@ void ConfidenceCalculator::postProcessedData()
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Posting the confidence set object to: " << postUrl;
ConfidenceFileEncryptor postConfFile(m_confident_sets, m_confidence_level);
sendNoReplyObjectWithRetry(postConfFile,
I_Messaging::Method::PUT,
HTTPMethod::PUT,
postUrl);
}

29
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -187,6 +187,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
bool isCookiePayload = (m_key.first().size() == 6 && m_key.first() == "cookie");
bool isBodyPayload = (m_key.first().size() == 4 && m_key.first() == "body");
// If csrf/antibot cookie - send to Waf2Transaction for collection of cookie value.
if (m_depth == 1 && isCookiePayload && (m_key.str() == "x-chkp-csrf-token" || m_key.str() == "__fn1522082288")) {
std::string cur_val = std::string(v, v_len);
@@ -388,21 +389,17 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
&& m_parsersDeque.size() > parser_depth
&&!m_parsersDeque.at(parser_depth)->getRecursionFlag()
) {
ScopedContext ctx;
ctx.registerValue<IWaf2Transaction *>("waap_transaction", m_pTransaction);
rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth);
if (rc != CONTINUE_PARSING) {
if (shouldUpdateKeyStack) {
m_key.pop("deep parser key");
}
m_depth--;
return rc;
}
}
if (rc == CONTINUE_PARSING) {
// Try to eliminate m_multipart_boundary to allow other parser to work instead of multipart
if (m_depth == 1
@@ -853,21 +850,17 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
&& m_parsersDeque.size() > parser_depth
&&!m_parsersDeque.at(parser_depth)->getRecursionFlag()
) {
ScopedContext ctx;
ctx.registerValue<IWaf2Transaction *>("waap_transaction", m_pTransaction);
rc = pushValueToTopParser(cur_val, flags, base64ParamFound, offset, parser_depth);
if (rc != CONTINUE_PARSING) {
if (shouldUpdateKeyStack) {
m_key.pop("deep parser key");
}
m_depth--;
return rc;
}
}
return rc;
}
@@ -1091,7 +1084,12 @@ DeepParser::createInternalParser(
) {
// Graphql value detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse graphql";
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserGql>>(*this, parser_depth + 1));
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserGql>>(
*this,
parser_depth + 1,
m_pTransaction));
offset = 0;
} else if (cur_val.length() > 0
&& (cur_val[0] == '[' || cur_val[0] == '{')
@@ -1123,12 +1121,12 @@ DeepParser::createInternalParser(
// but only if the JSON is passed in body and on the top level.
bool should_collect_for_oa_schema_updater = false;
m_parsersDeque.push_back(
std::make_shared<BufferedParser<ParserJson>>(
*this,
should_collect_for_oa_schema_updater,
parser_depth + 1
));
m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserJson>>(
*this,
parser_depth + 1,
m_pTransaction,
should_collect_for_oa_schema_updater
));
offset = 0;
}
}
@@ -1326,7 +1324,6 @@ DeepParser::createInternalParser(
return offset;
}
void
DeepParser::apiProcessKey(const char *v, size_t v_len)
{

1
components/security_apps/waap/waap_clib/IndicatorsFilterBase.h
View File

@@ -14,7 +14,6 @@
#pragma once
#include "i_indicatorsFilter.h"
#include "i_messaging.h"
#include "waap.h"
#include "TrustedSources.h"
#include "TrustedSourcesConfidence.h"

1
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.h
View File

@@ -19,7 +19,6 @@
#include "TypeIndicatorsFilter.h"
#include "WaapParameters.h"
#include "i_waapConfig.h"
#include "i_messaging.h"
#include "ScannersDetector.h"
#include "TuningDecisions.h"
#include <cereal/cereal.hpp>

18
components/security_apps/waap/waap_clib/ParserGql.cc
View File

@@ -16,15 +16,23 @@
#include "graphqlparser/AstVisitor.h"
#include "graphqlparser/GraphQLParser.h"
#include "debug.h"
#include "oas_updater_entry_saver.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_GQL);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
const std::string ParserGql::m_parserName = "gqlParser";
ParserGql::ParserGql(IParserReceiver &receiver, size_t parser_depth) :
ParserGql::ParserGql(
IParserReceiver &receiver,
size_t parser_depth,
IWaf2Transaction *pTransaction)
:
m_receiver(receiver),
m_error(false),
m_curNameValues(0),
m_pTransaction(pTransaction),
field_depth(0),
m_parser_depth(parser_depth)
{
dbgFlow(D_WAAP_PARSER_GQL);
@@ -93,6 +101,7 @@ bool ParserGql::visitValue(const char *value)
bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
{
dbgTrace(D_WAAP_PARSER_GQL) << node.getValue() << "'";
bool ret = true;
if (m_curNameValues == 0 && !m_curNodeName.empty()) {
ret = m_receiver.onKv(
@@ -105,6 +114,13 @@ bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
return ret;
}
bool ParserGql::visitOperationDefinition(const facebook::graphql::ast::OperationDefinition &node)
{
dbgFlow(D_OA_SCHEMA_UPDATER) << "getOperation()";
return true;
}
bool ParserGql::visitIntValue(const facebook::graphql::ast::IntValue &node)
{
dbgFlow(D_WAAP_PARSER_GQL);

17
components/security_apps/waap/waap_clib/ParserGql.h
View File

@@ -22,10 +22,19 @@
#include "graphqlparser/AstNode.h"
#include "graphqlparser/AstVisitor.h"
#include "KeyStack.h"
#include "i_transaction.h"
#include "singleton.h"
#include "i_oa_schema_updater.h"
class ParserGql : public ParserBase, public facebook::graphql::ast::visitor::AstVisitor {
class ParserGql :
public ParserBase,
public facebook::graphql::ast::visitor::AstVisitor,
Singleton::Consume<I_OASUpdater> {
public:
ParserGql(IParserReceiver &receiver, size_t parser_depth);
ParserGql(
IParserReceiver &receiver,
size_t parser_depth,
IWaf2Transaction *pTransaction=nullptr);
virtual ~ParserGql();
size_t push(const char *data, size_t data_len);
void finish();
@@ -38,10 +47,12 @@ private:
std::string m_buffer;
std::string m_curNodeName;
int m_curNameValues;
IWaf2Transaction *m_pTransaction;
int field_depth;
bool visitValue(const char *value);
// Callbacks from the parser
bool visitOperationDefinition(const facebook::graphql::ast::OperationDefinition &node) override;
bool visitName(const facebook::graphql::ast::Name &node) override;
bool visitIntValue(const facebook::graphql::ast::IntValue &node) override;
bool visitFloatValue(const facebook::graphql::ast::FloatValue &node) override;

9
components/security_apps/waap/waap_clib/ParserJson.cc
View File

@@ -249,8 +249,9 @@ ParserJson::p_end_array(void *ctx)
ParserJson::ParserJson(
IParserReceiver &receiver,
bool should_collect_oas,
size_t parser_depth,
IWaf2Transaction *pTransaction,
bool should_collect_oas,
IParserReceiver2 *receiver2)
:
m_receiver(receiver),
@@ -260,10 +261,12 @@ ParserJson::ParserJson(
m_key("json_parser"),
m_jsonHandler(NULL),
is_map_empty(false),
should_collect_for_oa_schema_updater(should_collect_oas),
m_parser_depth(parser_depth)
m_pTransaction(pTransaction),
m_parser_depth(parser_depth),
is_graphql_operation_name(false)
{
dbgTrace(D_WAAP_PARSER_JSON) << "parser_depth= " << parser_depth;
should_collect_for_oa_schema_updater = should_collect_oas;
// TODO:: do we really want to clear this?
memset(m_buf, 0, sizeof(m_buf));

12
components/security_apps/waap/waap_clib/ParserJson.h
View File

@@ -20,17 +20,22 @@
#include "ParserBase.h"
#include "KeyStack.h"
#include "yajl/yajl_parse.h"
#include "singleton.h"
#include "i_oa_schema_updater.h"
#define FIRST_JSON_BUFFER_SIZE 4 // must buffer at least 4 first bytes to allow unicode autodetection (BOM).
typedef size_t yajl_size_t;
class ParserJson : public ParserBase {
class ParserJson :
public ParserBase,
Singleton::Consume<I_OASUpdater> {
public:
ParserJson(
IParserReceiver &receiver,
bool should_collect_for_oa_schema_updater=false,
size_t parser_depth=0,
IWaf2Transaction *pTransaction = nullptr,
bool should_collect_for_oa_schema_updater=false,
IParserReceiver2 *receiver2=NULL);
virtual ~ParserJson();
size_t push(const char *data, size_t data_len);
@@ -86,8 +91,9 @@ private:
yajl_handle m_jsonHandler;
bool is_map_empty;
bool should_collect_for_oa_schema_updater;
IWaf2Transaction *m_pTransaction;
size_t m_parser_depth;
bool is_graphql_operation_name;
public:
static const std::string m_parserName;
};

5
components/security_apps/waap/waap_clib/ScannerDetector.cc
View File

@@ -13,7 +13,6 @@
#include "ScannersDetector.h"
#include "waap.h"
#include "i_messaging.h"
#include <boost/algorithm/string/predicate.hpp>
USE_DEBUG_FLAG(D_WAAP);
@@ -101,7 +100,7 @@ bool ScannerDetector::postData()
SourcesMonitorPost currentWindow(m_sources_monitor_backup);
bool ok = sendNoReplyObjectWithRetry(currentWindow,
I_Messaging::Method::PUT,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP) << "Failed to post collected data to: " << url;
@@ -123,7 +122,7 @@ void ScannerDetector::pullData(const std::vector<std::string>& files)
dbgTrace(D_WAAP) << "Pulling the file: " << file;
SourcesMonitorGet getMonitor;
bool ok = sendObjectWithRetry(getMonitor,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/" + file);
if (!ok) {

54
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -16,6 +16,7 @@
#include "Waf2Util.h"
#include "WaapAssetState.h"
#include "i_instance_awareness.h"
#include <boost/regex.hpp>
#include <sstream>
#include <fstream>
#include <functional>
@@ -387,15 +388,14 @@ const vector<FileMetaData>& RemoteFilesList::getFilesMetadataList() const
return files.get();
}
SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
ch::minutes interval,
ch::seconds waitForSync,
const string& filePath,
const string& remotePath,
const string& assetId,
const string& owner)
:
const string &filePath,
const string &remotePath,
const string &assetId,
const string &owner
) :
SerializeToFileBase(filePath),
m_remotePath(remotePath),
m_interval(0),
@@ -408,6 +408,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_intervalsCounter(0),
m_remoteSyncEnabled(true),
m_assetId(assetId),
m_isAssetIdUuid(Waap::Util::isUuid(assetId)),
m_shared_storage_host(genError("not set")),
m_learning_host(genError("not set"))
{
@@ -600,7 +601,7 @@ bool SerializeToLocalAndRemoteSyncBase::localSyncAndProcess()
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Getting files of all agents";
bool isSuccessful = sendObjectWithRetry(rawDataFiles,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/?list-type=2&prefix=" + m_remotePath + "/" + getWindowId() + "/");
if (!isSuccessful)
@@ -655,14 +656,16 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
if (!m_remoteSyncEnabled || isBase() || !postData() ||
mode == OrchestrationMode::OFFLINE)
{
if (mode == OrchestrationMode::OFFLINE || !m_remoteSyncEnabled || isBase() ||
(mode == OrchestrationMode::ONLINE && !m_isAssetIdUuid) || !postData()) {
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR)
<< "Did not synchronize the data. Remote URL: "
<< "Did not synchronize the data. for asset: "
<< m_assetId
<< " Remote URL: "
<< m_remotePath
<< " is enabled: "
<< to_string(m_remoteSyncEnabled);
<< to_string(m_remoteSyncEnabled)
<< ", mode: " << int(mode);
processData();
saveData();
return;
@@ -701,16 +704,15 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
SyncLearningObject syncObj(m_assetId, m_type, getWindowId());
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId();
bool ok = messaging->sendNoReplyObject(syncObj,
I_Messaging::Method::POST,
getLearningHost(),
80,
conn_flags,
"/api/sync",
tenant_header);
MessageMetadata req_md(getLearningHost(), 80);
req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
bool ok = messaging->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
"/api/sync",
syncObj,
MessageCategory::GENERIC,
req_md
);
dbgDebug(D_WAAP_CONFIDENCE_CALCULATOR) << "sent learning sync notification ok: " << ok;
if (!ok) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "failed to send learning notification";
@@ -725,7 +727,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
ReportIS::AudienceTeam::WAAP,
syncNotification,
false,
MessageTypeTag::WAAP_LEARNING,
MessageCategory::GENERIC,
ReportIS::Tags::WAF,
ReportIS::Notification::SYNC_LEARNING
);
@@ -762,7 +764,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getRemoteProcessedFilesList()
bool isSuccessful = sendObject(
remoteFiles,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/?list-type=2&prefix=" + m_remotePath + "/remote");
if (!isSuccessful)
@@ -795,7 +797,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
bool isSuccessful = sendObject(
processedFilesList,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/?list-type=2&prefix=" + m_remotePath + "/processed");
if (!isSuccessful)
@@ -832,7 +834,7 @@ RemoteFilesList SerializeToLocalAndRemoteSyncBase::getProcessedFilesList()
isSuccessful = sendObject(
processedFilesList,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/?list-type=2&prefix=" + bcRemotePath + "/processed");
if (!isSuccessful)

163
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@@ -23,8 +23,6 @@
USE_DEBUG_FLAG(D_WAAP);
#define LOGGING_INTERVAL_IN_MINUTES 10
using namespace std;
const static string default_host = "open-appsec-tuning-svc";
@@ -34,26 +32,25 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const
{
OrchestrationMode mode = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getOrchestrationMode();
GenericMetric::sendLog(metric_client_rest);
if (mode == OrchestrationMode::ONLINE) {
GenericMetric::sendLog(metric_client_rest);
return;
}
auto svc_host = getConfigurationWithDefault(default_host, "Logging", "K8sSvc Log host");
Flags<MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
string fog_metric_uri = getConfigurationWithDefault<string>("/api/v1/agents/events", "metric", "fogMetricUri");
std::string tenant_header = "X-Tenant-Id: " +
Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getTenantId();
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendNoReplyObject(
metric_client_rest,
I_Messaging::Method::POST,
svc_host,
80,
conn_flags,
MessageMetadata req_md(svc_host, 80);
req_md.insertHeader(
"X-Tenant-Id",
Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getTenantId()
);
Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
fog_metric_uri,
tenant_header,
nullptr,
MessageTypeTag::METRIC);
metric_client_rest,
MessageCategory::METRIC,
req_md
);
}
void
@@ -140,6 +137,56 @@ WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryDa
}
}
void
WaapTrafficTelemetrics::initMetrics()
{
post_requests.report(0);
get_requests.report(0);
put_requests.report(0);
patch_requests.report(0);
delete_requests.report(0);
other_requests.report(0);
response_2xx.report(0);
response_4xx.report(0);
response_5xx.report(0);
}
void
WaapTrafficTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
initMetrics();
switch (data.method)
{
case POST:
post_requests.report(1);
break;
case GET:
get_requests.report(1);
break;
case PUT:
put_requests.report(1);
break;
case PATCH:
patch_requests.report(1);
break;
case DELETE:
delete_requests.report(1);
break;
default:
other_requests.report(1);
break;
}
if (data.responseCode >= 500) {
response_5xx.report(1);;
} else if (data.responseCode >= 400) {
response_4xx.report(1);
} else if (data.responseCode >= 200 && data.responseCode < 300) {
response_2xx.report(1);
}
}
void
WaapAttackTypesMetrics::initMetrics()
{
@@ -195,85 +242,17 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
<< ", Threat level: "
<< data.threat;
if (!telemetries.count(asset_id)) {
telemetries.emplace(asset_id, make_shared<WaapTelemetrics>());
telemetries[asset_id]->init(
"WAAP telemetry",
ReportIS::AudienceTeam::WAAP,
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::SECURITY
);
telemetries[asset_id]->registerContext<string>(
"pracitceType",
string("Threat Prevention"),
EnvKeyAttr::LogSection::SOURCE
);
telemetries[asset_id]->registerContext<string>(
"practiceSubType",
string("Web Application"),
EnvKeyAttr::LogSection::SOURCE
);
telemetries[asset_id]->registerContext<string>("assetId", asset_id, EnvKeyAttr::LogSection::SOURCE);
telemetries[asset_id]->registerContext<string>("assetName", data.assetName, EnvKeyAttr::LogSection::SOURCE);
telemetries[asset_id]->registerContext<string>("practiceId", data.practiceId, EnvKeyAttr::LogSection::SOURCE);
telemetries[asset_id]->registerContext<string>(
"practiceName",
data.practiceName,
EnvKeyAttr::LogSection::SOURCE
);
telemetries[asset_id]->registerListener();
}
if (!attack_types_telemetries.count(asset_id)) {
attack_types_telemetries.emplace(asset_id, make_shared<WaapAttackTypesMetrics>());
attack_types_telemetries[asset_id]->init(
"WAAP attack type telemetry",
ReportIS::AudienceTeam::WAAP,
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::SECURITY
);
attack_types_telemetries[asset_id]->registerContext<string>(
"pracitceType",
string("Threat Prevention"),
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerContext<string>(
"practiceSubType",
string("Web Application"),
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerContext<string>(
"assetId",
asset_id,
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerContext<string>(
"assetName",
data.assetName,
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerContext<string>(
"practiceId",
data.practiceId,
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerContext<string>(
"practiceName",
data.practiceName,
EnvKeyAttr::LogSection::SOURCE
);
attack_types_telemetries[asset_id]->registerListener();
}
initializeTelemetryData<WaapTelemetrics>(asset_id, data, "WAAP telemetry", telemetries);
initializeTelemetryData<WaapAttackTypesMetrics>(
asset_id, data,
"WAAP attack type telemetry",
attack_types_telemetries
);
initializeTelemetryData<WaapTrafficTelemetrics>(asset_id, data, "WAAP traffic telemetry", traffic_telemetries);
telemetries[asset_id]->updateMetrics(asset_id, data);
attack_types_telemetries[asset_id]->updateMetrics(asset_id, data);
traffic_telemetries[asset_id]->updateMetrics(asset_id, data);
auto agent_mode = Singleton::Consume<I_AgentDetails>::by<WaapMetricWrapper>()->getOrchestrationMode();
string tenant_id = Singleton::Consume<I_AgentDetails>::by<WaapMetricWrapper>()->getTenantId();

9
components/security_apps/waap/waap_clib/TrustedSourcesConfidence.cc
View File

@@ -12,7 +12,6 @@
// limitations under the License.
#include "TrustedSourcesConfidence.h"
#include "i_messaging.h"
#include "waap.h"
#include "Waf2Util.h"
@@ -97,7 +96,7 @@ bool TrustedSourcesConfidenceCalculator::postData()
TrsutedSourcesLogger logger(m_logger);
bool ok = sendNoReplyObjectWithRetry(logger,
I_Messaging::Method::PUT,
HTTPMethod::PUT,
url);
if (!ok) {
dbgError(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to post collected data to: " << url;
@@ -118,7 +117,7 @@ void TrustedSourcesConfidenceCalculator::pullData(const std::vector<std::string>
}
GetTrustedFile getTrustFile;
bool res = sendObjectWithRetry(getTrustFile,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/" + file);
if (!res)
{
@@ -149,7 +148,7 @@ void TrustedSourcesConfidenceCalculator::pullProcessedData(const std::vector<std
for (auto file: files) {
GetTrustedFile getTrustFile;
bool res = sendObjectWithRetry(getTrustFile,
I_Messaging::Method::GET,
HTTPMethod::GET,
getUri() + "/" + file);
pull_ok |= res;
if (res && getTrustFile.getTrustedLogs().ok()) {
@@ -168,7 +167,7 @@ void TrustedSourcesConfidenceCalculator::postProcessedData()
TrsutedSourcesLogger logger(m_logger);
sendNoReplyObjectWithRetry(logger,
I_Messaging::Method::PUT,
HTTPMethod::PUT,
url);
}

4
components/security_apps/waap/waap_clib/TuningDecision.cc
View File

@@ -126,7 +126,7 @@ void TuningDecision::updateDecisions()
}
dbgTrace(D_WAAP) << "URI prefix: " << m_baseUri;
bool isSuccessful = sendObject(tuningDecisionFiles,
I_Messaging::Method::GET,
HTTPMethod::GET,
m_baseUri + "?list-type=2&prefix=" + m_remotePath);
if (!isSuccessful || tuningDecisionFiles.getFilesList().empty())
@@ -136,7 +136,7 @@ void TuningDecision::updateDecisions()
}
if (!sendObject(tuningEvents,
I_Messaging::Method::GET,
HTTPMethod::GET,
m_baseUri + tuningDecisionFiles.getFilesList()[0]))
{
return;

33
components/security_apps/waap/waap_clib/TuningDecisions.h
View File

@@ -52,34 +52,29 @@ private:
std::string getSharedStorageHost();
template<typename T>
bool sendObject(T &obj, I_Messaging::Method method, std::string uri)
bool sendObject(T &obj, HTTPMethod method, std::string uri)
{
I_Messaging *messaging = Singleton::Consume<I_Messaging>::by<WaapComponent>();
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) {
Flags <MessageConnConfig> conn_flags;
conn_flags.setFlag(MessageConnConfig::EXTERNAL);
std::string tenant_header = "X-Tenant-Id: " + agentDetails->getTenantId();
return messaging->sendObject(
obj,
MessageMetadata req_md(getSharedStorageHost(), 80);
req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
auto req_status = messaging->sendSyncMessage(
method,
getSharedStorageHost(),
80,
conn_flags,
uri,
tenant_header,
nullptr,
MessageTypeTag::WAAP_LEARNING);
obj,
MessageCategory::GENERIC,
req_md
);
return req_status.ok();
}
return messaging->sendObject(
obj,
auto req_status = messaging->sendSyncMessage(
method,
uri,
"",
nullptr,
true,
MessageTypeTag::WAAP_LEARNING);
obj,
MessageCategory::GENERIC
);
return req_status.ok();
}
std::string m_remotePath;

2
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -255,6 +255,8 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar)
}
const std::vector<std::string> &
WaapConfigBase::get_applicationUrls() const
{

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -49,7 +49,7 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
boost::cmatch what;
try {
if (tag == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUri().c_str(), what, rx);
}
else if (tag == "hostname") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);

1
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -181,7 +181,6 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
std::string key = std::string(k, k_len);
std::string value = std::string(v, v_len);
res.clear();
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: k='" << key <<
"' v='" << value << "'";

1
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -33,7 +33,6 @@ namespace Waap {
{
}
bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp,
const std::string &location, const std::string &param_name, const std::string &key);
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) override;

103
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -274,7 +274,6 @@ void Waf2Transaction::setCurrentAssetState(IWaapConfig* sitePolicy)
"couldn't set waapAssetState for asset... using original waapAssetState";
return;
}
m_pWaapAssetState = pCurrentWaapAssetState;
}
@@ -314,18 +313,7 @@ Waf2Transaction::Waf2Transaction() :
m_index(-1),
m_triggerLog(),
m_waf2TransactionFlags()
{
is_hybrid_mode =
Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<Waf2Transaction>()->getOrchestrationMode() == OrchestrationMode::HYBRID
: false;
if (is_hybrid_mode) {
max_grace_logs = getProfileAgentSettingWithDefault<int>(
10,
"rulebase.initialForcedSecurityLogsToLocalStorage.count"
);
}
}
{}
Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState) :
TableOpaqueSerialize<Waf2Transaction>(this),
@@ -356,18 +344,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
m_index(-1),
m_triggerLog(),
m_waf2TransactionFlags()
{
is_hybrid_mode =
Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<Waf2Transaction>()->getOrchestrationMode() == OrchestrationMode::HYBRID
: false;
if (is_hybrid_mode) {
max_grace_logs = getProfileAgentSettingWithDefault<int>(
10,
"rulebase.initialForcedSecurityLogsToLocalStorage.count"
);
}
}
{}
Waf2Transaction::~Waf2Transaction() {
dbgTrace(D_WAAP) << "Waf2Transaction::~Waf2Transaction: deleting m_requestBodyParser";
@@ -470,7 +447,6 @@ HeaderType Waf2Transaction::checkCleanHeader(const char* name, int name_len, con
}
}
}
}
return UNKNOWN_HEADER;
}
@@ -539,6 +515,7 @@ bool Waf2Transaction::checkIsScanningRequired()
m_siteConfig = &m_ngenAPIConfig;
auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
result |= m_siteConfig->get_WebAttackMitigation();
if(rateLimitingPolicy) {
result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus();
}
@@ -547,15 +524,14 @@ bool Waf2Transaction::checkIsScanningRequired()
if (userLimitsPolicy) {
result = true;
}
}
if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) {
} else if (WaapConfigApplication::getWaapSiteConfig(m_ngenSiteConfig)) {
m_siteConfig = &m_ngenSiteConfig;
auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
auto errorLimitingPolicy = m_siteConfig ? m_siteConfig->get_ErrorLimitingPolicy() : NULL;
auto csrfPolicy = m_siteConfig ? m_siteConfig->get_CsrfPolicy() : NULL;
auto userLimitsPolicy = m_siteConfig ? m_siteConfig->get_UserLimitsPolicy() : nullptr;
result |= m_siteConfig->get_WebAttackMitigation();
if (rateLimitingPolicy) {
result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus();
}
@@ -1592,7 +1568,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
// Count of bytes available to send to the log
std::string requestBodyToLog = (send_extended_log || triggerLog->webBody) ?
std::string requestBodyToLog = (triggerLog->webBody) ?
m_request_body : std::string();
std::string responseBodyToLog = m_response_body;
if (!shouldBlock && responseBodyToLog.empty())
@@ -1671,6 +1647,25 @@ Waf2Transaction::sendLog()
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
if (m_methodStr == "POST") {
telemetryData.method = POST;
} else if (m_methodStr == "GET") {
telemetryData.method = GET;
} else if (m_methodStr == "PUT") {
telemetryData.method = PUT;
} else if (m_methodStr == "PATCH") {
telemetryData.method = PATCH;
} else if (m_methodStr == "DELETE") {
telemetryData.method = DELETE;
} else {
telemetryData.method = OTHER;
}
if (m_responseStatus != 0) {
telemetryData.responseCode = m_responseStatus;
}
telemetryData.source = getSourceIdentifier();
telemetryData.assetName = m_siteConfig->get_AssetName();
telemetryData.practiceId = m_siteConfig->get_PracticeId();
@@ -1739,17 +1734,7 @@ Waf2Transaction::sendLog()
return;
}
static int cur_grace_logs = 0;
bool grace_period = is_hybrid_mode && cur_grace_logs < max_grace_logs;
bool send_extended_log = shouldSendExtendedLog(triggerLog);
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: current grace log index: "
<< cur_grace_logs + 1
<< " out of "
<< max_grace_logs;
}
shouldBlock |= m_waapDecision.getShouldBlockFromHighestPriorityDecision();
// Do not send Detect log if trigger disallows it
if (!send_extended_log && shouldBlock == false && !triggerLog->tpDetect &&
@@ -1787,13 +1772,6 @@ Waf2Transaction::sendLog()
if (decision_type == DecisionType::NO_WAAP_DECISION) {
if (send_extended_log || autonomousSecurityDecision->getOverridesLog()) {
sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes);
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: Sending log in grace period. Log "
<< ++cur_grace_logs
<< "out of "
<< max_grace_logs;
}
}
dbgTrace(D_WAAP) << "Waf2Transaction::sendLog: decisions marked for block only";
return;
@@ -1833,13 +1811,6 @@ Waf2Transaction::sendLog()
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
waap_log << LogField("waapIncidentDetails", incidentDetails);
waap_log << LogField("eventConfidence", "High");
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: Sending log in grace period. Log "
<< ++cur_grace_logs
<< "out of "
<< max_grace_logs;
}
break;
}
case OPEN_REDIRECT_DECISION:
@@ -1872,15 +1843,7 @@ Waf2Transaction::sendLog()
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
waap_log << LogField("waapIncidentDetails", incidentDetails);
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: Sending log in grace period. Log "
<< ++cur_grace_logs
<< "out of "
<< max_grace_logs;
}
break;
}
case CSRF_DECISION: {
@@ -1896,13 +1859,6 @@ Waf2Transaction::sendLog()
LogGen& waap_log = logGenWrapper.getLogGen();
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery");
waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered.");
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: Sending log in grace period. Log "
<< ++cur_grace_logs
<< "out of "
<< max_grace_logs;
}
break;
}
case AUTONOMOUS_SECURITY_DECISION: {
@@ -1911,13 +1867,6 @@ Waf2Transaction::sendLog()
autonomousSecurityDecision->getThreatLevel() != ThreatLevel::NO_THREAT ||
autonomousSecurityDecision->getOverridesLog()) {
sendAutonomousSecurityLog(triggerLog, shouldBlock, logOverride, attackTypes);
if (grace_period) {
dbgTrace(D_WAAP)
<< "Waf2Transaction::sendLog: Sending log in grace period. Log "
<< ++cur_grace_logs
<< "out of "
<< max_grace_logs;
}
}
break;
}
@@ -2204,7 +2153,7 @@ bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgInfo(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
return false;
}
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -346,10 +346,6 @@ private:
// Cached pointer to const triggerLog (hence mutable)
mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
Waf2TransactionFlags m_waf2TransactionFlags;
// Grace period for logging
int max_grace_logs;
bool is_hybrid_mode = false;
};
#endif // __WAF2_TRANSACTION_H__99e4201a

1
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -624,3 +624,4 @@ ReportIS::Severity Waf2Transaction::computeEventSeverityFromDecision() const
return ReportIS::Severity::INFO;
}

28
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -42,7 +42,6 @@ USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_EVASIONS);
USE_DEBUG_FLAG(D_WAAP_BASE64);
USE_DEBUG_FLAG(D_WAAP_JSON);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
#define MIN_HEX_LENGTH 6
#define charToDigit(c) (c - '0')
@@ -2056,6 +2055,33 @@ string extractForwardedIp(const string &x_forwarded_hdr_val)
return forward_ip;
}
bool isUuid(const string& str) {
if (str.length() != 36) {
return false;
}
static bool err;
static const SingleRegex uuid_detector_re(
"[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}",
err,
"uuid_detector"
);
// Check if the string matches the UUID format
return uuid_detector_re.hasMatch(str);
/*
boost::cmatch what;
try {
static const boost::regex uuidRegex("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}");
// Check if the string matches the UUID format
return boost::regex_match(str.c_str(), what, uuidRegex);
} catch (std::runtime_error &e) {
dbgError(D_WAAP) << e.what();
}
return false;
*/
}
bool isIpAddress(const string &ip_address)
{
struct in_addr source_inaddr;

1
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -1185,6 +1185,7 @@ namespace Util {
std::string stripOptionalPort(const std::string::const_iterator &first, const std::string::const_iterator &last);
std::string extractKeyValueFromCookie(const std::string &cookie, const std::string &key);
bool isIpAddress(const std::string &ip_address);
bool isUuid(const std::string& str);
bool vectorStringContain(const std::vector<std::string>& vec, const std::string& str);
bool isIpTrusted(const std::string &ip, const std::vector<std::string> &trusted_ips);

0
components/security_apps/waap/waap_clib/events_for_oa_schema.h Normal file
View File

20
components/security_apps/waap/waap_clib/oasu_key_types.h Normal file
View File

@@ -0,0 +1,20 @@
#ifndef __OA_SCHEMA_UPDATER_KEYS_H__
#define __OA_SCHEMA_UPDATER_KEYS_H__
enum SchemaKeyType
{
NullKeyName,
BooleanKeyName,
NumberKeyName,
StringKeyName,
ObjectMapKeyName,
ObjectInArrayKeyName,
EmptyObjectKeyName,
EndObjectKeyName,
StartObjectKeyName,
StartArrayKeyName,
EndArrayKeyName,
OtherKey
};
#endif // __OA_SCHEMA_UPDATER_KEYS_H__