github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-11-18 02:00:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2025-02-10 16:15:01 +00:00
parent 8d03b49176
commit 19bb4518af
68 changed files with 1101 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
components/security_apps/waap/include/i_serialize.h
View File

@@ -137,9 +137,13 @@ public:
void setRemoteSyncEnabled(bool enabled);
protected:
void mergeProcessedFromRemote();
std::string getWindowId();
void waitSync();
std::string getPostDataUrl();
std::string getUri();
size_t getIntervalsCount();
void incrementIntervalsCount();
bool isBase();
template<typename T>
bool sendObject(T &obj, HTTPMethod method, std::string uri)
@@ -252,14 +256,13 @@ protected:
const std::string m_remotePath; // Created from tenentId + / + assetId + / + class
std::chrono::seconds m_interval;
std::string m_owner;
const std::string m_assetId;
private:
bool localSyncAndProcess();
void updateStateFromRemoteService();
RemoteFilesList getProcessedFilesList();
RemoteFilesList getRemoteProcessedFilesList();
std::string getWindowId();
bool isBase();
std::string getLearningHost();
std::string getSharedStorageHost();
@@ -270,7 +273,6 @@ private:
size_t m_windowsCount;
size_t m_intervalsCounter;
bool m_remoteSyncEnabled;
const std::string m_assetId;
const bool m_isAssetIdUuid;
std::string m_type;
std::string m_lastProcessedModified;

6
components/security_apps/waap/include/i_waapConfig.h
View File

@@ -19,12 +19,14 @@
#include "../waap_clib/WaapParameters.h"
#include "../waap_clib/WaapOpenRedirectPolicy.h"
#include "../waap_clib/WaapErrorDisclosurePolicy.h"
#include "../waap_clib/DecisionType.h"
#include "../waap_clib/CsrfPolicy.h"
#include "../waap_clib/UserLimitsPolicy.h"
#include "../waap_clib/RateLimiting.h"
#include "../waap_clib/SecurityHeadersPolicy.h"
#include <memory>
enum class BlockingLevel {
NO_BLOCKING = 0,
LOW_BLOCKING_LEVEL,
@@ -44,8 +46,8 @@ public:
virtual const std::string& get_AssetId() const = 0;
virtual const std::string& get_AssetName() const = 0;
virtual const BlockingLevel& get_BlockingLevel() const = 0;
virtual const std::string& get_PracticeId() const = 0;
virtual const std::string& get_PracticeName() const = 0;
virtual const std::string& get_PracticeIdByPactice(DecisionType practiceType) const = 0;
virtual const std::string& get_PracticeNameByPactice(DecisionType practiceType) const = 0;
virtual const std::string& get_PracticeSubType() const = 0;
virtual const std::string& get_RuleId() const = 0;
virtual const std::string& get_RuleName() const = 0;

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -91,6 +91,7 @@ add_library(waap_clib
ParserScreenedJson.cc
ParserBinaryFile.cc
RegexComparator.cc
RequestsMonitor.cc
)
add_definitions("-Wno-unused-function")

20
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -407,6 +407,7 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_remotePath(replaceAllCopy(remotePath, "//", "/")),
m_interval(0),
m_owner(owner),
m_assetId(replaceAllCopy(assetId, "/", "")),
m_pMainLoop(nullptr),
m_waitForSync(waitForSync),
m_workerRoutineId(0),
@@ -414,7 +415,6 @@ SerializeToLocalAndRemoteSyncBase::SerializeToLocalAndRemoteSyncBase(
m_windowsCount(0),
m_intervalsCounter(0),
m_remoteSyncEnabled(true),
m_assetId(replaceAllCopy(assetId, "/", "")),
m_isAssetIdUuid(Waap::Util::isUuid(assetId)),
m_shared_storage_host(genError("not set")),
m_learning_host(genError("not set"))
@@ -469,6 +469,15 @@ bool SerializeToLocalAndRemoteSyncBase::isBase()
return m_remotePath == "";
}
void SerializeToLocalAndRemoteSyncBase::waitSync()
{
if (m_pMainLoop == nullptr)
{
return;
}
m_pMainLoop->yield(m_waitForSync);
}
string SerializeToLocalAndRemoteSyncBase::getUri()
{
static const string hybridModeUri = "/api";
@@ -484,6 +493,11 @@ size_t SerializeToLocalAndRemoteSyncBase::getIntervalsCount()
return m_intervalsCounter;
}
void SerializeToLocalAndRemoteSyncBase::incrementIntervalsCount()
{
m_intervalsCounter++;
}
SerializeToLocalAndRemoteSyncBase::~SerializeToLocalAndRemoteSyncBase()
{
@@ -659,7 +673,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
{
dbgInfo(D_WAAP_CONFIDENCE_CALCULATOR) << "Running the sync worker for assetId='" << m_assetId << "', owner='" <<
m_owner << "'" << " last modified state: " << m_lastProcessedModified;
m_intervalsCounter++;
incrementIntervalsCount();
OrchestrationMode mode = Singleton::exists<I_AgentDetails>() ?
Singleton::Consume<I_AgentDetails>::by<WaapComponent>()->getOrchestrationMode() : OrchestrationMode::ONLINE;
@@ -678,7 +692,7 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
}
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "Waiting for all agents to post their data";
m_pMainLoop->yield(m_waitForSync);
waitSync();
// check if learning service is operational
if (m_lastProcessedModified == "")
{

20
components/security_apps/waap/waap_clib/Telemetry.cc
View File

@@ -33,6 +33,7 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const
OrchestrationMode mode = Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getOrchestrationMode();
GenericMetric::sendLog(metric_client_rest);
dbgTrace(D_WAAP) << "Waap telemetry log sent: " << metric_client_rest.genJson().unpack();
if (mode == OrchestrationMode::ONLINE) {
return;
@@ -79,7 +80,16 @@ void
WaapTelemetrics::updateMetrics(const string &asset_id, const DecisionTelemetryData &data)
{
initMetrics();
requests.report(1);
auto is_keep_alive_ctx = Singleton::Consume<I_Environment>::by<GenericMetric>()->get<bool>(
"keep_alive_request_ctx"
);
if (!is_keep_alive_ctx.ok() || !*is_keep_alive_ctx) {
requests.report(1);
} else {
dbgTrace(D_WAAP) << "Not increasing the number of requests due to keep alive";
}
if (sources_seen.find(data.source) == sources_seen.end()) {
if (sources.getCounter() == 0) sources_seen.clear();
sources_seen.insert(data.source);
@@ -274,7 +284,9 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::INTERNAL
ReportIS::Audience::INTERNAL,
false,
asset_id
);
metrics[asset_id]->registerListener();
}
@@ -286,7 +298,9 @@ WaapMetricWrapper::upon(const WaapTelemetryEvent &event)
ReportIS::IssuingEngine::AGENT_CORE,
chrono::minutes(LOGGING_INTERVAL_IN_MINUTES),
true,
ReportIS::Audience::INTERNAL
ReportIS::Audience::INTERNAL,
false,
asset_id
);
attack_types[asset_id]->registerListener();
}

5
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -135,6 +135,7 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
m_Signatures(signatures),
m_waapDataFileName(waapDataFileName),
m_assetId(assetId),
m_requestsMonitor(nullptr),
scoreBuilder(this),
m_rateLimitingState(nullptr),
m_errorLimitingState(nullptr),
@@ -152,10 +153,14 @@ WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
I_AgentDetails* agentDetails = Singleton::Consume<I_AgentDetails>::by<WaapComponent>();
std::string path = agentDetails->getTenantId() + "/" + assetId;
m_filtersMngr = std::make_shared<IndicatorsFiltersManager>(path, assetId, this);
m_requestsMonitor = std::make_shared<SourcesRequestMonitor>
(getWaapDataDir() + "/monitor.data", path, assetId, "State");
}
else
{
m_filtersMngr = std::make_shared<IndicatorsFiltersManager>("", "", this);
m_requestsMonitor = std::make_shared<SourcesRequestMonitor>
(getWaapDataDir() + "/monitor.data", "", assetId, "State");
}
// Load keyword scores - copy from ScoreBuilder
updateScores();

4
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -33,6 +33,7 @@
#include "KeywordTypeValidator.h"
#include "ScanResult.h"
#include "WaapSampleValue.h"
#include "RequestsMonitor.h"
enum space_stage {SPACE_SYNBOL, BR_SYMBOL, BN_SYMBOL, BRN_SEQUENCE, BNR_SEQUENCE, NO_SPACES};
@@ -67,6 +68,8 @@ public:
const std::string m_assetId;
std::shared_ptr<SourcesRequestMonitor> m_requestsMonitor;
ScoreBuilder scoreBuilder;
std::shared_ptr<Waap::RateLimiting::State> m_rateLimitingState;
std::shared_ptr<Waap::RateLimiting::State> m_errorLimitingState;
@@ -90,6 +93,7 @@ public:
void logIndicatorsInFilters(const std::string &param, Waap::Keywords::KeywordsSet& keywords,
IWaf2Transaction* pTransaction);
void logParamHit(Waf2ScanResult& res, IWaf2Transaction* pTransaction);
void logSourceHit(const std::string& source);
void filterKeywords(const std::string &param, Waap::Keywords::KeywordsSet& keywords,
std::vector<std::string>& filteredKeywords);
void clearFilterVerbose();

31
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -329,14 +329,37 @@ const std::string& WaapConfigBase::get_AssetName() const
return m_assetName;
}
const std::string& WaapConfigBase::get_PracticeId() const
const std::string& WaapConfigBase::get_PracticeIdByPactice(DecisionType practiceType) const
{
return m_practiceId;
switch (practiceType)
{
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_practiceId;
default:
dbgError(D_WAAP)
<< "Can't find practice type for practice ID by practice: "
<< practiceType
<< ", return web app practice ID";
return m_practiceId;
}
}
const std::string& WaapConfigBase::get_PracticeName() const
const std::string& WaapConfigBase::get_PracticeNameByPactice(DecisionType practiceType) const
{
return m_practiceName;
switch (practiceType)
{
case DecisionType::AUTONOMOUS_SECURITY_DECISION:
return m_practiceName;
default:
dbgError(D_WAAP)
<< "Can't find practice type for practice name by practice: "
<< practiceType
<< ", return web app practice name";
return m_practiceName;
}
}
const std::string& WaapConfigBase::get_RuleId() const

4
components/security_apps/waap/waap_clib/WaapConfigBase.h
View File

@@ -39,8 +39,8 @@ public:
virtual const std::string& get_AssetId() const;
virtual const std::string& get_AssetName() const;
virtual const BlockingLevel& get_BlockingLevel() const;
virtual const std::string& get_PracticeId() const;
virtual const std::string& get_PracticeName() const;
virtual const std::string& get_PracticeIdByPactice(DecisionType practiceType) const;
virtual const std::string& get_PracticeNameByPactice(DecisionType practiceType) const;
virtual const std::string& get_RuleId() const;
virtual const std::string& get_RuleName() const;
virtual const bool& get_WebAttackMitigation() const;

2
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -89,7 +89,7 @@ bool WaapOverrideFunctor::operator()(
}
else if (tagLower == "url") {
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getUriStr)) return true;
if (W2T_REGX_MATCH(getUri)) return true;
}
return false;
}

15
components/security_apps/waap/waap_clib/WaapResponseInjectReasons.cc
View File

@@ -23,6 +23,7 @@ ResponseInjectReasons::ResponseInjectReasons()
:
csrf(false),
antibot(false),
captcha(false),
securityHeaders(false)
{
}
@@ -53,6 +54,13 @@ ResponseInjectReasons::setAntibot(bool flag)
antibot = flag;
}
void
ResponseInjectReasons::setCaptcha(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInjectReasons(Captcha) " << captcha << " to " << flag;
captcha = flag;
}
void
ResponseInjectReasons::setCsrf(bool flag)
{
@@ -74,6 +82,13 @@ ResponseInjectReasons::shouldInjectAntibot() const
return antibot;
}
bool
ResponseInjectReasons::shouldInjectCaptcha() const
{
dbgTrace(D_WAAP) << "shouldInjectCaptcha():: " << captcha;
return captcha;
}
bool
ResponseInjectReasons::shouldInjectCsrf() const
{

3
components/security_apps/waap/waap_clib/WaapResponseInjectReasons.h
View File

@@ -21,14 +21,17 @@ public:
void clear();
bool shouldInject() const;
void setAntibot(bool flag);
void setCaptcha(bool flag);
void setCsrf(bool flag);
void setSecurityHeaders(bool flag);
bool shouldInjectAntibot() const;
bool shouldInjectCaptcha() const;
bool shouldInjectCsrf() const;
bool shouldInjectSecurityHeaders() const;
private:
bool csrf;
bool antibot;
bool captcha;
bool securityHeaders;
};

48
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -1098,6 +1098,7 @@ void Waf2Transaction::end_request_hdrs() {
// but the State itself is not needed now
Waap::Override::State overrideState = getOverrideState(m_siteConfig);
}
m_pWaapAssetState->m_requestsMonitor->logSourceHit(m_source_identifier);
IdentifiersEvent ids(m_source_identifier, m_pWaapAssetState->m_assetId);
ids.notify();
// Read relevant headers and extract meta information such as host name
@@ -1421,6 +1422,15 @@ Waf2Transaction::completeInjectionResponseBody(std::string& strInjection)
m_responseInjectReasons.setAntibot(false);
}
if(m_responseInjectReasons.shouldInjectCaptcha()) {
dbgTrace(D_WAAP_BOT_PROTECTION) <<
"Waf2Transaction::completeInjectionResponseBody(): Injecting data (captcha)";
//todo add captcha script
strInjection += "<script src=\"cp-cp.js\"></script>";
// No need to inject more than once
m_responseInjectReasons.setCaptcha(false);
}
if (m_responseInjectReasons.shouldInjectCsrf()) {
dbgTrace(D_WAAP) << "Waf2Transaction::completeInjectionResponseBody(): Injecting data (csrf)";
strInjection += "<script src=\"cp-csrf.js\"></script>";
@@ -1646,7 +1656,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
bool shouldBlock,
const std::string& logOverride,
const std::string& incidentType) const
const std::string& incidentType,
const std::string& practiceID,
const std::string& practiceName) const
{
auto env = Singleton::Consume<I_Environment>::by<WaapComponent>();
auto active_id = env->get<std::string>("ActiveTenantId");
@@ -1737,8 +1749,8 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("practiceType", "Threat Prevention");
waapLog << LogField("practiceSubType", m_siteConfig->get_PracticeSubType());
waapLog << LogField("ruleName", m_siteConfig->get_RuleName());
waapLog << LogField("practiceId", m_siteConfig->get_PracticeId());
waapLog << LogField("practiceName", m_siteConfig->get_PracticeName());
waapLog << LogField("practiceId", practiceID);
waapLog << LogField("practiceName", practiceName);
waapLog << LogField("waapIncidentType", incidentType);
// Registering this value would append the list of matched override IDs to the unified log
@@ -1805,8 +1817,8 @@ Waf2Transaction::sendLog()
telemetryData.source = getSourceIdentifier();
telemetryData.assetName = m_siteConfig->get_AssetName();
telemetryData.practiceId = m_siteConfig->get_PracticeId();
telemetryData.practiceName = m_siteConfig->get_PracticeName();
telemetryData.practiceId = m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION);
telemetryData.practiceName = m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION);
if (m_scanResult) {
telemetryData.attackTypes = m_scanResult->attack_types;
}
@@ -1947,7 +1959,11 @@ Waf2Transaction::sendLog()
shouldBlock);
LogGen& waap_log = logGenWrapper.getLogGen();
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, incidentType,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", incidentDetails);
waap_log << LogField("eventConfidence", "High");
break;
@@ -1980,7 +1996,11 @@ Waf2Transaction::sendLog()
waap_log << LogField("waapFoundIndicators", getKeywordMatchesStr(), LogFieldOption::XORANDB64);
}
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, incidentType,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", incidentDetails);
break;
@@ -1996,7 +2016,11 @@ Waf2Transaction::sendLog()
shouldBlock);
LogGen& waap_log = logGenWrapper.getLogGen();
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery");
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, "Cross Site Request Forgery",
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
waap_log << LogField("waapIncidentDetails", "CSRF Attack discovered.");
break;
}
@@ -2177,19 +2201,13 @@ Waf2Transaction::decideAutonomousSecurity(
" effective overrides count: " << m_effectiveOverrideIds.size() <<
" learned overrides count: " << m_exceptionLearned.size();
bool log_all = false;
const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = sitePolicy.get_TriggerPolicy();
if (triggerPolicy) {
const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
if (triggerLog && triggerLog->webRequests) log_all = true;
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) {
decision->setLog(false);
} else {
decision->setLog(true);
}
return decision->shouldBlock();
}

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -247,7 +247,9 @@ private:
const std::shared_ptr<Waap::Trigger::Log> &triggerLog,
bool shouldBlock,
const std::string& logOverride,
const std::string& incidentType) const;
const std::string& incidentType,
const std::string& practiceID,
const std::string& practiceName) const;
std::string getUserReputationStr(double relativeReputation) const;
bool isTrustedSource() const;

6
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -381,7 +381,11 @@ void Waf2Transaction::sendAutonomousSecurityLog(
waap_log << LogField("eventConfidence", confidence);
}
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, attackTypes);
appendCommonLogFields(
waap_log, triggerLog, shouldBlock, logOverride, attackTypes,
m_siteConfig->get_PracticeIdByPactice(AUTONOMOUS_SECURITY_DECISION),
m_siteConfig->get_PracticeNameByPactice(AUTONOMOUS_SECURITY_DECISION)
);
std::string sampleString = getSample();
if (sampleString.length() > MAX_LOG_FIELD_SIZE) {

1
components/security_apps/waap/waap_component_impl.h
View File

@@ -19,6 +19,7 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"