github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jul 4th update

Browse Source
This commit is contained in:
Ned Wright
2024-07-04 14:10:34 +00:00
parent 01770475ec
commit 962bd31d46
17 changed files with 152 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
components/security_apps/waap/waap_clib/ScanResult.cc
View File

@@ -23,6 +23,7 @@ unescaped_line(),
param_name(),
location(),
score(0.0f),
scoreNoFilter(0.0f),
scoreArray(),
keywordCombinations(),
attack_types(),
@@ -40,6 +41,7 @@ void Waf2ScanResult::clear()
param_name.clear();
location.clear();
score = 0;
scoreNoFilter = 0;
scoreArray.clear();
keywordCombinations.clear();
attack_types.clear();

1
components/security_apps/waap/waap_clib/ScanResult.h
View File

@@ -29,6 +29,7 @@ struct Waf2ScanResult {
std::string param_name;
std::string location;
double score;
double scoreNoFilter;
std::vector<double> scoreArray;
std::vector<std::string> keywordCombinations;
std::set<std::string> attack_types;

2
components/security_apps/waap/waap_clib/WaapConversions.cc
View File

@@ -39,7 +39,7 @@ namespace Conversions {
return HIGH_THREAT;
}
bool shouldDoWafBlocking(const IWaapConfig* pWaapConfig, ThreatLevel threatLevel)
bool shouldDoWafBlocking(const IWaapConfig* const pWaapConfig, ThreatLevel threatLevel)
{
if (pWaapConfig == NULL)
{

2
components/security_apps/waap/waap_clib/WaapConversions.h
View File

@@ -20,7 +20,7 @@
namespace Waap {
namespace Conversions {
ThreatLevel convertFinalScoreToThreatLevel(double finalScore);
bool shouldDoWafBlocking(const IWaapConfig* pSitePolicy, ThreatLevel threatLevel);
bool shouldDoWafBlocking(const IWaapConfig* const pSitePolicy, ThreatLevel threatLevel);
}
}

39
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -25,7 +25,7 @@ USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
// id generated by xml parser for an entity attribute
const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f";
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName)
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning)
{
std::string source = m_transaction->getSourceIdentifier();
@@ -33,21 +33,24 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
Waap::Keywords::KeywordsSet keywordsSet;
Waap::Keywords::computeKeywordsSet(keywordsSet, res.keyword_matches, res.found_patterns);
std::string param_name = IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction);
dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name;
m_transaction->getAssetState()->logIndicatorsInFilters(param_name, keywordsSet, m_transaction);
if (applyLearning) {
std::string param_name = IndicatorsFiltersManager::generateKey(res.location, res.param_name, m_transaction);
dbgTrace(D_WAAP_SCANNER) << "filter processing for parameter: " << param_name <<
", indicators count: " << keywordsSet.size();
m_transaction->getAssetState()->logIndicatorsInFilters(param_name, keywordsSet, m_transaction);
m_transaction->getAssetState()->filterKeywords(param_name, keywordsSet, res.filtered_keywords);
if (m_transaction->getSiteConfig() != nullptr)
{
auto waapParams = m_transaction->getSiteConfig()->get_WaapParametersPolicy();
if (waapParams != nullptr && waapParams->getParamVal("filtersVerbose", "false") == "true") {
m_transaction->getAssetState()->filterVerbose(param_name, res.filtered_keywords);
m_transaction->getAssetState()->filterKeywords(param_name, keywordsSet, res.filtered_keywords);
if (m_transaction->getSiteConfig() != nullptr)
{
auto waapParams = m_transaction->getSiteConfig()->get_WaapParametersPolicy();
if (waapParams != nullptr && waapParams->getParamVal("filtersVerbose", "false") == "true") {
m_transaction->getAssetState()->filterVerbose(param_name, res.filtered_keywords);
}
}
m_transaction->getAssetState()->filterKeywordsByParameters(res.param_name, keywordsSet);
dbgTrace(D_WAAP_SCANNER) << "post filtering indicators count: " << keywordsSet.size();
}
m_transaction->getAssetState()->filterKeywordsByParameters(res.param_name, keywordsSet);
// The keywords are only removed in production, they are still used while building scores
if (!m_transaction->get_ignoreScore()) {
m_transaction->getAssetState()->removeKeywords(keywordsSet);
@@ -148,9 +151,16 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
// Select scores pool by location
std::string poolName = Waap::Scores::getScorePoolNameByLocation(location);
Waf2ScanResult nonFilterRes = res;
res.scoreNoFilter = getScoreData(nonFilterRes, poolName, false);
double score = getScoreData(res, poolName);
dbgTrace(D_WAAP_SCANNER) << "score: " << score;
// call shouldIgnoreOverride post score calculation and filtering to evaluate ignore override effectivness
res.score = score;
m_transaction->shouldIgnoreOverride(res);
dbgTrace(D_WAAP_SCANNER) << "score: " << score << " should ignore: " << ignoreOverride;
// Add record about scores to the notes[] log (also reported in logs)
if (score > 1.0f) {
DetectionEvent(location, res.keyword_matches).notify();
@@ -166,6 +176,7 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
if (isKeyCspReport(key, res, dp) || ignoreOverride) {
dbgTrace(D_WAAP_SCANNER) << "Ignoring parameter key/value " << res.param_name <<
" due to ignore action in override";
res.score = 0;
m_bIgnoreOverride = true;
return false;
}

2
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -43,7 +43,7 @@ namespace Waap {
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);

85
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -329,6 +329,7 @@ Waf2Transaction::Waf2Transaction() :
is_schema_validation(false),
m_waf2TransactionFlags()
{
m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = 0;
I_TimeGet *timeGet = Singleton::Consume<I_TimeGet>::by<Waf2Transaction>();
m_entry_time = chrono::duration_cast<chrono::milliseconds>(timeGet->getMonotonicTime());
}
@@ -1729,6 +1730,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin());
waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds));
}
if (!m_exceptionLearned.empty()) {
std::vector<std::string> vLearningAffected(m_exceptionLearned.size());
std::copy(m_exceptionLearned.begin(), m_exceptionLearned.end(), vLearningAffected.begin());
waapLog.addToOrigin(LogField("redundantExceptionIdList", vLearningAffected));
}
}
}
@@ -1809,12 +1815,6 @@ Waf2Transaction::sendLog()
return;
}
dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size();
bool shouldBlock = false;
if (m_overrideState.bForceBlock) {
// If override forces "reject" decision, mention it in the "override" log field.
@@ -2091,7 +2091,30 @@ Waf2Transaction::decideAutonomousSecurity(
transactionResult.threatLevel = threat;
}
dbgTrace(D_WAAP_OVERRIDE) << "override ids count: " << m_matchedOverrideIds.size();
// Apply overrides
for (auto it = m_overridePostFilterMaxScore.begin(); it != m_overridePostFilterMaxScore.end(); it++) {
const string id = it->first;
if (m_overrideState.forceBlockIds.find(id) != m_overrideState.forceBlockIds.end()) {
// blocked effectivness is calculates later from the force block exception ids list
continue;
}
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(it->second);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threat);
dbgTrace(D_WAAP_OVERRIDE) << "checking effectivness of override: " << id << ", should have blocked: " << shouldBlock
<< ", scores: " << m_overridePostFilterMaxScore[id] << ", " << m_overrideOriginalMaxScore[id];
if (shouldBlock) {
m_effectiveOverrideIds.insert(id);
} else {
ThreatLevel threatNoFilter = Waap::Conversions::convertFinalScoreToThreatLevel(
m_overrideOriginalMaxScore[id]
);
if (Waap::Conversions::shouldDoWafBlocking(m_siteConfig, threatNoFilter)) {
m_exceptionLearned.insert(id);
}
}
}
if (m_overrideState.bForceBlock) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces REJECT ...";
@@ -2105,25 +2128,25 @@ Waf2Transaction::decideAutonomousSecurity(
}
}
else if (m_overrideState.bForceException) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
dbgTrace(D_WAAP) << "de cideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces ALLOW ...";
if (m_scanResult) {
// on accept exception the decision is not set and needs to be calculated to determine effectivness
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat);
if (shouldBlock) {
m_effectiveOverrideIds.insert(
m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end()
);
}
}
decision->setBlock(false);
if (!m_overrideState.bIgnoreLog)
{
decision->setOverridesLog(true);
}
} else if (!m_matchedOverrideIds.empty()) {
if (!m_overrideState.bIgnoreLog)
{
decision->setOverridesLog(true);
}
}
dbgTrace(D_WAAP_OVERRIDE) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size() <<
" learned overrides count: " << m_exceptionLearned.size();
bool log_all = false;
@@ -2262,7 +2285,7 @@ bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) {
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions error: " << exceptions.getErr();
return false;
}
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
@@ -2305,6 +2328,24 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (const auto &behavior : behaviors) {
if (!res.filtered_keywords.empty() || res.score > 0) {
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators";
std::string overrideId = behavior.getId();
if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
m_overridePostFilterMaxScore[overrideId] = res.score;
} else {
if (res.scoreNoFilter > m_overrideOriginalMaxScore[overrideId]) {
m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter;
}
if (res.score > m_overridePostFilterMaxScore[overrideId]) {
m_overridePostFilterMaxScore[overrideId] = res.score;
}
}
if (res.scoreNoFilter > m_overrideOriginalMaxScore[OVERRIDE_ACCEPT]) {
m_overrideOriginalMaxScore[OVERRIDE_ACCEPT] = res.scoreNoFilter;
}
}
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
@@ -2312,12 +2353,6 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);
}
if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId)
{
if (!overrideId.empty()) {
m_effectiveOverrideIds.insert(overrideId);
}
}
return true;
}
}

3
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -293,6 +293,9 @@ private:
// Matched override IDs
std::set<std::string> m_matchedOverrideIds;
std::set<std::string> m_effectiveOverrideIds;
std::set<std::string> m_exceptionLearned;
std::map<std::string, double> m_overrideOriginalMaxScore;
std::map<std::string, double> m_overridePostFilterMaxScore;
//csrf state
Waap::CSRF::State m_csrfState;

12
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -459,9 +459,15 @@ Waf2Transaction::getUserLimitVerdict()
}
else if (mode == AttackMitigationMode::PREVENT) {
decision->setLog(true);
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
if (!m_overrideState.bForceException) {
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "BLOCK" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
} else {
decision->setBlock(true);
dbgInfo(D_WAAP_ULIMITS) << msg << "Override Accept" << reason;
verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
}
}
return verdict;