github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

sync code

Browse Source
This commit is contained in:
Ned Wright
2024-10-14 14:51:28 +00:00
parent b58f7781e6
commit c2ea2cda6d
89 changed files with 2545 additions and 447 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
components/security_apps/waap/include/i_transaction.h
View File

@@ -70,6 +70,7 @@ public:
virtual const std::string getParam() const = 0;
virtual const std::vector<std::string> getKeywordMatches() const = 0;
virtual const std::vector<std::string> getKeywordsCombinations() const = 0;
virtual const std::vector<std::string> getKeywordsAfterFilter() const = 0;
virtual const std::string getContentTypeStr() const = 0;
virtual Waap::Util::ContentType getContentType() const = 0;
virtual const std::string getKeywordMatchesStr() const = 0;
@@ -84,6 +85,7 @@ public:
virtual const std::string getUriStr() const = 0;
virtual const std::string& getSourceIdentifier() const = 0;
virtual double getScore() const = 0;
virtual double getOtherModelScore() const = 0;
virtual const std::vector<double> getScoreArray() const = 0;
virtual Waap::CSRF::State& getCsrfState() = 0;
virtual ngx_http_cp_verdict_e getUserLimitVerdict() = 0;

96
components/security_apps/waap/waap_clib/CidrMatch.cc
View File

@@ -25,6 +25,51 @@
#include "log_generator.h"
#include <stdexcept>
static in_addr applyMaskV4(const in_addr& addr, uint8_t prefixLength) {
in_addr maskedAddr;
if (prefixLength == 0) {
maskedAddr.s_addr = 0;
} else {
uint32_t mask = htonl(~((1 << (32 - prefixLength)) - 1)); // Create mask
maskedAddr.s_addr = addr.s_addr & mask; // Apply mask
}
return maskedAddr;
}
// Function to apply a network mask to an IPv6 address
static in6_addr applyMaskV6(const in6_addr& addr, uint8_t prefixLength) {
in6_addr maskedAddr = addr;
int fullBytes = prefixLength / 8;
int remainingBits = prefixLength % 8;
// Mask full bytes
for (int i = fullBytes; i < 16; ++i) {
maskedAddr.s6_addr[i] = 0;
}
// Mask remaining bits
if (remainingBits > 0) {
uint8_t mask = ~((1 << (8 - remainingBits)) - 1);
maskedAddr.s6_addr[fullBytes] &= mask;
}
return maskedAddr;
}
// Helper function to convert an IPv4 address to string
static std::string ipv4ToString(const in_addr& ipv4) {
char str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ipv4, str, INET_ADDRSTRLEN);
return std::string(str);
}
// Helper function to convert an IPv6 address to string
static std::string ipv6ToString(const in6_addr& ipv6) {
char str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, &ipv6, str, INET6_ADDRSTRLEN);
return std::string(str);
}
USE_DEBUG_FLAG(D_WAAP);
namespace Waap {
namespace Util {
@@ -38,6 +83,15 @@ bool CIDRData::operator==(const CIDRData &other) const {
isIPV6 == other.isIPV6;
}
bool CIDRData::operator<(const CIDRData &other) const {
if (isIPV6) {
if (!other.isIPV6) return false;
return memcmp(ipCIDRV6.s6_addr, other.ipCIDRV6.s6_addr, sizeof(ipCIDRV6.s6_addr)) < 0;
}
if (other.isIPV6) return true;
return ntohl(ipCIDRV4.s_addr) < ntohl(other.ipCIDRV4.s_addr);
}
bool cidr4_match(const in_addr &addr, const in_addr &net, uint8_t bits) {
if (bits == 0) {
// C99 6.5.7 (3): u32 << 32 is undefined behaviour
@@ -114,9 +168,11 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
memset(&cidr.ipCIDRV6, 0, sizeof(struct in6_addr));
if (inet_pton(AF_INET, strPrefix.c_str(), &cidr.ipCIDRV4) == 1 && bits <= 32) {
cidr.ipCIDRV4 = applyMaskV4(cidr.ipCIDRV4, bits);
cidr.isIPV6 = false;
}
else if (inet_pton(AF_INET6, strPrefix.c_str(), &cidr.ipCIDRV6) == 1 && bits <= 128) {
cidr.ipCIDRV6 = applyMaskV6(cidr.ipCIDRV6, bits);
cidr.isIPV6 = true;
}
else
@@ -128,6 +184,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
return true;
}
bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
CIDRData cidrData;
@@ -139,6 +196,7 @@ bool cidrMatch(const std::string& sourceip, const std::string& targetCidr) {
return cidrMatch(sourceip, cidrData);
}
bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
struct in_addr source_inaddr;
struct in6_addr source_inaddr6;
@@ -155,5 +213,43 @@ bool cidrMatch(const std::string & sourceip, const CIDRData & cidr){
dbgDebug(D_WAAP) << "Source IP address does not match any of the CIDR definitions.";
return false;
}
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second) {
if (first.isIPV6 != second.isIPV6) return false; // IPv4 and IPv6 cannot overlap
if (first.networkBits >= second.networkBits) return false;
if (!first.isIPV6) {
// IPv4 containment check
in_addr smallerNetwork = applyMaskV4(second.ipCIDRV4, first.networkBits);
return (first.ipCIDRV4.s_addr == smallerNetwork.s_addr);
}
// IPv6 containment check
in6_addr smallerNetwork = applyMaskV6(second.ipCIDRV6, first.networkBits);
for (int i = 0; i < 16; ++i) {
if (first.ipCIDRV6.s6_addr[i] != smallerNetwork.s6_addr[i]) {
return false;
}
}
return true;
}
std::string cidrsToString(const std::vector<CIDRData>& cidrs) {
std::stringstream ss;
bool is_first = true;
ss << "[";
for (const auto& cidr : cidrs) {
if (!is_first) ss << ", ";
if (cidr.isIPV6) {
ss << ipv6ToString(cidr.ipCIDRV6) << "/" << static_cast<int>(cidr.networkBits);
} else {
ss << ipv4ToString(cidr.ipCIDRV4) << "/" << static_cast<int>(cidr.networkBits);
}
is_first = false;
}
ss << "]";
return ss.str();
}
}
}

4
components/security_apps/waap/waap_clib/CidrMatch.h
View File

@@ -18,6 +18,7 @@
#include <sys/socket.h>
#include <arpa/inet.h>
#include <errno.h>
#include <vector>
namespace Waap {
namespace Util {
@@ -29,11 +30,14 @@ struct CIDRData {
uint8_t networkBits;
bool isIPV6;
bool operator==(const CIDRData &other) const;
bool operator<(const CIDRData &other) const;
};
bool isCIDR(const std::string& strCIDR, CIDRData& cidr);
bool cidrMatch(const std::string& sourceip, const CIDRData& cidr);
bool cidrMatch(const std::string &sourceip, const std::string &target);
bool doesFirstCidrContainSecond(const CIDRData &first, const CIDRData &second);
std::string cidrsToString(const std::vector<CIDRData>& cidrs);
}
}

7
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -1308,7 +1308,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
} else if (!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "!Waap::Util::testUrlBareUtf8Evasion(cur_val)";
@@ -1323,7 +1323,7 @@ DeepParser::createInternalParser(
*this,
parser_depth + 1,
'&',
valueStats.isUrlEncoded)
valueStats.isUrlEncoded && !Waap::Util::testUrlBadUtf8Evasion(cur_val))
);
offset = 0;
return offset;
@@ -1545,5 +1545,6 @@ DeepParser::isPDFDetected(const std::string &cur_val) const
static const std::string PDF_header("%PDF-");
if (cur_val.size() < 10)
return false;
return cur_val.substr(0, cur_val.size() > 64 ? 64 : cur_val.size()).find(PDF_header) != std::string::npos;
return cur_val.substr(0, cur_val.size() > MAX_PDF_HEADER_LOOKUP ? MAX_PDF_HEADER_LOOKUP : cur_val.size())
.find(PDF_header) != std::string::npos;
}

3
components/security_apps/waap/waap_clib/ParserBinaryFile.cc
View File

@@ -150,7 +150,8 @@ ParserBinaryFile::push(const char *buf, size_t len)
}
} else {
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
c = strstr(buf + len - tail.size(), tail.c_str());
size_t tail_lookup_offset = (len > MAX_TAIL_LOOKUP) ? len - MAX_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, tail.c_str());
dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
if (c) {
m_state = s_end;

2
components/security_apps/waap/waap_clib/ParserBinaryFile.h
View File

@@ -20,7 +20,7 @@
#define MIN_HEADER_LOOKUP 16
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_TAIL_LOOKUP 20
class ParserBinaryFile : public ParserBase {
public:

26
components/security_apps/waap/waap_clib/ParserPDF.cc
View File

@@ -38,9 +38,10 @@ size_t
ParserPDF::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PDF)
<< "buf="
<< buf
<< "len="
<< "buf='"
<< std::string(buf, std::min((size_t)200, len))
<< (len > 200 ? "..." : "")
<< "' len="
<< len;
const char *c;
@@ -65,13 +66,18 @@ ParserPDF::push(const char *buf, size_t len)
m_state = s_body;
CP_FALL_THROUGH;
case s_body:
c = strstr(buf + len - MAX_TAIL_LOOKUP, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF) << "ParserPDF::push(): c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
{
size_t tail_lookup_offset = (len > MAX_PDF_TAIL_LOOKUP) ? len - MAX_PDF_TAIL_LOOKUP : 0;
c = strstr(buf + tail_lookup_offset, PDF_TAIL);
dbgTrace(D_WAAP_PARSER_PDF)
<< "string to search: " << std::string(buf + tail_lookup_offset)
<< " c=" << c;
if (c) {
m_state = s_end;
CP_FALL_THROUGH;
} else {
break;
}
}
case s_end:
if (m_receiver.onKey("PDF", 3) != 0) {

4
components/security_apps/waap/waap_clib/ParserPDF.h
View File

@@ -17,8 +17,8 @@
#include "ParserBase.h"
#include <string.h>
#define MAX_HEADER_LOOKUP 64
#define MAX_TAIL_LOOKUP 5
#define MAX_PDF_HEADER_LOOKUP 64
#define MAX_PDF_TAIL_LOOKUP 20
class ParserPDF : public ParserBase {
public:

4
components/security_apps/waap/waap_clib/ScanResult.cc
View File

@@ -23,9 +23,11 @@ unescaped_line(),
param_name(),
location(),
score(0.0f),
other_model_score(0.0f),
scoreNoFilter(0.0f),
scoreArray(),
keywordCombinations(),
keywordsAfterFilter(),
attack_types(),
m_isAttackInParam(false)
{
@@ -41,9 +43,11 @@ void Waf2ScanResult::clear()
param_name.clear();
location.clear();
score = 0;
other_model_score = 0;
scoreNoFilter = 0;
scoreArray.clear();
keywordCombinations.clear();
keywordsAfterFilter.clear();
attack_types.clear();
}

3
components/security_apps/waap/waap_clib/ScanResult.h
View File

@@ -29,9 +29,12 @@ struct Waf2ScanResult {
std::string param_name;
std::string location;
double score;
double other_model_score;
double scoreNoFilter;
std::vector<double> scoreArray;
std::vector<double> coefArray;
std::vector<std::string> keywordCombinations;
std::vector<std::string> keywordsAfterFilter;
std::set<std::string> attack_types;
bool m_isAttackInParam;
void clear(); // clear Waf2ScanResult

75
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -95,6 +95,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
m_keywordsDataMap[it->first] = it->second;
}
}
m_stats.linModelIntercept = baseScores.m_stats.linModelIntercept;
m_stats.linModelNNZCoef = baseScores.m_stats.linModelNNZCoef;
m_stats.isLinModel = baseScores.m_stats.isLinModel;
}
@@ -118,7 +122,6 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
m_pWaapAssetState(pWaapAssetState)
{
restore();
// merge
mergeScores(baseScores);
}
@@ -352,16 +355,42 @@ void ScoreBuilder::snap()
const std::string &poolName = pool.first;
const KeywordsScorePool& keywordScorePool = pool.second;
m_snapshotKwScoreMap[poolName];
m_snapshotKwCoefMap[poolName];
if (keywordScorePool.m_keywordsDataMap.empty()) {
m_snapshotStatsMap[poolName] = KeywordsStats();
} else {
m_snapshotStatsMap[poolName] = keywordScorePool.m_stats;
}
for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
{
const std::string &kwName = kwData.first;
double kwScore = kwData.second.score;
m_snapshotKwScoreMap[poolName][kwName] = kwScore;
if (keywordScorePool.m_stats.isLinModel) {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.new_score;
} else {
m_snapshotKwScoreMap[poolName][kwName] = kwData.second.score;
}
m_snapshotKwCoefMap[poolName][kwName] = kwData.second.coef;
}
}
}
KeywordsStats
ScoreBuilder::getSnapshotStats(const std::string &poolName) const
{
std::map<std::string, KeywordsStats>::const_iterator poolIt = m_snapshotStatsMap.find(poolName);
if (poolIt == m_snapshotStatsMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting stats from base pool";
poolIt = m_snapshotStatsMap.find(KEYWORDS_SCORE_POOL_BASE);
if (poolIt == m_snapshotStatsMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning empty stats";
return KeywordsStats();
}
}
return poolIt->second;
}
double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double defaultScore,
const std::string &poolName) const
{
@@ -369,12 +398,11 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting score from base pool";
poolIt = m_snapshotKwScoreMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
if (poolIt == m_snapshotKwScoreMap.end()) {
dbgDebug(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default score " << defaultScore;
return defaultScore;
}
}
const KeywordScoreMap &kwScoreMap = poolIt->second;
@@ -391,6 +419,35 @@ double ScoreBuilder::getSnapshotKeywordScore(const std::string &keyword, double
return kwScoreFound->second;
}
double
ScoreBuilder::getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const
{
std::map<std::string, KeywordScoreMap>::const_iterator poolIt = m_snapshotKwCoefMap.find(poolName);
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "pool " << poolName << " does not exist. Getting coef from base pool";
poolIt = m_snapshotKwCoefMap.find(KEYWORDS_SCORE_POOL_BASE);
}
if (poolIt == m_snapshotKwCoefMap.end()) {
dbgWarning(D_WAAP_SCORE_BUILDER) <<
"base pool does not exist! This is probably a bug. Returning default coef " << defaultCoef;
return defaultCoef;
}
const KeywordScoreMap &kwCoefMap = poolIt->second;
KeywordScoreMap::const_iterator kwCoefFound = kwCoefMap.find(keyword);
if (kwCoefFound == kwCoefMap.end()) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << defaultCoef <<
" (default, keyword not found in pool '" << poolName << "')";
return defaultCoef;
}
dbgTrace(D_WAAP_SCORE_BUILDER) << "keywordCoef:'" << keyword << "': " << kwCoefFound->second << " (pool '" <<
poolName << "')";
return kwCoefFound->second;
}
keywords_set ScoreBuilder::getIpItemKeywordsSet(std::string ip)
{
return m_fpStore.ipItems[ip];

40
components/security_apps/waap/waap_clib/ScoreBuilder.h
View File

@@ -26,6 +26,8 @@
#include <cereal/types/string.hpp>
#include "WaapDefines.h"
USE_DEBUG_FLAG(D_WAAP_SCORE_BUILDER);
struct ScoreBuilderData {
std::string m_sourceIdentifier;
std::string m_userAgent;
@@ -52,11 +54,15 @@ enum KeywordType {
};
struct KeywordData {
KeywordData() : truePositiveCtr(0), falsePositiveCtr(0), score(0.0), type(KEYWORD_TYPE_UNKNOWN) {}
KeywordData() :
truePositiveCtr(0), falsePositiveCtr(0), score(0.0), coef(0.0), new_score(0.0), type(KEYWORD_TYPE_UNKNOWN)
{}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double score;
double coef;
double new_score;
KeywordType type;
template <class Archive>
@@ -65,20 +71,42 @@ struct KeywordData {
cereal::make_nvp("true_positives", truePositiveCtr),
cereal::make_nvp("score", score),
cereal::make_nvp("type", type));
try {
ar(cereal::make_nvp("coef", coef),
cereal::make_nvp("new_score", new_score));
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
coef = 0;
new_score = 0;
}
}
};
struct KeywordsStats {
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0) {}
KeywordsStats() : truePositiveCtr(0), falsePositiveCtr(0), linModelIntercept(0), linModelNNZCoef(0), isLinModel(0)
{}
template <class Archive>
void serialize(Archive& ar) {
ar(cereal::make_nvp("false_positives", falsePositiveCtr),
cereal::make_nvp("true_positives", truePositiveCtr));
cereal::make_nvp("true_positives", truePositiveCtr));
try {
ar(cereal::make_nvp("intercept", linModelIntercept),
cereal::make_nvp("log_nnz_coef", linModelNNZCoef));
isLinModel = true;
} catch (const cereal::Exception &e) {
ar.setNextName(nullptr);
linModelIntercept = 0;
linModelNNZCoef = 0;
isLinModel = false;
}
}
unsigned int truePositiveCtr;
unsigned int falsePositiveCtr;
double linModelIntercept;
double linModelNNZCoef;
bool isLinModel;
};
typedef std::unordered_set<std::string> keywords_set;
@@ -106,6 +134,7 @@ struct KeywordsScorePool {
KeywordsScorePool();
// LCOV_EXCL_START Reason: no test exist
template <typename _A>
KeywordsScorePool(_A &iarchive)
{
@@ -120,6 +149,7 @@ struct KeywordsScorePool {
m_keywordsDataMap[key] = item.second;
}
}
// LCOV_EXCL_STOP
template <class Archive>
void serialize(Archive& ar) {
@@ -148,6 +178,8 @@ public:
void snap();
double getSnapshotKeywordScore(const std::string &keyword, double defaultScore, const std::string &poolName) const;
double getSnapshotKeywordCoef(const std::string &keyword, double defaultCoef, const std::string &poolName) const;
KeywordsStats getSnapshotStats(const std::string &poolName) const;
keywords_set getIpItemKeywordsSet(std::string ip);
keywords_set getUaItemKeywordsSet(std::string userAgent);
@@ -200,6 +232,8 @@ protected:
SerializedData m_serializedData;
std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordScoreMap> m_snapshotKwCoefMap; // the snapshot is updated only by a call to snap()
std::map<std::string, KeywordsStats> m_snapshotStatsMap; // the snapshot is updated only by a call to snap()
std::list<std::string> m_falsePositivesSetsIntersection;
I_WaapAssetState* m_pWaapAssetState;
};

3
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -195,6 +195,9 @@ void SerializeToFileBase::saveData()
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
}
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
filestream << ss.str();

5
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@@ -12,7 +12,6 @@
// limitations under the License.
#include "WaapOverride.h"
#include "Waf2Util.h"
USE_DEBUG_FLAG(D_WAAP);
@@ -25,8 +24,8 @@ bool Match::operator==(const Match &other) const
(m_operand1 == other.m_operand1) &&
(m_operand2 == other.m_operand2) &&
(m_tag == other.m_tag) &&
Waap::Util::compareObjects(m_valueRegex, other.m_valueRegex) &&
m_cidr == other.m_cidr &&
(m_valuesRegex == other.m_valuesRegex) &&
m_ip_addr_values == other.m_ip_addr_values &&
m_isCidr == other.m_isCidr;
}

105
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@@ -21,6 +21,7 @@
#include <memory>
#include "debug.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
@@ -52,23 +53,52 @@ public:
m_isValid = false;
dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
}
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
try {
ar(cereal::make_nvp("values", m_values));
dbgDebug(D_WAAP_OVERRIDE) << "Values list is missing, using single value instead.";
} catch (const cereal::Exception &e) {
// The name "value" here is misleading. The real meaning is "regex pattern string"
ar(cereal::make_nvp("value", m_value));
m_values.insert(m_value);
}
if (m_tag == "sourceip" || m_tag == "sourceidentifier") {
m_isCidr = Waap::Util::isCIDR(m_value, m_cidr);
m_isCidr = true;
m_ip_addr_values.resize(m_values.size());
int val_idx = 0;
for (const auto &cur_val : m_values) {
if (!Waap::Util::isCIDR(cur_val, m_ip_addr_values[val_idx])) {
dbgDebug(D_WAAP_OVERRIDE) << "Invalid value in list of IP addresses: " << cur_val;
m_isValid = false;
break;
}
val_idx++;
}
sortAndMergeCIDRs();
dbgTrace(D_WAAP_OVERRIDE) << "CIDR list: " << cidrsToString(m_ip_addr_values);
}
m_isOverrideResponse = (m_tag == "responsebody" || m_tag == "responseBody");
if (!m_isCidr) {
// regex build may throw boost::regex_error
m_valueRegex = nullptr;
try {
m_valueRegex = std::make_shared<boost::regex>(m_value);
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE) << "Waap::Override::Match(): Failed to compile regex pattern '" <<
m_value << "' on position " << err.position() << ". Reason: '" << err.what() << "'";
for (const auto &cur_val : m_values) {
try {
m_valuesRegex.emplace(std::make_shared<boost::regex>(cur_val));
}
catch (const boost::regex_error &err) {
dbgDebug(D_WAAP_OVERRIDE)
<< "Waap::Override::Match(): Failed to compile regex pattern '"
<< cur_val
<< "' on position "
<< err.position()
<< ". Reason: '"
<< err.what()
<< "'";
m_isValid = false;
m_valuesRegex.clear();
break;
}
}
}
}
@@ -95,13 +125,21 @@ public:
template<typename TestFunctor>
bool match(TestFunctor testFunctor) const {
if (m_op == "basic" && m_isCidr) {
bool result = testFunctor(m_tag, m_cidr);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching CIDR: " << m_value << " result: " << result;
bool result = testFunctor(m_tag, m_ip_addr_values);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching CIDR list: "
<< cidrsToString(m_ip_addr_values)
<< " result: "
<< result;
return result;
}
else if (m_op == "basic" && m_valueRegex) {
bool result = testFunctor(m_tag, *m_valueRegex);
dbgTrace(D_WAAP_OVERRIDE) << "Override matching regex: " << m_value << " result: " << result;
else if (m_op == "basic" && !m_valuesRegex.empty()) {
bool result = testFunctor(m_tag, m_valuesRegex);
dbgTrace(D_WAAP_OVERRIDE)
<< "Override matching regex list: "
<< regexSetToString(m_valuesRegex)
<< " result: "
<< result;
return result;
}
if (m_op == "and") {
@@ -125,24 +163,39 @@ public:
return false;
}
bool isOverrideResponse() const {
return m_isOverrideResponse;
}
bool isOverrideResponse() const { return m_isOverrideResponse; }
bool isValidMatch() const{
return m_isValid;
}
bool isValidMatch() const { return m_isValid; }
private:
void sortAndMergeCIDRs() {
if (m_ip_addr_values.empty()) return;
std::sort(m_ip_addr_values.begin(), m_ip_addr_values.end());
size_t mergedIndex = 0;
for (size_t i = 1; i < m_ip_addr_values.size(); ++i) {
Waap::Util::CIDRData &current = m_ip_addr_values[mergedIndex];
Waap::Util::CIDRData &next = m_ip_addr_values[i];
if (!doesFirstCidrContainSecond(current, next)) {
++mergedIndex;
if (i != mergedIndex) m_ip_addr_values[mergedIndex] = next;
}
}
m_ip_addr_values.resize(mergedIndex + 1);
}
std::string m_op;
std::shared_ptr<Match> m_operand1;
std::shared_ptr<Match> m_operand2;
std::string m_tag;
std::string m_value;
std::shared_ptr<boost::regex> m_valueRegex;
Waap::Util::CIDRData m_cidr;
bool m_isCidr;
bool m_isOverrideResponse;
std::set<std::string> m_values;
std::vector<Waap::Util::CIDRData> m_ip_addr_values;
std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> m_valuesRegex;
bool m_isCidr;
bool m_isOverrideResponse;
bool m_isValid;
};

165
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -17,100 +17,150 @@
#include "WaapOverrideFunctor.h"
#include "Waf2Engine.h"
#include "CidrMatch.h"
#include "RegexComparator.h"
#include "agent_core_utilities.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_OVERRIDE);
#define REGX_MATCH(FIELD_FETCH) \
NGEN::Regex::regexMatch(__FILE__, __LINE__, FIELD_FETCH.c_str(), what, *rx)
#define W2T_REGX_MATCH(FIELD_GETTER) \
REGX_MATCH(waf2Transaction.FIELD_GETTER())
WaapOverrideFunctor::WaapOverrideFunctor(Waf2Transaction& waf2Transaction) :waf2Transaction(waf2Transaction)
{
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const Waap::Util::CIDRData& value) {
bool WaapOverrideFunctor::operator()(const std::string& tag, const std::vector<Waap::Util::CIDRData> &values) {
std::string sourceIp;
if (tag == "sourceip") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getRemoteAddr();
// match sourceIp against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
dbgDebug(D_WAAP_OVERRIDE)
<< "Remote IP Address : "
<< waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getRemoteAddr();
} else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr();
sourceIp = waf2Transaction.getSourceIdentifier();
} else {
dbgWarning(D_WAAP_OVERRIDE) << "Unsupported tag: " << tag;
return false;
}
else if (tag == "sourceidentifier") {
dbgDebug(D_WAAP_OVERRIDE) << "Remote IP Address : " << waf2Transaction.getRemoteAddr() << " CIDR: " <<
value.cidrString;
std::string sourceIp = waf2Transaction.getSourceIdentifier();
// match source against the cidr
return Waap::Util::cidrMatch(sourceIp, value);
Waap::Util::CIDRData source_cidr;
if (!Waap::Util::isCIDR(sourceIp, source_cidr)) {
dbgWarning(D_WAAP_OVERRIDE) << "Failed to create subnet from: " << sourceIp;
return false;
}
int left = 0;
int right = values.size() - 1;
while (left <= right) {
int mid = left + (right - left) / 2;
if (Waap::Util::cidrMatch(sourceIp, values[mid])) return true;
if (values[mid] < source_cidr) {
left = mid + 1;
} else {
right = mid - 1;
}
}
return false;
}
bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex& rx)
bool WaapOverrideFunctor::operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes)
{
boost::cmatch what;
std::string tagLower = tag;
std::transform(tagLower.begin(), tagLower.end(), tagLower.begin(), ::tolower);
try {
if (tagLower == "method") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getMethod().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getMethod)) return true;
}
return false;
}
else if (tagLower == "url") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getUriStr().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getUriStr)) return true;
}
return false;
}
else if (tagLower == "hostname") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getHost().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getHost)) return true;
}
return false;
}
else if (tagLower == "sourceidentifier") {
return NGEN::Regex::regexMatch(
__FILE__,
__LINE__,
waf2Transaction.getSourceIdentifier().c_str(),
what,
rx
);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getSourceIdentifier)) return true;
}
return false;
}
else if (tagLower == "keyword") {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordStr.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) {
if (REGX_MATCH(keywordStr)) {
return true;
}
}
}
return false;
}
else if (tagLower == "paramname") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getName().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getName())) {
return true;
}
}
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParamKey().c_str(), what, rx)) {
return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getParam().c_str(), what, rx)) {
return true;
if (W2T_REGX_MATCH(getParamKey)) return true;
if (W2T_REGX_MATCH(getParam)) return true;
}
return false;
}
else if (tagLower == "paramvalue") {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, keywordInfo.getValue().c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (const DeepParser::KeywordInfo& keywordInfo : waf2Transaction.getKeywordInfo()) {
if (REGX_MATCH(keywordInfo.getValue())) {
return true;
}
}
if (W2T_REGX_MATCH(getSample)) return true;
}
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getSample().c_str(), what, rx)) {
return true;
}
return false;
}
else if (tagLower == "paramlocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
for (const auto &rx : rxes) {
if (W2T_REGX_MATCH(getLocation)) return true;
}
return false;
}
else if (tagLower == "responsebody") {
waf2Transaction.getResponseInspectReasons().setApplyOverride(true);
if (!waf2Transaction.getResponseBody().empty()) {
boost::smatch matcher;
return NGEN::Regex::regexSearch(__FILE__, __LINE__,
waf2Transaction.getResponseBody().c_str(), matcher, rx);
for (const auto &rx : rxes) {
boost::smatch matcher;
if (NGEN::Regex::regexSearch(
__FILE__,
__LINE__,
waf2Transaction.getResponseBody().c_str(),
matcher,
*rx
)) {
return true;
}
}
return false;
} else {
return false;
}
@@ -119,11 +169,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header name override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if(NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.first;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@@ -132,11 +184,13 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "Header value override scan is not required";
return false;
}
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, value.c_str(), what, rx)) {
return true;
for (const auto &rx : rxes) {
for (auto& hdr_pair : waf2Transaction.getHdrPairs()) {
std::string value = hdr_pair.second;
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
if (REGX_MATCH(value)) {
return true;
}
}
}
return false;
@@ -146,7 +200,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();
return false;
}
// Unknown tag: should not occur
dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
return false;

12
components/security_apps/waap/waap_clib/WaapOverrideFunctor.h
View File

@@ -16,6 +16,7 @@
namespace Waap {
namespace Util {
struct CIDRData; // forward decleration
struct RegexComparator;
}
}
@@ -24,12 +25,15 @@ class Waf2Transaction;
// Functor used to match Override rules against request data
class WaapOverrideFunctor {
public:
WaapOverrideFunctor(Waf2Transaction& waf2Transaction);
WaapOverrideFunctor(Waf2Transaction &waf2Transaction);
bool operator()(const std::string& tag, const Waap::Util::CIDRData& value);
bool operator()(const std::string &tag, const std::vector<Waap::Util::CIDRData> &values);
bool operator()(const std::string& tag, const boost::regex& rx);
bool operator()(
const std::string &tag,
const std::set<std::shared_ptr<boost::regex>, Waap::Util::RegexComparator> &rxes
);
private:
Waf2Transaction& waf2Transaction;
Waf2Transaction &waf2Transaction;
};

49
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -13,7 +13,9 @@
#include "WaapScanner.h"
#include "WaapScores.h"
#include "Waf2Engine.h"
#include "i_transaction.h"
#include "WaapModelResultLogger.h"
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
@@ -105,20 +107,59 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
}
std::sort(newKeywords.begin(), newKeywords.end());
res.keywordsAfterFilter.clear();
for (auto keyword : newKeywords) {
res.keywordsAfterFilter.push_back(keyword);
}
res.scoreArray.clear();
res.coefArray.clear();
res.keywordCombinations.clear();
double res_score = getScoreFromPool(res, newKeywords, poolName);
string other_pool_name = Waap::Scores::getOtherScorePoolName();
Waap::Scores::ModelLoggingSettings modelLoggingSettings = Waap::Scores::getModelLoggingSettings();
if (applyLearning && poolName != other_pool_name &&
modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) {
double other_score = getScoreFromPool(res, newKeywords, other_pool_name);
dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score
<< ", vs. pool " << other_pool_name << ": " << other_score
<< ", score difference: " << res_score - other_score
<< ", sample: " << res.unescaped_line;
Singleton::Consume<I_WaapModelResultLogger>::by<WaapComponent>()->logModelResult(
modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score
);
res.other_model_score = other_score;
} else {
res.other_model_score = res_score;
}
return res_score;
}
double Waap::Scanner::getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
)
{
KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName);
if (!newKeywords.empty()) {
// Collect scores of individual keywords
Waap::Scores::calcIndividualKeywords(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray);
res.scoreArray, res.coefArray);
// Collect keyword combinations and their scores. Append scores to scoresArray,
// and also populate m_scanResultKeywordCombinations list
Waap::Scores::calcCombinations(m_transaction->getAssetState()->scoreBuilder, poolName, newKeywords,
res.scoreArray, res.keywordCombinations);
res.scoreArray, res.coefArray, res.keywordCombinations);
}
if (stats.isLinModel) {
return Waap::Scores::calcLogisticRegressionScore(
res.coefArray, stats.linModelIntercept, stats.linModelNNZCoef
);
}
// use base_scores calculation
return Waap::Scores::calcArrayScore(res.scoreArray);
}
@@ -127,7 +168,7 @@ bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res,
{
if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
return true;
}

3
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -44,6 +44,9 @@ namespace Waap {
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName, bool applyLearning = true);
double getScoreFromPool(
Waf2ScanResult &res, const std::vector<std::string> &newKeywords, const std::string &poolName
);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);

79
components/security_apps/waap/waap_clib/WaapScores.cc
View File

@@ -14,6 +14,8 @@
#include "WaapScores.h"
#include <vector>
#include <string>
#include <cmath>
#include "ScoreBuilder.h"
#include "WaapDefines.h"
#include "debug.h"
@@ -24,11 +26,44 @@ namespace Waap {
namespace Scores {
std::string getScorePoolNameByLocation(const std::string &location) {
std::string poolName = KEYWORDS_SCORE_POOL_BASE;
if (location == "header") {
poolName = KEYWORDS_SCORE_POOL_HEADERS;
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.scorePoolName");
std::string res = KEYWORDS_SCORE_POOL_BASE;
if (maybePoolName.ok()) {
res = maybePoolName.unpack();
}
return poolName;
else if (location == "header") {
res = KEYWORDS_SCORE_POOL_HEADERS;
}
return res;
}
std::string getOtherScorePoolName()
{
auto maybePoolName = getProfileAgentSetting<std::string>("agent.waap.otherScorePoolName");
if (maybePoolName.ok()) {
return maybePoolName.unpack();
}
return KEYWORDS_SCORE_POOL_BASE;
}
ModelLoggingSettings getModelLoggingSettings()
{
ModelLoggingSettings settings = {.logLevel = ModelLogLevel::DIFF,
.logToS3 = false,
.logToStream = true};
auto maybeLogS3 = getProfileAgentSetting<bool>("agent.waap.modelLogToS3");
if (maybeLogS3.ok()) {
settings.logToS3 = maybeLogS3.unpack();
}
auto maybeLogKusto = getProfileAgentSetting<bool>("agent.waap.modelLogToStream");
if (maybeLogKusto.ok()) {
settings.logToStream = maybeLogKusto.unpack();
}
auto maybeLogLevel = getProfileAgentSetting<uint>("agent.waap.modelLogLevel");
if (maybeLogLevel.ok() && (settings.logToS3 || settings.logToStream)) {
settings.logLevel = static_cast<ModelLogLevel>(maybeLogLevel.unpack());
}
return settings;
}
void
@@ -37,9 +72,16 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray)
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
scoresArray.push_back(scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName));
double score = scoreBuilder.getSnapshotKeywordScore(keyword, defaultScore, poolName);
double coef = scoreBuilder.getSnapshotKeywordCoef(keyword, defaultCoef, poolName);
dbgDebug(D_WAAP_SCORE_BUILDER) << "Adding score: " << score << " coef: " << coef
<< " keyword: '" << keyword << "' pool: " << poolName;
scoresArray.push_back(score);
coefArray.push_back(coef);
}
// Calculate score of individual keywords
@@ -48,13 +90,14 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray)
std::vector<double>& scoresArray,
std::vector<double>& coefArray)
{
std::vector<std::string> keywords = keyword_matches; // deep copy!! (PERFORMANCE WARNING!)
std::sort(keywords.begin(), keywords.end());
for (auto pKeyword = keywords.begin(); pKeyword != keywords.end(); ++pKeyword) {
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, scoresArray);
addKeywordScore(scoreBuilder, poolName, *pKeyword, 2.0f, 0.3f, scoresArray, coefArray);
}
}
@@ -65,10 +108,12 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations)
{
keyword_combinations.clear();
static const double max_combi_score = 1.0f;
double default_coef = 0.8f;
for (size_t i = 0; i < keyword_matches.size(); ++i) {
std::vector<std::string> combinations;
@@ -93,7 +138,7 @@ calcCombinations(
}
// set default combination score to be the sum of its keywords, bounded by 1
default_score = std::min(default_score, max_combi_score);
addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
addKeywordScore(scoreBuilder, poolName, combination, default_score, default_coef, scoresArray, coefArray);
keyword_combinations.push_back(combination);
}
}
@@ -105,7 +150,6 @@ calcArrayScore(std::vector<double>& scoreArray)
// Calculate cumulative score from array of individual scores
double score = 1.0f;
for (auto pScore = scoreArray.begin(); pScore != scoreArray.end(); ++pScore) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "scoreArr[]=" << *pScore;
double left = 10.0f - score;
double divisor = (*pScore / 3.0f + 10.0f); // note: divisor can't be empty because
// *pScore is always positive and there's a +10 offset
@@ -115,5 +159,20 @@ calcArrayScore(std::vector<double>& scoreArray)
return score;
}
double
calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef)
{
// Sparse logistic regression model, with boolean feature values
// Instead of performing a dot product of features*coefficients, we sum the coefficients of the non-zero features
// An additional feature was added for the log of the number of non-zero features, as a regularization term
double log_odds = intercept + nnzCoef * log(static_cast<double>(coefArray.size()) + 1);
for (double &pCoef : coefArray) {
log_odds += pCoef;
}
// Apply the expit function to the log-odds to obtain the probability,
// and multiply by 10 to obtain a 'score' in the range [0, 10]
return 1.0f / (1.0f + exp(-log_odds)) * 10.0f;
}
}
}

23
components/security_apps/waap/waap_clib/WaapScores.h
View File

@@ -20,7 +20,21 @@
namespace Waap {
namespace Scores {
enum class ModelLogLevel {
OFF = 0,
DIFF = 1,
ALL = 2
};
struct ModelLoggingSettings {
ModelLogLevel logLevel;
bool logToS3;
bool logToStream;
};
std::string getScorePoolNameByLocation(const std::string &location);
std::string getOtherScorePoolName();
ModelLoggingSettings getModelLoggingSettings();
void
addKeywordScore(
@@ -28,7 +42,9 @@ addKeywordScore(
const std::string &poolName,
std::string keyword,
double defaultScore,
std::vector<double>& scoresArray);
double defaultCoef,
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate score of individual keywords
void
@@ -36,7 +52,8 @@ calcIndividualKeywords(
const ScoreBuilder& scoreBuilder,
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray);
std::vector<double>& scoresArray,
std::vector<double>& coefArray);
// Calculate keyword combinations and their scores
void
@@ -45,9 +62,11 @@ calcCombinations(
const std::string &poolName,
const std::vector<std::string>& keyword_matches,
std::vector<double>& scoresArray,
std::vector<double>& coefArray,
std::vector<std::string>& keyword_combinations);
double calcArrayScore(std::vector<double>& scoreArray);
double calcLogisticRegressionScore(std::vector<double> &coefArray, double intercept, double nnzCoef=0.0);
}
}

2
components/security_apps/waap/waap_clib/WaapTrigger.h
View File

@@ -125,6 +125,7 @@ struct Log {
struct Trigger {
template <typename _A>
void serialize(_A &ar) {
ar(cereal::make_nvp("id", triggerId));
ar(cereal::make_nvp("$triggerType", triggerType));
triggerType = to_lower_copy(triggerType);
@@ -137,6 +138,7 @@ struct Trigger {
Trigger();
bool operator==(const Trigger &other) const;
std::string triggerId;
std::string triggerType;
std::shared_ptr<Log> log;
};

28
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -819,8 +819,15 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = uriEnd - p;
dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
bool should_decode = checkUrlEncoded(p, buff_len);
if (should_decode) {
std::string url_as_string(p, buff_len);
should_decode = should_decode && (Waap::Util::testUrlBadUtf8Evasion(url_as_string) != true);
}
dbgTrace(D_WAAP) << "should_decode % = " << should_decode;
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, should_decode);
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@@ -1300,6 +1307,15 @@ void Waf2Transaction::set_ignoreScore(bool ignoreScore) {
m_ignoreScore = ignoreScore;
}
bool Waf2Transaction::get_ignoreScore() const
{
auto maybe_override_ignore_score = getProfileAgentSetting<std::string>("agent.waap.alwaysReportScore");
if (maybe_override_ignore_score.ok()) {
return maybe_override_ignore_score.unpack() == "true";
}
return m_ignoreScore;
}
void
Waf2Transaction::decide(
bool& bForceBlock,
@@ -1889,6 +1905,14 @@ Waf2Transaction::sendLog()
return;
}
std::set<std::string> triggers_set;
for (const Waap::Trigger::Trigger &trigger : triggerPolicy->triggers) {
triggers_set.insert(trigger.triggerId);
dbgTrace(D_WAAP) << "Add waap log trigger id to triggers set:" << trigger.triggerId;
}
ScopedContext ctx;
ctx.registerValue<std::set<GenericConfigId>>(TriggerMatcher::ctx_key, triggers_set);
auto maybeLogTriggerConf = getConfiguration<LogTriggerConf>("rulebase", "log");
switch (decision_type)
{

9
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -20,6 +20,8 @@
#include "DeepParser.h"
#include "WaapAssetState.h"
#include "PatternMatcher.h"
#include "generic_rulebase/rulebase_config.h"
#include "generic_rulebase/evaluators/trigger_eval.h"
#include "Waf2Util.h"
#include "WaapConfigApplication.h"
#include "WaapConfigApi.h"
@@ -39,6 +41,7 @@
#include "i_waap_telemetry.h"
#include "i_deepAnalyzer.h"
#include "i_time_get.h"
#include "i_waap_model_result_logger.h"
#include "table_opaque.h"
#include "WaapResponseInspectReasons.h"
#include "WaapResponseInjectReasons.h"
@@ -91,6 +94,7 @@ public:
const std::vector<std::string> getFilteredKeywords() const;
const std::map<std::string, std::vector<std::string>> getFilteredVerbose() const;
virtual const std::vector<std::string> getKeywordsCombinations() const;
virtual const std::vector<std::string> getKeywordsAfterFilter() const;
const std::vector<DeepParser::KeywordInfo>& getKeywordInfo() const;
const std::vector<std::pair<std::string, std::string> >& getKvPairs() const;
const std::string getKeywordMatchesStr() const;
@@ -99,6 +103,9 @@ public:
const std::string getLastScanSample() const;
virtual const std::string& getLastScanParamName() const;
double getScore() const;
// LCOV_EXCL_START Reason: model testing
double getOtherModelScore() const;
// LCOV_EXCL_STOP
const std::vector<double> getScoreArray() const;
Waap::CSRF::State& getCsrfState();
const std::set<std::string> getFoundPatterns() const;
@@ -161,7 +168,7 @@ public:
// decision functions
void set_ignoreScore(bool ignoreScore);
bool get_ignoreScore() const { return m_ignoreScore; }
bool get_ignoreScore() const;
void decide(
bool& bForceBlock,
bool& bForceException,

19
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -189,6 +189,15 @@ const std::vector<std::string> Waf2Transaction::getKeywordsCombinations() const
}
return std::vector<std::string>();
}
const std::vector<std::string> Waf2Transaction::getKeywordsAfterFilter() const
{
if (m_scanResult)
{
return m_scanResult->keywordsAfterFilter;
}
return std::vector<std::string>();
}
const std::vector<DeepParser::KeywordInfo>& Waf2Transaction::getKeywordInfo() const
{
return m_deepParser.m_keywordInfo;
@@ -230,6 +239,15 @@ double Waf2Transaction::getScore() const
}
return 0;
}
// LCOV_EXCL_START Reason: model testing
double Waf2Transaction::getOtherModelScore() const
{
if (m_scanResult) {
return m_scanResult->other_model_score;
}
return 0;
}
// LCOV_EXCL_STOP
const std::vector<double> Waf2Transaction::getScoreArray() const
{
if (m_scanResult) {
@@ -397,6 +415,7 @@ void Waf2Transaction::sendAutonomousSecurityLog(
waap_log << LogField("waapUriFalsePositiveScore", (int)(
autonomousSecurityDecision->getFpMitigationScore() * 100));
waap_log << LogField("waapKeywordsScore", (int)(getScore() * 100));
waap_log << LogField("reservedNgenA", (int)(getOtherModelScore() * 100));
waap_log << LogField("waapFinalScore", (int)(autonomousSecurityDecision->getFinalScore() * 100));
waap_log << LogField("waapCalculatedThreatLevel", autonomousSecurityDecision->getThreatLevel());
}

39
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -508,6 +508,8 @@ const char* g_htmlTags[] = {
};
static const string b64_prefix("base64,");
// Empirically calculated entropy threshold for base64 encoded data, value above it is considered as base64 encoded
static const double base64_entropy_threshold = 4.01;
const size_t g_htmlTagsCount = sizeof(g_htmlTags) / sizeof(g_htmlTags[0]);
@@ -966,7 +968,8 @@ base64_decode_status decodeBase64Chunk(
string::const_iterator it,
string::const_iterator end,
string& decoded,
bool clear_on_error)
bool clear_on_error,
bool called_with_prefix)
{
decoded.clear();
uint32_t acc = 0;
@@ -974,6 +977,7 @@ base64_decode_status decodeBase64Chunk(
int terminatorCharsSeen = 0; // whether '=' character was seen, and how many of them.
uint32_t nonPrintableCharsCount = 0;
uint32_t spacer_count = 0;
uint32_t length = end - it;
dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
string::const_iterator begin = it;
@@ -986,6 +990,8 @@ base64_decode_status decodeBase64Chunk(
return B64_DECODE_INVALID;
}
std::unordered_map<char, double> frequency;
while (it != end) {
unsigned char c = *it;
@@ -1008,6 +1014,7 @@ base64_decode_status decodeBase64Chunk(
// allow for more terminator characters
it++;
frequency[c]++;
continue;
}
@@ -1032,6 +1039,7 @@ base64_decode_status decodeBase64Chunk(
// Start tracking terminator characters
terminatorCharsSeen++;
it++;
frequency[c]++;
continue;
}
else {
@@ -1062,6 +1070,7 @@ base64_decode_status decodeBase64Chunk(
}
it++;
frequency[c]++;
}
// end of encoded sequence decoded.
@@ -1078,6 +1087,27 @@ base64_decode_status decodeBase64Chunk(
<< "; decoded='"
<< decoded << "'";
// Check if entropy is correlates with b64 threshold (initially > 4.5)
if (!called_with_prefix) {
double entropy = 0;
double p = 0;
for (const auto& pair : frequency) {
p = pair.second / length;
entropy -= p * std::log2(p);
}
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: base entropy = " << entropy << "length = " << length;
// Add short payload factor
if (length < 16)
entropy = entropy * 16 / length;
// Enforce tailoring '=' characters
entropy+=terminatorCharsSeen;
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: corrected entropy = " << entropy << "length = " << length;
if (entropy <= base64_entropy_threshold) {
return B64_DECODE_INVALID;
}
}
// Return success only if decoded.size>=5 and there are less than 10% of non-printable
// characters in output.
if (decoded.size() >= 5) {
@@ -1323,10 +1353,11 @@ processDecodedChunk(
string::const_iterator start,
string::const_iterator end,
string &value,
BinaryFileType &binaryFileType
BinaryFileType &binaryFileType,
bool called_with_prefix = false
)
{
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false, called_with_prefix);
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
<< " value.size()=" << value.size();
if (retVal != B64_DECODE_INVALID && !value.empty()) {
@@ -1349,7 +1380,7 @@ bool isBase64PrefixProcessingOK (
if (detectBase64Chunk(s, start, end)) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
if ((start != s.end()) && (end == s.end())) {
retVal = processDecodedChunk(s, start, end, value, binaryFileType);
retVal = processDecodedChunk(s, start, end, value, binaryFileType, true);
}
} else if (start != s.end()) {
dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."

3
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -871,7 +871,8 @@ decodeBase64Chunk(
std::string::const_iterator it,
std::string::const_iterator end,
std::string &decoded,
bool clear_on_error = true);
bool clear_on_error = true,
bool called_with_prefix = false);
bool
b64DecodeChunk(

3
components/security_apps/waap/waap_component_impl.cc
View File

@@ -50,7 +50,8 @@ WaapComponent::Impl::Impl() :
drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP),
waapStateTable(NULL),
transactionsCount(0),
deepAnalyzer()
deepAnalyzer(),
waapModelResultLogger()
{
}

2
components/security_apps/waap/waap_component_impl.h
View File

@@ -19,6 +19,7 @@
#include "table_opaque.h"
#include "i_transaction.h"
#include "waap_clib/DeepAnalyzer.h"
#include "waap_clib/WaapModelResultLogger.h"
#include "waap_clib/WaapAssetState.h"
#include "waap_clib/WaapAssetStatesManager.h"
#include "reputation_features_agg.h"
@@ -80,6 +81,7 @@ private:
uint64_t transactionsCount;
// instance of singleton classes
DeepAnalyzer deepAnalyzer;
WaapModelResultLogger waapModelResultLogger;
WaapAssetStatesManager waapAssetStatesManager;
std::unordered_set<std::string> m_seen_assets_id;
};