github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Jun 16th update

Browse Source
This commit is contained in:
noam
2023-01-17 11:34:09 +02:00
parent 90bcc544a2
commit ad04b8d063
168 changed files with 64034 additions and 932 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
components/security_apps/waap/include/i_serialize.h
View File

@@ -15,7 +15,6 @@
#include <chrono>
#include <fstream>
#include "i_time_get.h"
#include "i_encryptor.h"
#include "rest.h"
#include "i_messaging.h"
#include "i_mainloop.h"
@@ -92,7 +91,6 @@ public:
virtual void saveData();
virtual void restore();
virtual void setFilePath(const std::string &new_file_path);
protected:
// saved file name for testing

4
components/security_apps/waap/include/i_waap_asset_state.h
View File

@@ -18,8 +18,8 @@
class I_WaapAssetState {
public:
virtual void updateScores() = 0;
virtual std::string getSignaturesScoresFilePath() const = 0;
virtual std::string getSignaturesFilterDir() const = 0;
virtual std::string getWaapDataFileName() const = 0;
virtual std::string getWaapDataDir() const = 0;
virtual bool isKeywordOfType(const std::string& keyword, ParamType type) const = 0;
virtual bool isBinarySampleType(const std::string & sample) const = 0;
virtual bool isWBXMLSampleType(const std::string & sample) const = 0;

59902
components/security_apps/waap/resources/waap.data Normal file
View File

File diff suppressed because it is too large Load Diff

2
components/security_apps/waap/waap_clib/CidrMatch.cc
View File

@@ -80,7 +80,7 @@ bool isCIDR(const std::string& strCIDR, CIDRData& cidr)
// get ip from targetCidr
std::string strPrefix = pos != std::string::npos ? strCIDR.substr(0, pos) : strCIDR;
// get subnet mask from targetCidr or calculate it based on ipv4 / ipv6
std::string strSuffix = pos != std::string::npos ? strCIDR.substr(pos + 1) :
std::string strSuffix = (pos != std::string::npos && (pos + 1) <= strCIDR.size()) ? strCIDR.substr(pos + 1) :
(strCIDR.find(':') == std::string::npos) ? "32" : "128";
int bits = -1;

6
components/security_apps/waap/waap_clib/ConfidenceCalculator.cc
View File

@@ -642,7 +642,11 @@ void ConfidenceCalculator::calculateInterval()
double factor = 1.0;
if (m_tuning != nullptr)
{
std::string param_name = key.substr(key.find("#") + 1); // not always accurate but good enough
std::string param_name = key;
auto param_name_pos = key.find("#");
if (param_name_pos != std::string::npos && (param_name_pos + 1) <= key.size()) {
param_name = key.substr(param_name_pos + 1); // not always accurate but good enough
}
if (m_tuning->getDecision(param_name, PARAM_NAME) == BENIGN)
{
factor = BENIGN_PARAM_FACTOR;

4
components/security_apps/waap/waap_clib/CsrfPolicy.h
View File

@@ -30,6 +30,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("csrfProtection", level));
level = boost::algorithm::to_lower_copy(level);

1
components/security_apps/waap/waap_clib/DecisionFactory.cc
View File

@@ -95,6 +95,7 @@ void DecisionFactory::initCsrfDecision()
}
}
void DecisionFactory::initOpenRedirectDecision()
{
DecisionType type = DecisionType::OPEN_REDIRECT_DECISION;

66
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -63,6 +63,7 @@ DeepParser::~DeepParser()
{
}
void DeepParser::setWaapAssetState(std::shared_ptr<WaapAssetState> pWaapAssetState)
{
m_pWaapAssetState = pWaapAssetState;
@@ -284,18 +285,21 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
if (flags & BUFFERED_RECEIVER_F_FIRST)
{
createInternalParser(orig_val,
createInternalParser(k, k_len, orig_val,
valueStats,
isBodyPayload,
isRefererPayload,
isRefererParamPayload,
isUrlPayload,
isUrlParamPayload);
isUrlParamPayload,
flags);
}
// If there's a parser in parsers stack, push the value to the top parser
if (!m_parsersDeque.empty() && !m_parsersDeque.front()->getRecursionFlag())
{
ScopedContext ctx;
ctx.registerValue<IWaf2Transaction*>("waap_transaction", m_pTransaction);
rc = pushValueToTopParser(cur_val, flags, base64ParamFound);
if (rc != CONTINUE_PARSING)
{
@@ -680,13 +684,32 @@ int DeepParser::pushValueToTopParser(std::string& cur_val, int flags, bool base6
return CONTINUE_PARSING;
}
void DeepParser::createInternalParser(std::string& cur_val,
class StubParserReceiver : public IParserReceiver {
int
onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
{
return 0;
}
};
static bool
validateJson(const char *v, size_t v_len)
{
StubParserReceiver rcvr;
ParserJson jsParser(rcvr);
jsParser.push(v, v_len);
dbgTrace(D_WAAP_DEEP_PARSER) << "json validation: " << (jsParser.error() ? "invalid" : "valid");
return !jsParser.error();
}
void DeepParser::createInternalParser(const char *k, size_t k_len, std::string& cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload)
bool isUrlParamPayload,
int flags)
{
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false,
isCommaType = false, isAmperType = false;
@@ -795,12 +818,31 @@ void DeepParser::createInternalParser(std::string& cur_val,
}
// This flag is enabled when current value is either top level (depth==1), or one-level inside multipart-encoded
// container (depth==2 and type of top parser is )
// container (depth==2 and type of top parser is "ParserMultipartForm")
bool isTopData = m_depth == 1
|| (m_depth == 2 && !m_parsersDeque.empty() && m_parsersDeque.front()->name() == "ParserMultipartForm");
// GQL query can potentially be in one of three places in HTTP request:
// 1. In url parameter named "query"
// 2. In the body when Content-Type is "application/graphql"
// 3. In the JSON contained in body, where top-level JSON parameter is named "query"
// Note: we consider decoding Graphql format only if it is contained whole within the MAX_VALUE_SIZE (64k) buffer
// size (you can find the value of MAX_VALUE_SIZE defined in ParserBase.cc).
Waap::Util::ContentType requestContentType = m_pTransaction->getContentType();
bool isPotentialGqlQuery = false;
if (flags == BUFFERED_RECEIVER_F_BOTH) { // TODO:: should we limit ourselves to the 64k buffer?
static std::string strQuery("query");
bool isParamQuery = strQuery.size() == k_len && std::equal(k, k + k_len, strQuery.begin());
isPotentialGqlQuery |= isParamQuery && m_depth == 1 && (isUrlParamPayload || isRefererParamPayload);
isPotentialGqlQuery |= m_depth == 1 && isBodyPayload && requestContentType == Waap::Util::CONTENT_TYPE_GQL;
isPotentialGqlQuery |= isParamQuery && m_depth == 2 && isBodyPayload &&
requestContentType == Waap::Util::CONTENT_TYPE_JSON;
}
dbgTrace(D_WAAP_DEEP_PARSER)
<< "isTopData="
<< "isPotentialGqlQuery="
<< isPotentialGqlQuery
<< ";isTopData="
<< isTopData
<< ";depth="
<< m_depth
@@ -838,6 +880,8 @@ void DeepParser::createInternalParser(std::string& cur_val,
// JSON value detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file";
// Send openApiReceiver as secondary receiver, but only if the JSON is passed in body and on the top level.
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserJson>>(*this));
}
}
@@ -1020,6 +1064,16 @@ bool DeepParser::isBinaryData() const
return false;
}
const std::string DeepParser::getLastParser() const
{
if (m_parsersDeque.empty()) {
return "";
}
else {
return m_parsersDeque.front()->name();
}
}
bool DeepParser::isWBXmlData() const
{
return m_is_wbxml;

6
components/security_apps/waap/waap_clib/DeepParser.h
View File

@@ -43,6 +43,7 @@ public:
void setMultipartBoundary(const std::string &boundary);
const std::string &getMultipartBoundary() const;
bool isBinaryData() const;
const std::string getLastParser() const;
bool isWBXmlData() const;
Maybe<std::string> getSplitType() const;
std::vector<std::pair<std::string, std::string> > kv_pairs;
@@ -114,13 +115,14 @@ private:
// note: This function calls onKv(), and the call can be recursive!
// TODO:: maybe convert this splitter to Parser-derived class?!
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix);
void createInternalParser(std::string& cur_val,
void createInternalParser(const char *k, size_t k_len, std::string& cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload);
bool isUrlParamPayload,
int flags);
int pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound);
int parseBuffer(ValueStatsAnalyzer& valueStats, const std::string &cur_val, bool base64ParamFound,
bool shouldUpdateKeyStack);

8
components/security_apps/waap/waap_clib/IndicatorsFiltersManager.cc
View File

@@ -23,13 +23,13 @@
IndicatorsFiltersManager::IndicatorsFiltersManager(const std::string& remotePath, const std::string &assetId,
I_WaapAssetState* pWaapAssetState)
:
SerializeToFileBase(pWaapAssetState->getSignaturesFilterDir() + "/6.data"),
m_ignoreSources(pWaapAssetState->getSignaturesFilterDir(), remotePath, assetId),
SerializeToFileBase(pWaapAssetState->getWaapDataDir() + "/6.data"),
m_ignoreSources(pWaapAssetState->getWaapDataDir(), remotePath, assetId),
m_tuning(remotePath)
{
restore();
m_keywordsFreqFilter = std::make_unique<KeywordIndicatorFilter>(
pWaapAssetState->getSignaturesFilterDir(),
pWaapAssetState->getWaapDataDir(),
remotePath,
assetId,
&m_ignoreSources,
@@ -206,7 +206,7 @@ std::string IndicatorsFiltersManager::extractUri(const std::string& referer, con
std::string url;
size_t pos = referer.find("://");
if (pos == std::string::npos)
if (pos == std::string::npos || (pos + 3) > referer.size())
{
url = referer;
}

35
components/security_apps/waap/waap_clib/KeywordTypeValidator.cc
View File

@@ -16,13 +16,13 @@
#include <cereal/types/unordered_map.hpp>
#include <cereal/types/unordered_set.hpp>
#include "debug.h"
#include "Waf2Util.h"
USE_DEBUG_FLAG(D_WAAP);
KeywordTypeValidator::KeywordTypeValidator(const std::string& mapFilePath) :
SerializeToFileBase(mapFilePath),
m_keywordTypeMap()
m_serializedData(),
m_keywordTypeMap(m_serializedData.m_keywordTypeMap)
{
restore();
}
@@ -44,25 +44,22 @@ void KeywordTypeValidator::saveData()
void KeywordTypeValidator::deserialize(std::istream& stream)
{
cereal::JSONInputArchive archive(stream);
std::unordered_map<std::string, std::unordered_set<std::string>> typesStrToKeysMap;
archive(cereal::make_nvp("keywordsTypeMap", typesStrToKeysMap));
for (auto typeStrItr : typesStrToKeysMap)
try
{
ParamType type = Waap::Util::convertTypeStrToEnum(typeStrItr.first);
for (auto keyword : typeStrItr.second)
{
if (m_keywordTypeMap.find(keyword) == m_keywordTypeMap.end())
{
// initialize type set
m_keywordTypeMap[keyword];
}
m_keywordTypeMap[keyword].insert(type);
}
cereal::JSONInputArchive archive(stream);
archive(
cereal::make_nvp("waap_kw_type_map", m_serializedData)
);
}
catch (std::runtime_error & e) {
dbgWarning(D_WAAP) << "failed to deserialize keyword types validator file. Error: " << e.what();
}
}
void KeywordTypeValidator::operator=(const KeywordTypeValidator &other) {
m_serializedData.m_keywordTypeMap = other.m_serializedData.m_keywordTypeMap;
}
bool KeywordTypeValidator::isKeywordOfType(const std::string& keyword, ParamType type) const

31
components/security_apps/waap/waap_clib/KeywordTypeValidator.h
View File

@@ -16,6 +16,7 @@
#include <unordered_map>
#include <unordered_set>
#include "WaapEnums.h"
#include "Waf2Util.h"
#include "i_serialize.h"
class KeywordTypeValidator : public SerializeToFileBase
@@ -30,6 +31,34 @@ public:
virtual void deserialize(std::istream& stream);
virtual void saveData();
void operator=(const KeywordTypeValidator &other);
private:
std::unordered_map<std::string, std::unordered_set<ParamType>> m_keywordTypeMap;
struct SerializedData {
template <class Archive>
void serialize(Archive& archive) {
std::unordered_map<std::string, std::unordered_set<std::string>> typesStrToKeysMap;
archive(cereal::make_nvp("keywordsTypeMap", typesStrToKeysMap));
for (auto typeStrItr : typesStrToKeysMap)
{
ParamType type = Waap::Util::convertTypeStrToEnum(typeStrItr.first);
for (auto keyword : typeStrItr.second)
{
if (m_keywordTypeMap.find(keyword) == m_keywordTypeMap.end())
{
// initialize type set
m_keywordTypeMap[keyword];
}
m_keywordTypeMap[keyword].insert(type);
}
}
}
std::unordered_map<std::string, std::unordered_set<ParamType>> m_keywordTypeMap;
};
SerializedData m_serializedData;
std::unordered_map<std::string, std::unordered_set<ParamType>> &m_keywordTypeMap;
};

5
components/security_apps/waap/waap_clib/ParserJson.cc
View File

@@ -102,11 +102,12 @@ int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char*)s, slen) << "'";
m_key.push((char*)s, slen);
if (m_receiver2) {
m_receiver2->onMapKey((const char*)s, slen);
m_receiver2->onMapKey(m_key.c_str(), m_key.size());
}
m_key.push((char*)s, slen);
return 1;
}

11
components/security_apps/waap/waap_clib/ParserXML.cc
View File

@@ -15,6 +15,8 @@
#include "Waf2Util.h"
#include "debug.h"
#include <assert.h>
#include <boost/algorithm/string/case_conv.hpp>
#include <string>
USE_DEBUG_FLAG(D_WAAP_PARSER_XML);
@@ -34,6 +36,15 @@ void ParserXML::onStartElementNs(
ParserXML* p = (ParserXML*)ctx;
dbgTrace(D_WAAP_PARSER_XML) << "XML OPEN: '" << localname << "'";
std::string aux_localname((const char*)localname, xmlStrlen(localname));
boost::algorithm::to_lower(aux_localname);
if (aux_localname == "script") {
dbgTrace(D_WAAP_PARSER_XML) << "Failing parser on <script> tag";
p->m_state = s_error;
}
p->m_key.push((const char*)localname, xmlStrlen(localname));
int i;

85
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -99,10 +99,10 @@ void KeywordsScorePool::mergeScores(const KeywordsScorePool& baseScores)
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
SerializeToFilePeriodically(duration_cast<seconds>(minutes(10)), pWaapAssetState->getSignaturesScoresFilePath()),
m_scoreTrigger(0),
m_fpStore(),
m_keywordsScorePools(),
m_serializedData(),
m_keywordsScorePools(m_serializedData.m_keywordsScorePools),
m_falsePositivesSetsIntersection(),
m_pWaapAssetState(pWaapAssetState)
{
@@ -110,10 +110,10 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState) :
}
ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores) :
SerializeToFilePeriodically(duration_cast<seconds>(minutes(10)), pWaapAssetState->getSignaturesScoresFilePath()),
m_scoreTrigger(0),
m_fpStore(),
m_keywordsScorePools(),
m_serializedData(),
m_keywordsScorePools(m_serializedData.m_keywordsScorePools),
m_falsePositivesSetsIntersection(),
m_pWaapAssetState(pWaapAssetState)
{
@@ -123,44 +123,52 @@ ScoreBuilder::ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& base
mergeScores(baseScores);
}
void ScoreBuilder::serialize(std::ostream& stream) {
cereal::JSONOutputArchive archive(stream);
static const size_t version = 1;
archive(
cereal::make_nvp("version", version),
cereal::make_nvp("scorePools", m_keywordsScorePools)
);
}
void ScoreBuilder::restore()
{
const std::string filePath = this->m_pWaapAssetState->getWaapDataFileName();
void ScoreBuilder::deserialize(std::istream& stream) {
cereal::JSONInputArchive iarchive(stream);
dbgTrace(D_WAAP_SCORE_BUILDER) << "loadFromFile() file: " << filePath;
std::fstream filestream;
size_t version = 0;
try {
iarchive(cereal::make_nvp("version", version));
filestream.open(filePath, std::fstream::in);
if (filestream.is_open() == false) {
dbgTrace(D_WAAP_SCORE_BUILDER) << "failed to open file: " << filePath << " Error: " << errno;
return;
}
dbgTrace(D_WAAP_SCORE_BUILDER) << "loading from file: " << filePath;
int length;
filestream.seekg(0, std::ios::end); // go to the end
length = filestream.tellg(); // report location (this is the length)
dbgTrace(D_WAAP_SCORE_BUILDER) << "file length: " << length;
assert(length >= 0); // length -1 really happens if filePath is a directory (!)
char* buffer = new char[length]; // allocate memory for a buffer of appropriate dimension
filestream.seekg(0, std::ios::beg); // go back to the beginning
if (!filestream.read(buffer, length)) // read the whole file into the buffer
{
filestream.close();
delete[] buffer;
dbgWarning(D_WAAP_SCORE_BUILDER) << "Failed to read file, file: " << filePath;
return;
}
filestream.close();
std::stringstream ss(std::string(buffer, length));
delete[] buffer;
try
{
cereal::JSONInputArchive iarchive(ss);
iarchive(
cereal::make_nvp("waap_scores", m_serializedData)
);
}
catch (std::runtime_error & e) {
iarchive.setNextName(nullptr);
version = 0;
dbgDebug(D_WAAP_SCORE_BUILDER) << "ScoreBuilder version absent, using version " << version <<
" e.what() is " << e.what();
}
dbgDebug(D_WAAP_SCORE_BUILDER) << "Loading scores from file version " << version << "...";
switch (version)
{
case 1: {
iarchive(cereal::make_nvp("scorePools", m_keywordsScorePools));
break;
}
case 0: {
m_keywordsScorePools[KEYWORDS_SCORE_POOL_BASE] = KeywordsScorePool(iarchive);
break;
}
default: {
dbgDebug(D_WAAP_SCORE_BUILDER) << "Unknown scores file version: " << version;
}
dbgWarning(D_WAAP_SCORE_BUILDER) << "failed to deserialize file: " << filePath << ", error: " <<
e.what();
}
}
@@ -279,7 +287,6 @@ void ScoreBuilder::pumpKeywordScore(ScoreBuilderData& data, const std::string &p
{
m_pWaapAssetState->updateScores();
}
backupWorker();
}
}

40
components/security_apps/waap/waap_clib/ScoreBuilder.h
View File

@@ -24,6 +24,7 @@
#include <cereal/archives/json.hpp>
#include <cereal/types/unordered_map.hpp>
#include <cereal/types/string.hpp>
#include "WaapDefines.h"
struct ScoreBuilderData {
std::string m_sourceIdentifier;
@@ -131,7 +132,7 @@ struct KeywordsScorePool {
void mergeScores(const KeywordsScorePool& baseScores);
};
class ScoreBuilder : public SerializeToFilePeriodically {
class ScoreBuilder {
public:
ScoreBuilder(I_WaapAssetState* pWaapAssetState);
ScoreBuilder(I_WaapAssetState* pWaapAssetState, ScoreBuilder& baseScores);
@@ -152,13 +153,43 @@ public:
keywords_set getUaItemKeywordsSet(std::string userAgent);
unsigned int getFpStoreCount();
virtual void serialize(std::ostream& stream);
virtual void deserialize(std::istream& stream);
void restore();
void mergeScores(const ScoreBuilder& baseScores);
protected:
typedef std::map<std::string, double> KeywordScoreMap;
struct SerializedData {
template <class Archive>
void serialize(Archive& ar) {
size_t version = 0;
try {
ar(cereal::make_nvp("version", version));
}
catch (std::runtime_error & e) {
ar.setNextName(nullptr);
version = 0;
}
switch (version)
{
case 1: {
ar(cereal::make_nvp("scorePools", m_keywordsScorePools));
break;
}
case 0: {
m_keywordsScorePools[KEYWORDS_SCORE_POOL_BASE] = KeywordsScorePool(ar);
break;
}
default: {
break;
}
}
}
std::map<std::string, KeywordsScorePool> m_keywordsScorePools; // live data continuously updated during traffic
};
void pumpKeywordScorePerKeyword(ScoreBuilderData& data,
const std::string& keyword,
KeywordType keywordSource,
@@ -166,7 +197,8 @@ protected:
unsigned int m_scoreTrigger;
FalsePoisitiveStore m_fpStore;
std::map<std::string, KeywordsScorePool> m_keywordsScorePools; // live data continuously updated during traffic
SerializedData m_serializedData;
std::map<std::string, KeywordsScorePool> &m_keywordsScorePools; // live data continuously updated during traffic
std::map<std::string, KeywordScoreMap> m_snapshotKwScoreMap; // the snapshot is updated only by a call to snap()
std::list<std::string> m_falsePositivesSetsIntersection;
I_WaapAssetState* m_pWaapAssetState;

12
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -53,7 +53,8 @@ isGZipped(const std::string &stream)
bool RestGetFile::loadJson(const std::string& json)
{
std::string json_str = json;
std::string json_str = json;
if (isGZipped(json_str) == 0)
{
return ClientRest::loadJson(json_str);
@@ -94,6 +95,7 @@ Maybe<std::string> RestGetFile::genJson() const
return genError("Failed to compress data");
}
data = std::string((const char *)res.output, res.num_output_bytes);
json = data;
if (res.output) free(res.output);
@@ -175,6 +177,7 @@ void SerializeToFileBase::saveData()
}
serialize(ss);
filestream << ss.str();
filestream.close();
}
@@ -235,6 +238,7 @@ void SerializeToFileBase::loadFromFile(std::string filePath)
delete[] buffer;
std::stringstream ss(dataObfuscated);
try
@@ -252,12 +256,6 @@ void SerializeToFileBase::restore()
loadFromFile(m_filePath);
}
void SerializeToFileBase::setFilePath(const std::string& new_file_path)
{
m_filePath = new_file_path;
}
RemoteFilesList::RemoteFilesList() : files(), filesPathsList()
{

14
components/security_apps/waap/waap_clib/Signatures.cc
View File

@@ -12,7 +12,6 @@
// limitations under the License.
#include "Signatures.h"
#include "i_encryptor.h"
#include "waap.h"
#include <fstream>
@@ -242,13 +241,13 @@ bool Signatures::fail()
return error;
}
picojson::value::object Signatures::loadSource(const std::string& sigsFname)
picojson::value::object Signatures::loadSource(const std::string& waapDataFileName)
{
picojson::value doc;
std::ifstream f(sigsFname.c_str());
std::ifstream f(waapDataFileName);
if (f.fail()) {
dbgError(D_WAAP) << "Failed to open json data file '" << sigsFname << "'!";
dbgError(D_WAAP) << "Failed to open json data file '" << waapDataFileName << "'!";
error = true; // flag an error
return picojson::value::object();
}
@@ -264,15 +263,18 @@ picojson::value::object Signatures::loadSource(const std::string& sigsFname)
std::string dataObfuscated(buffer, length);
delete[] buffer;
std::stringstream ss(dataObfuscated);
ss >> doc;
if (!picojson::get_last_error().empty()) {
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << sigsFname << "') failed (parse error: '" <<
dbgError(D_WAAP) << "WaapAssetState::loadSource('" << waapDataFileName << "') failed (parse error: '" <<
picojson::get_last_error() << "').";
error = true; // flag an error
return picojson::value::object();
}
return doc.get<picojson::value::object>();
return doc.get<picojson::value::object>()["waap_signatures"].get<picojson::value::object>();
}

2
components/security_apps/waap/waap_clib/Signatures.h
View File

@@ -87,7 +87,7 @@ public:
const boost::regex wbxml_data_kw_filter;
private:
picojson::value::object loadSource(const std::string& sigsFname);
picojson::value::object loadSource(const std::string& waapDataFileName);
};
#endif

4
components/security_apps/waap/waap_clib/TypeIndicatorsFilter.cc
View File

@@ -36,8 +36,8 @@ TypeIndicatorFilter::TypeIndicatorFilter(I_WaapAssetState* pWaapAssetState,
size_t minIntervals,
std::chrono::minutes intervalDuration,
double ratioThreshold) :
IndicatorFilterBase(TYPES_FILTER_PATH(pWaapAssetState->getSignaturesFilterDir()),
TYPES_FILTER_TRUST_PATH(pWaapAssetState->getSignaturesFilterDir()),
IndicatorFilterBase(TYPES_FILTER_PATH(pWaapAssetState->getWaapDataDir()),
TYPES_FILTER_TRUST_PATH(pWaapAssetState->getWaapDataDir()),
(remotePath == "") ? remotePath : remotePath + "/Type",
assetId,
minSources,

163
components/security_apps/waap/waap_clib/WaapAssetState.cc
View File

@@ -75,21 +75,38 @@ static void print_found_patterns(const Waap::Util::map_of_stringlists_t& m) {
#endif
static bool err_hex = false;
static const std::string path_traversal_chars_regex = "[\\w.%?*\\/\\\\]";
static const std::string evasion_hex_regex_unallowed_prefix_helper =
"(?:(?<!(?<!0x|%u)[0-9a-f][0-9a-f])|(?<!(?<!%)[0-9a-f][0-9a-f]))";
static const std::string evasion_hex_regex_helper = "(0x[0-9a-f][0-9a-f])";
static const SingleRegex evasion_hex_regex(
"(0x[0-9a-f][0-9a-f])[\\w.%?*\\/\\\\]|[\\w.%?*\\/\\\\](0x[0-9a-f][0-9a-f])",
evasion_hex_regex_unallowed_prefix_helper + evasion_hex_regex_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + evasion_hex_regex_unallowed_prefix_helper + evasion_hex_regex_helper,
err_hex,
"evasion_hex_regex");
static const boost::regex bad_hex_regex = boost::regex("%[cC]1%[19][cC]");
static const std::string bad_hex_regex_helper = "(%[cC]1%(([19][cC])|([pP][cC])|(8[sS])))";
static const boost::regex bad_hex_regex(bad_hex_regex_helper);
static const SingleRegex evasion_bad_hex_regex(
"(%[cC]1%[19][cC])[\\w.%?*\\/\\\\]|[\\w.%?*\\/\\\\](%[cC]1%[19][cC])",
bad_hex_regex_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + bad_hex_regex_helper,
err_hex,
"evasion_bad_hex_regex");
static const std::string utf_evasion_for_dot_helper =
"(%[cC]0%[562aAfFeE][eE])";
static const SingleRegex utf_evasion_for_dot(
utf_evasion_for_dot_helper + path_traversal_chars_regex +
"|" + path_traversal_chars_regex + utf_evasion_for_dot_helper,
err_hex,
"utf_evasion_for_dot");
static const boost::regex utf_evasion_for_dot_regex(utf_evasion_for_dot_helper);
static const std::string sqli_comma_evasion_regex_helper = "\"\\s*,\\s*\"";
static const boost::regex sqli_comma_evasion_regex(sqli_comma_evasion_regex_helper);
WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState,
const std::string& sigScoresFname,
const std::string& waapDataFileName,
const std::string& id) :
WaapAssetState(pWaapAssetState->m_Signatures,
sigScoresFname,
waapDataFileName,
pWaapAssetState->m_cleanValuesCache.capacity(),
pWaapAssetState->m_suspiciousValuesCache.capacity(),
pWaapAssetState->m_sampleTypeCache.capacity(),
@@ -110,20 +127,22 @@ WaapAssetState::WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAsset
}
WaapAssetState::WaapAssetState(std::shared_ptr<Signatures> signatures,
const std::string& sigScoresFname,
const std::string& waapDataFileName,
size_t cleanValuesCacheCapacity,
size_t suspiciousValuesCacheCapacity,
size_t sampleTypeCacheCapacity,
const std::string& assetId) :
m_Signatures(signatures),
m_SignaturesScoresFilePath(sigScoresFname),
m_waapDataFileName(waapDataFileName),
m_assetId(assetId),
scoreBuilder(this),
m_rateLimitingState(nullptr),
m_errorLimitingState(nullptr),
m_securityHeadersState(nullptr),
m_filtersMngr(nullptr),
m_typeValidator(getSignaturesFilterDir() + "/8.data"),
m_typeValidator(getWaapDataDir() + "/waap.data"),
m_cleanValuesCache(cleanValuesCacheCapacity),
m_suspiciousValuesCache(suspiciousValuesCacheCapacity),
m_sampleTypeCache(sampleTypeCacheCapacity)
@@ -1039,7 +1058,7 @@ WaapAssetState::apply(
if (found != std::string::npos)
{
unescaped += res.unescaped_line.substr(pos, found-pos);
if (found < res.unescaped_line.size() - 3 &&
if (found + 3 < res.unescaped_line.size() &&
res.unescaped_line[found+1] == res.unescaped_line[found+2] && res.unescaped_line[found+3] == ']')
{
unescaped += res.unescaped_line[found+1];
@@ -1145,16 +1164,11 @@ WaapAssetState::apply(
}
}
std::string *utf8_broken_line_ptr = nullptr;
if (Waap::Util::containsBrokenUtf8(unquote_line)) {
utf8_broken_line_ptr = &unquote_line;
} else if (Waap::Util::containsBrokenUtf8(line)) {
utf8_broken_line_ptr = (std::string*)&line;
}
Maybe<std::string> broken_utf8_line = Waap::Util::containsBrokenUtf8(line, unquote_line);
if (utf8_broken_line_ptr) {
if (broken_utf8_line.ok()) {
dbgTrace(D_WAAP_EVASIONS) << "broken-down utf-8 evasion found";
std::string unescaped = Waap::Util::unescapeBrokenUtf8(*utf8_broken_line_ptr);
std::string unescaped = Waap::Util::unescapeBrokenUtf8(broken_utf8_line.unpack());
size_t kwCount = res.keyword_matches.size();
unescaped = unescape(unescaped);
@@ -1283,6 +1297,35 @@ WaapAssetState::apply(
}
}
boost::cmatch what;
if (boost::regex_search(res.unescaped_line.c_str(), what, sqli_comma_evasion_regex)) {
// Possible SQLi evasion detected (","): - clean up and scan with regexes again.
dbgTrace(D_WAAP_EVASIONS) << "Possible SQLi evasion detected (\",\"): - clean up and scan with regexes again.";
std::string unescaped = res.unescaped_line;
unescaped = boost::regex_replace(unescaped, sqli_comma_evasion_regex, "");
unescaped = unescape(unescaped);
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
if ((res.unescaped_line.find("0x") != std::string::npos) && evasion_hex_regex.hasMatch(res.unescaped_line)) {
dbgTrace(D_WAAP_EVASIONS) << "hex evasion found (in unescaped line)";
@@ -1306,9 +1349,10 @@ WaapAssetState::apply(
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
for (const auto &kw : res.keyword_matches) {
if (kw.size() < 2 || str_contains(kw, "os_cmd_high_acuracy_fast_reg") ||
kw == "os_cmd_sep_medium_acuracy" || str_contains(kw, "regex_code_execution") ||
str_contains(kw, "regex_code_execution") || kw == "character_encoding" ||
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
{
keywordsToRemove.push_back(kw);
}
@@ -1333,7 +1377,7 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, false, binaryDataFound);
@@ -1346,9 +1390,10 @@ WaapAssetState::apply(
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
for (const auto &kw : res.keyword_matches) {
if (kw.size() < 2 || str_contains(kw, "os_cmd_high_acuracy_fast_reg") ||
kw == "os_cmd_sep_medium_acuracy" || str_contains(kw, "regex_code_execution") ||
str_contains(kw, "regex_code_execution") || kw == "character_encoding" ||
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
str_contains(kw, "quotes_ev_fast_reg") || str_contains(kw, "encoded_") ||
str_contains(kw, "medium_acuracy") || str_contains(kw, "high_acuracy_fast_reg_xss"))
{
keywordsToRemove.push_back(kw);
}
@@ -1405,6 +1450,37 @@ WaapAssetState::apply(
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
if ((res.unescaped_line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(res.unescaped_line)) {
dbgTrace(D_WAAP_EVASIONS) <<
"UTF evasion for dot found (%c0%*e) in unescaped line";
std::string unescaped = res.unescaped_line;
unescaped = boost::regex_replace(unescaped, utf_evasion_for_dot_regex, ".");
unescaped = unescape(unescaped);
dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'";
size_t kwCount = res.keyword_matches.size();
if (res.unescaped_line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
@@ -1425,6 +1501,38 @@ WaapAssetState::apply(
}
}
if ((line.find("%") != std::string::npos) && utf_evasion_for_dot.hasMatch(line)) {
dbgTrace(D_WAAP_EVASIONS) << "UTF evasion for dot found (%c0%*e) in raw line";
std::string unescaped = line;
unescaped = boost::regex_replace(unescaped, utf_evasion_for_dot_regex, ".");
unescaped = unescape(unescaped);
dbgTrace(D_WAAP_EVASIONS) << "unescaped == '" << unescaped << "'";
size_t kwCount = res.keyword_matches.size();
if (line != unescaped) {
SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions);
checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches,
res.found_patterns, longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns,
longTextFound, binaryDataFound);
checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns,
longTextFound, binaryDataFound);
}
if (kwCount != res.keyword_matches.size() && !binaryDataFound) {
// Recalculate repetition and/or probing indicators
unsigned int newWordsCount = 0;
calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing,
newWordsCount);
// Take minimal words count because empirically it means evasion was probably succesfully decoded
wordsCount = std::min(wordsCount, newWordsCount);
}
}
// python: escape ='hi_acur_fast_reg_evasion' in found_patterns
bool escape = Waap::Util::find_in_map_of_stringlists_keys("evasion", res.found_patterns);
@@ -1690,8 +1798,8 @@ void WaapAssetState::updateScores()
scoreBuilder.snap();
}
std::string WaapAssetState::getSignaturesScoresFilePath() const {
return m_SignaturesScoresFilePath;
std::string WaapAssetState::getWaapDataFileName() const {
return m_waapDataFileName;
}
std::map<std::string, std::vector<std::string>>& WaapAssetState::getFilterVerbose()
@@ -1699,10 +1807,10 @@ std::map<std::string, std::vector<std::string>>& WaapAssetState::getFilterVerbos
return m_filtered_keywords_verbose;
}
std::string WaapAssetState::getSignaturesFilterDir() const {
size_t lastSlash = m_SignaturesScoresFilePath.find_last_of('/');
std::string WaapAssetState::getWaapDataDir() const {
size_t lastSlash = m_waapDataFileName.find_last_of('/');
std::string sigsFilterDir = ((lastSlash == std::string::npos) ?
m_SignaturesScoresFilePath : m_SignaturesScoresFilePath.substr(0, lastSlash));
m_waapDataFileName : m_waapDataFileName.substr(0, lastSlash));
dbgTrace(D_WAAP_ASSET_STATE) << " signatures filters directory: " << sigsFilterDir;
return sigsFilterDir;
}
@@ -1948,6 +2056,7 @@ std::shared_ptr<Waap::SecurityHeaders::State>& WaapAssetState::getSecurityHeader
return m_securityHeadersState;
}
void WaapAssetState::clearRateLimitingState()
{
m_rateLimitingState.reset();

14
components/security_apps/waap/waap_clib/WaapAssetState.h
View File

@@ -24,7 +24,6 @@
#include <set>
#include <boost/noncopyable.hpp>
#include "ScoreBuilder.h"
#include "i_encryptor.h"
#include "i_waap_asset_state.h"
#include "RateLimiting.h"
#include "SecurityHeadersPolicy.h"
@@ -41,7 +40,7 @@ class WaapAssetState : public boost::noncopyable, public I_WaapAssetState
{
private: //ugly but needed for build
std::shared_ptr<Signatures> m_Signatures;
std::string m_SignaturesScoresFilePath;
std::string m_waapDataFileName;
std::map<std::string, std::vector<std::string>> m_filtered_keywords_verbose;
void checkRegex(const SampleValue &sample, const Regex & pattern, std::vector<std::string>& keyword_matches,
@@ -51,13 +50,13 @@ private: //ugly but needed for build
public:
// Load and compile signatures from file
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& sigScoresFname,
explicit WaapAssetState(std::shared_ptr<Signatures> signatures, const std::string& waapDataFileName,
size_t cleanCacheCapacity = SIGS_APPLY_CLEAN_CACHE_CAPACITY,
size_t suspiciousCacheCapacity = SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY,
size_t sampleTypeCacheCapacity = SIGS_SAMPLE_TYPE_CACHE_CAPACITY,
const std::string& assetId = "");
explicit WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState, const std::string& sigScoresFname,
const std::string& assetId);
explicit WaapAssetState(const std::shared_ptr<WaapAssetState>& pWaapAssetState,
const std::string& waapDataFileName, const std::string& assetId);
virtual ~WaapAssetState();
std::shared_ptr<Signatures> getSignatures() const;
@@ -76,8 +75,8 @@ public:
const Maybe<std::string> splitType=genError("not splitted")) const;
virtual void updateScores();
virtual std::string getSignaturesScoresFilePath() const;
virtual std::string getSignaturesFilterDir() const;
virtual std::string getWaapDataFileName() const;
virtual std::string getWaapDataDir() const;
std::map<std::string, std::vector<std::string>>& getFilterVerbose();
void updateFilterManagerPolicy(IWaapConfig* pConfig);
@@ -105,6 +104,7 @@ public:
void clearErrorLimitingState();
void clearSecurityHeadersState();
std::shared_ptr<Waap::RateLimiting::State>& getRateLimitingState();
std::shared_ptr<Waap::RateLimiting::State>& getErrorLimitingState();
std::shared_ptr<Waap::SecurityHeaders::State>& getSecurityHeadersState();

16
components/security_apps/waap/waap_clib/WaapAssetStatesManager.cc
View File

@@ -33,9 +33,9 @@ void WaapAssetStatesManager::preload()
registerExpectedConfiguration<std::string>("waap data", "base folder");
}
bool WaapAssetStatesManager::initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname)
bool WaapAssetStatesManager::initBasicWaapSigs(const std::string& waapDataFileName)
{
return pimpl->initBasicWaapSigs(sigsFname, sigScoresFname);
return pimpl->initBasicWaapSigs(waapDataFileName);
}
std::shared_ptr<WaapAssetState> WaapAssetStatesManager::getWaapAssetStateGlobal()
@@ -65,7 +65,7 @@ WaapAssetStatesManager::Impl::~Impl()
{
}
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname)
bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& waapDataFileName)
{
if (m_signatures && !m_signatures->fail() && m_basicWaapSigs)
{
@@ -73,18 +73,18 @@ bool WaapAssetStatesManager::Impl::initBasicWaapSigs(const std::string& sigsFnam
return true;
}
try {
m_signatures = std::make_shared<Signatures>(sigsFname);
m_signatures = std::make_shared<Signatures>(waapDataFileName);
m_basicWaapSigs = std::make_shared<WaapAssetState>(
m_signatures,
sigScoresFname,
waapDataFileName,
SIGS_APPLY_CLEAN_CACHE_CAPACITY,
SIGS_APPLY_SUSPICIOUS_CACHE_CAPACITY);
}
catch (std::runtime_error & e) {
// TODO:: properly handle component initialization failure
dbgTrace(D_WAAP) <<
"WaapAssetStatesManager::initBasicWaapSigs(): " << e.what() << ". Failed to read signature files"
" "<< sigsFname << " and " << sigScoresFname << ".";
"WaapAssetStatesManager::initBasicWaapSigs(): " << e.what() << ". Failed to read data file '" <<
waapDataFileName << "'";
m_basicWaapSigs.reset();
return false;
}
@@ -178,7 +178,7 @@ WaapAssetStatesManager::Impl::CreateWaapSigsForAsset(const std::shared_ptr<WaapA
}
std::string basePath = pWaapAssetState->getSignaturesScoresFilePath();
std::string basePath = pWaapAssetState->getWaapDataFileName();
size_t lastSlash = basePath.find_last_of('/');
std::string assetScoresPath = assetPath +
((lastSlash == std::string::npos) ? basePath : basePath.substr(lastSlash));

6
components/security_apps/waap/waap_clib/WaapAssetStatesManager.h
View File

@@ -24,7 +24,7 @@ class WaapAssetState;
class I_WaapAssetStatesManager {
public:
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname) = 0;
virtual bool initBasicWaapSigs(const std::string& waapDataFileName) = 0;
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal() = 0;
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId) = 0;
virtual void setAssetDirectoryPath(const std::string &assetDirectoryPath) = 0;
@@ -36,7 +36,7 @@ public:
virtual ~WaapAssetStatesManager();
void preload();
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname);
virtual bool initBasicWaapSigs(const std::string& waapDataFileName);
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal();
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId);
@@ -53,7 +53,7 @@ public:
Impl();
virtual ~Impl();
virtual bool initBasicWaapSigs(const std::string& sigsFname, const std::string& sigScoresFname);
virtual bool initBasicWaapSigs(const std::string& waapDataFileName);
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateGlobal();
virtual std::shared_ptr<WaapAssetState> getWaapAssetStateById(const std::string& assetId);
virtual void setAssetDirectoryPath(const std::string &assetDirectoryPath);

5
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@@ -66,7 +66,7 @@ WaapConfigAPI::WaapConfigAPI(
string practiceName,
string ruleId,
string ruleName,
bool schemaValidation) :
bool schemaValidation) :
WaapConfigBase(
autonomousSecurity,
autonomousSecurityLevel,
@@ -84,8 +84,10 @@ WaapConfigAPI::WaapConfigAPI(
void WaapConfigAPI::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
readJSONByCereal(ar);
WaapConfigBase::load(ar);
assets_ids_aggregation.insert(m_assetId);
}
@@ -94,6 +96,7 @@ void WaapConfigAPI::readJSONByCereal(cereal::JSONInputArchive &ar)
{
}
bool WaapConfigAPI::operator==(const WaapConfigAPI& other) const
{
const WaapConfigBase* configBase = this;

2
components/security_apps/waap/waap_clib/WaapConfigApi.h
View File

@@ -18,6 +18,7 @@
#include <set>
#include "WaapConfigBase.h"
#include "log_generator.h"
#include "debug.h"
@@ -50,6 +51,7 @@ public:
private:
void readJSONByCereal(cereal::JSONInputArchive&ar);
std::string m_schemaValidationPoicyStatusMessage;
static const std::string s_PracticeSubType;

4
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -79,6 +79,7 @@ void WaapConfigBase::readJSONByCereal(cereal::JSONInputArchive& ar)
m_blockingLevel = blockingLevelBySensitivityStr(m_autonomousSecurityLevel);
}
void WaapConfigBase::loadCsrfPolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the CSRF policy of the current rule: " +
@@ -240,6 +241,7 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar)
}
}
void WaapConfigBase::loadErrorDisclosurePolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the WAAP Information Disclosure policy";
@@ -386,6 +388,7 @@ const std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter>& WaapConfig
return m_trustedSourcesPolicy;
}
const std::shared_ptr<Waap::Csrf::Policy>& WaapConfigBase::get_CsrfPolicy() const
{
return m_csrfPolicy;
@@ -411,6 +414,7 @@ const std::shared_ptr<Waap::OpenRedirect::Policy>& WaapConfigBase::get_OpenRedir
return m_openRedirectPolicy;
}
const std::shared_ptr<Waap::ErrorDisclosure::Policy>& WaapConfigBase::get_ErrorDisclosurePolicy() const
{
return m_errorDisclosurePolicy;

1
components/security_apps/waap/waap_clib/WaapDecision.cc
View File

@@ -13,6 +13,7 @@
#include "WaapDecision.h"
#include "OpenRedirectDecision.h"
#include "debug.h"
#include <algorithm>
#include <type_traits>

1
components/security_apps/waap/waap_clib/WaapDecision.h
View File

@@ -24,6 +24,7 @@
#include "AutonomousSecurityDecision.h"
#include <iterator>
std::ostream& operator<<(std::ostream& os, const std::list<std::shared_ptr<SingleDecision>>& decisions);
std::ostream& operator<<(std::ostream& os, const DecisionsArr& decisions);
typedef std::list<std::shared_ptr<SingleDecision>> decision_list;

4
components/security_apps/waap/waap_clib/WaapErrorDisclosurePolicy.h
View File

@@ -28,6 +28,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("errorDisclosure", level));
level = boost::algorithm::to_lower_copy(level);

4
components/security_apps/waap/waap_clib/WaapOpenRedirectPolicy.h
View File

@@ -30,6 +30,10 @@ struct Policy {
enable(false),
enforce(false)
{
bool web_attack_on;
ar(cereal::make_nvp("webAttackMitigation", web_attack_on));
if (!web_attack_on) return;
std::string level;
ar(cereal::make_nvp("openRedirect", level));
level = boost::algorithm::to_lower_copy(level);

3
components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc
View File

@@ -96,6 +96,9 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
}
return false;
}
else if (tag == "paramlocation" || tag == "paramLocation") {
return NGEN::Regex::regexMatch(__FILE__, __LINE__, waf2Transaction.getLocation().c_str(), what, rx);
}
}
catch (std::runtime_error & e) {
dbgDebug(D_WAAP_OVERRIDE) << "RegEx match for tag " << tag << " failed due to: " << e.what();

32
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -105,7 +105,22 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
return Waap::Scores::calcArrayScore(res.scoreArray);
}
bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, const std::string& location, const std::string& param_name) {
// Ignore scan results from specific fields on csp-report json in case those are not filtered by learning
bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp)
{
if (res.score < 8.0f && res.location == "body" && dp.getLastParser() == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
return true;
}
}
return false;
}
bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
const std::string& location, const std::string& param_name, const std::string& key)
{
dbgTrace(D_WAAP_SCANNER) << "suspiciousHit processing for parameter: " << param_name << " at " << location <<
" num of keywords " << res.keyword_matches.size();
@@ -130,7 +145,7 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, const std::string& locati
);
}
if (m_transaction->shouldIgnoreOverride(res)) {
if (isKeyCspReport(key, res, dp) || m_transaction->shouldIgnoreOverride(res)) {
dbgTrace(D_WAAP_SCANNER) << "Ignoring parameter key/value " << res.param_name <<
" due to ignore action in override";
m_bIgnoreOverride = true;
@@ -182,6 +197,15 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: skip scanning our own anti-bot cookie, by name";
return 0;
}
// Do not scan google analytics cookie
if (isCookiePayload &&
(fullKeyStr.find("_ga") != std::string::npos ||
fullKeyStr.find("_gid") != std::string::npos ||
fullKeyStr.find("_gat") != std::string::npos)) {
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: skip scanning google analytics cookie";
return 0;
}
// scan for csrf token.
if (isCookiePayload && fullKeyStr == "x-chkp-csrf-token") {
m_transaction->getCsrfState().set_CsrfToken(v, v_len);
@@ -263,7 +287,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
// Deep-scan parameter names
if (m_transaction->getAssetState()->apply(key, res, dp.m_key.first())) {
if (suspiciousHit(res, dp.m_key.first(), dp.m_key.str())) {
if (suspiciousHit(res, dp, dp.m_key.first(), dp.m_key.str(), key)) {
// Scanner found enough evidence to report this res
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: SUSPICIOUS PARAM NAME: k='" <<
key << "' v='" << value << "'";
@@ -295,7 +319,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
res.mergeFrom(param_name_res);
}
if (suspiciousHit(res, dp.m_key.first(), dp.m_key.str())) {
if (suspiciousHit(res, dp, dp.m_key.first(), dp.m_key.str(), key)) {
// Scanner found enough evidence to report this res
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: SUSPICIOUS VALUE: k='" << key <<
"' v='" << value << "'";

4
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -32,7 +32,8 @@ namespace Waap {
m_bIgnoreOverride(false)
{
}
bool suspiciousHit(Waf2ScanResult &res, const std::string &location, const std::string &param_name);
bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp,
const std::string &location, const std::string &param_name, const std::string &key);
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) override;
const std::string &getAntibotCookie() const { return m_antibotCookie; }
@@ -41,6 +42,7 @@ namespace Waap {
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
bool shouldIgnoreOverride(const Waf2ScanResult &res);
bool isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp);
Waf2ScanResult m_lastScanResult;
IWaf2Transaction *m_transaction;

8
components/security_apps/waap/waap_clib/WaapScores.cc
View File

@@ -68,6 +68,7 @@ calcCombinations(
std::vector<std::string>& keyword_combinations)
{
keyword_combinations.clear();
static const double max_combi_score = 1.0f;
for (size_t i = 0; i < keyword_matches.size(); ++i) {
std::vector<std::string> combinations;
@@ -79,6 +80,8 @@ calcCombinations(
// from signature_scores database.
std::sort(combinations.begin(), combinations.end());
std::string combination;
double default_score = 0.0f;
// note that std::set<> container output sorted data when iterated.
for (auto it = combinations.begin(); it != combinations.end(); it++) {
// add space between all items, except the first one
@@ -86,8 +89,11 @@ calcCombinations(
combination += " ";
}
combination += *it;
default_score += scoreBuilder.getSnapshotKeywordScore(*it, 0.0f, poolName);
}
addKeywordScore(scoreBuilder, poolName, combination, 1.0f, scoresArray);
// set default combination score to be the sum of its keywords, bounded by 1
default_score = std::min(default_score, max_combi_score);
addKeywordScore(scoreBuilder, poolName, combination, default_score, scoresArray);
keyword_combinations.push_back(combination);
}
}

81
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -129,7 +129,7 @@ void Waf2Transaction::add_response_hdr(const char* name, int name_len, const cha
dbgTrace(D_WAAP) << "[transaction:" << this << "] add_response_hdr(name='" << std::string(name, name_len) <<
"', value='" << std::string(value, value_len) << "')";
// Detect location header and remember its value
// Detect location header and remember it's value
static const char location[] = "location";
auto openRedirectPolicy = m_siteConfig ? m_siteConfig->get_OpenRedirectPolicy() : NULL;
@@ -260,6 +260,7 @@ void Waf2Transaction::end_response()
dbgTrace(D_WAAP) << "[transaction:" << this << "] end_response";
}
void Waf2Transaction::setCurrentAssetState(IWaapConfig* sitePolicy)
{
I_WaapAssetStatesManager* pWaapAssetStatesManager =
@@ -593,10 +594,14 @@ bool Waf2Transaction::setCurrentAssetContext()
return result;
}
void Waf2Transaction::processUri(const char* uri, const std::string& scanStage) {
void Waf2Transaction::processUri(const std::string &uri, const std::string& scanStage) {
m_processedUri = true;
const char* p = uri;
size_t uriSize = uri.length();
const char* p = uri.c_str();
const char* uriEnd = p+uriSize;
std::string baseUri;
char querySep = '?';
char paramSep = '&';
// TODO:: refactor out this block to method, and the next block (parsing url parameters), too.
{
@@ -606,27 +611,40 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// Parse URL
ParserRaw urlParser(m_deepParserReceiver, scanStage);
// Scan the uri until '?' character found (or until end of the uri string).
// Scan the uri until '?' or ';' character found, whichever comes first (or until end of the uri string),
// Do not account for last character as valid separator
do {
const char* q = strchr(p, '?');
const char* q = strpbrk(p, "?;");
if (q == NULL) {
// Handle special case found in customer traffic where instead of '?' there was ';' character.
q = strchr(p, ';');
if (q) {
if (q != NULL && q < uriEnd-1) {
querySep = *q;
// Handle special case found in customer traffic where instead of '?' there was a ';' character.
if (querySep == ';') {
// Check that after ';' the parameter name is valid and terminated with '='. This would normally be
// the case in legit traffic, but not in attacks. This covers a case of "sap login".
const char *qq;
for (qq = q + 1; isalpha(*qq) || isdigit(*qq) || *qq=='-' || *qq=='_' || *qq=='*'; ++qq);
for (qq = q + 1;
qq < uriEnd && (isalpha(*qq) || isdigit(*qq) || *qq=='-' || *qq=='_' || *qq=='*');
++qq);
if (*qq != '=') {
// Assume it might be attack and cancel the separation by the ';' character (scan whole URL)
q = NULL;
}
else {
const char *qqSep = strpbrk(qq, "&;");
// Handle special case (deprecated standard) where instead of '&' there was a ';' separator,
// Do not account for last character as valid separator
if (qqSep && qqSep < uriEnd-1) {
paramSep = *qqSep;
}
}
}
}
if (q == NULL) {
baseUri = std::string(p);
dbgTrace(D_WAAP) << "Query separator not found, use entire uri as baseUri";
baseUri = std::string(uri);
if (scanStage == "url") {
m_uriPath = baseUri;
}
@@ -637,7 +655,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// Push the last piece to URL scanner
pushed = true;
std::string url(p, strlen(p));
std::string url(uri);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
@@ -670,7 +688,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// parameters from the character next to '?'
p = q + 1;
break;
} while (1);
} while (p && p < uriEnd);
if (pushed) {
urlParser.finish();
@@ -721,7 +739,7 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
// at this point, p can either be NULL (if there are no URL parameters),
// or point to the parameters string (right after the '?' character)
if (p && *p) {
if (p && p < uriEnd && *p) {
// Decode URLEncoded data and send decoded key/value pairs to deep inspection
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the " << scanStage.c_str() << " parameters";
@@ -729,10 +747,12 @@ void Waf2Transaction::processUri(const char* uri, const std::string& scanStage)
m_uriQuery = std::string(p);
}
dbgTrace(D_WAAP) << "Query separator='" << querySep << "', " << "Param separator='" << paramSep << "'";
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = strlen(p);
ParserUrlEncode up(m_deepParserReceiver, '&', checkUrlEncoded(p, buff_len));
size_t buff_len = uriEnd - p;
ParserUrlEncode up(m_deepParserReceiver, paramSep, checkUrlEncoded(p, buff_len));
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@@ -807,7 +827,7 @@ void Waf2Transaction::parseReferer(const char* value, int value_len)
// Parse referer value as if it was a URL
if (value_len > 0)
{
processUri(std::string(value, value_len).c_str(), "referer");
processUri(std::string(value, value_len), "referer");
}
}
@@ -1011,13 +1031,14 @@ void Waf2Transaction::end_request_hdrs() {
}
// Scan URL and url query
if (m_isScanningRequired && !m_processedUri) {
processUri(m_uriStr.c_str(), "url");
processUri(m_uriStr, "url");
}
// Scan relevant headers for attacks
if (m_isScanningRequired && !m_processedHeaders) {
scanHeaders();
}
if(m_siteConfig != NULL) {
// Create rate limiting policy (lazy, on first request)
const std::shared_ptr<Waap::RateLimiting::Policy> rateLimitingPolicy = m_siteConfig->get_RateLimitingPolicy();
@@ -1059,6 +1080,7 @@ void Waf2Transaction::start_request_body() {
clearRequestParserState();
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, "body");
m_request_body_bytes_received = 0;
@@ -1362,7 +1384,7 @@ Waf2Transaction::checkShouldInject()
{
dbgTrace(D_WAAP) << "Waf2Transaction::checkShouldInject(): Should not inject CSRF scripts.";
}
if(csrf) {
dbgTrace(D_WAAP) << "Waf2Transaction::checkShouldInject(): Should inject CSRF script";
m_responseInjectReasons.setCsrf(true);
@@ -1370,6 +1392,7 @@ Waf2Transaction::checkShouldInject()
return;
}
bool
Waf2Transaction::decideAfterHeaders()
{
@@ -1544,7 +1567,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
const auto& autonomousSecurityDecision = std::dynamic_pointer_cast<AutonomousSecurityDecision>(
m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION));
bool send_extended_log = shouldSendExtendedLog(triggerLog);
if (send_extended_log || triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webUrlPath || autonomousSecurityDecision->getOverridesLog()) {
std::string httpUriPath = m_uriPath;
if (httpUriPath.length() > MAX_LOG_FIELD_SIZE)
@@ -1554,7 +1577,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
waapLog << LogField("httpUriPath", httpUriPath, LogFieldOption::XORANDB64);
}
if (send_extended_log || triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webUrlQuery || autonomousSecurityDecision->getOverridesLog()) {
std::string uriQuery = m_uriQuery;
if (uriQuery.length() > MAX_LOG_FIELD_SIZE)
{
@@ -1562,7 +1585,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
waapLog << LogField("httpUriQuery", uriQuery, LogFieldOption::XORANDB64);
}
if (send_extended_log || triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) {
if (triggerLog->webHeaders || autonomousSecurityDecision->getOverridesLog()) {
waapLog << LogField("httpRequestHeaders", logHeadersStr(), LogFieldOption::XORANDB64);
}
// Log http response code if it is known
@@ -1627,6 +1650,7 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
}
}
void
Waf2Transaction::sendLog()
{
@@ -1840,6 +1864,7 @@ Waf2Transaction::sendLog()
appendCommonLogFields(waap_log, triggerLog, shouldBlock, logOverride, incidentType);
waap_log << LogField("waapIncidentDetails", incidentDetails);
if (grace_period) {
dbgTrace(D_WAAP)
@@ -1871,7 +1896,7 @@ Waf2Transaction::sendLog()
<< max_grace_logs;
}
break;
}
}
case AUTONOMOUS_SECURITY_DECISION: {
if (triggerLog->webRequests ||
send_extended_log ||
@@ -1929,7 +1954,7 @@ Waf2Transaction::decideAutonomousSecurity(
if (!m_processedUri) {
dbgWarning(D_WAAP) << "decideAutonomousSecurity(): processing URI although is was supposed "
"to be processed earlier ...";
processUri(m_uriStr.c_str(), "url");
processUri(m_uriStr, "url");
}
if (!m_processedHeaders) {
@@ -2020,6 +2045,8 @@ Waf2Transaction::decideAutonomousSecurity(
}
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO) {
decision->setLog(false);
} else {
@@ -2103,7 +2130,6 @@ bool Waf2Transaction::decideResponse()
<< "Setting flag for collection of respond content logging to: "
<< (should_send_extended_log ? "True": "False");
m_responseInspectReasons.setCollectResponseForLog(should_send_extended_log);
}
dbgTrace(D_WAAP) << "Waf2Transaction::decideResponse: returns true (accept)";
@@ -2126,8 +2152,6 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) {
bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
dbgTrace(D_WAAP) << "reading exceptions";
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) return false;
@@ -2147,6 +2171,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// collect param value
exceptions_dict["paramValue"].insert(res.unescaped_line);
// collect param location
exceptions_dict["paramLocation"].insert(res.location);
ScopedContext ctx;
ctx.registerValue<std::string>("paramValue", res.unescaped_line);
ctx.registerValue<std::set<std::string>>("paramName", param_name_set);

4
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -184,6 +184,7 @@ public:
void handleSecurityHeadersInjection(std::vector<std::pair<std::string, std::string>>& injectHeaderStrs);
void disableShouldInjectSecurityHeaders();
bool shouldSendExtendedLog(const std::shared_ptr<Waap::Trigger::Log> &trigger_log) const;
// query
@@ -236,6 +237,7 @@ private:
std::string getUserReputationStr(double relativeReputation) const;
bool isTrustedSource() const;
void setCurrentAssetState(IWaapConfig* sitePolicy);
bool setCurrentAssetContext();
bool checkIsScanningRequired();
@@ -254,7 +256,7 @@ private:
size_t getViolatingUserLimitSize() const;
// Internal
void processUri(const char *uri, const std::string &scanStage);
void processUri(const std::string &uri, const std::string &scanStage);
void parseContentType(const char* value, int value_len);
void parseCookie(const char* value, int value_len);
void parseReferer(const char* value, int value_len);

2
components/security_apps/waap/waap_clib/Waf2EngineGetters.cc
View File

@@ -265,6 +265,8 @@ ParserBase* Waf2Transaction::getRequestBodyParser()
{
return m_requestBodyParser;
}
const std::string Waf2Transaction::getMethod() const
{
return m_methodStr;

28
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -1128,6 +1128,11 @@ static const SingleRegex broken_utf_evasion_re(
err,
"broken_utf_evasion"
);
static const SingleRegex csp_report_policy_re(
"default-src\\s+[^\\w]+.*report-uri\\s+[^\\w]+",
err,
"csp_report_policy"
);
static void b64TestChunk(const string &s,
string::const_iterator chunkStart,
@@ -1391,10 +1396,16 @@ unescapeInvalidUtf8(const string &payload)
return unescaped_text;
}
bool
containsBrokenUtf8(const string &payload)
Maybe<std::string>
containsBrokenUtf8(const string &payload, const string &unquoted_payload)
{
return broken_utf_evasion_re.hasMatch(payload);
if (broken_utf_evasion_re.hasMatch(unquoted_payload)) {
return unquoted_payload;
} else if (broken_utf_evasion_re.hasMatch(payload)) {
return payload;
} else {
return genError("does not contain broken-down UTF8");
}
}
string
@@ -1426,6 +1437,12 @@ unescapeBrokenUtf8(const string &payload)
return unescaped_text;
}
bool
containsCspReportPolicy(const string &payload)
{
return csp_report_policy_re.hasMatch(payload);
}
string
charToString(const char* s, int slen)
{
@@ -1825,6 +1842,11 @@ ContentType detectContentType(const char* hdr_value) {
return CONTENT_TYPE_JSON;
}
// Detect Graphql content type if Content-Type header value is application/graphql
if (my_stristarts_with(hdr_value, "application/graphql")) {
return CONTENT_TYPE_GQL;
}
// Detect HTML content type
if (my_stristarts_with(hdr_value, "text/html")) {
return CONTENT_TYPE_HTML;

12
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -757,7 +757,8 @@ inline char convertFromUnicodeHalfAndFullWidthRange(uint32_t code) {
inline bool isSpecialUnicode(uint32_t code) {
return isUnicodeHalfAndFullWidthRange(code)
|| 0x2028 == code || 0x2029 == code
|| 0x2216 == code || 0xEFC8 == code || 0xF025 == code;
|| 0x2215 == code || 0x2216 == code
|| 0xEFC8 == code || 0xF025 == code;
}
inline char convertSpecialUnicode(uint32_t code) {
@@ -768,6 +769,10 @@ inline char convertSpecialUnicode(uint32_t code) {
{
return '\\';
}
else if (0x2215 == code)
{
return '/';
}
// assuming 0x2028 == code || 0x2029 == code
else
{
@@ -968,6 +973,7 @@ namespace Util {
CONTENT_TYPE_UNKNOWN,
CONTENT_TYPE_XML,
CONTENT_TYPE_JSON,
CONTENT_TYPE_GQL,
CONTENT_TYPE_HTML,
CONTENT_TYPE_MULTIPART_FORM,
CONTENT_TYPE_URLENCODED,
@@ -1037,9 +1043,11 @@ namespace Util {
// based on invalid utf-8 evasion from here: https://www.cgisecurity.com/lib/URLEmbeddedAttacks.html
std::string unescapeInvalidUtf8(const std::string &text);
bool containsBrokenUtf8(const std::string &payload);
Maybe<std::string> containsBrokenUtf8(const std::string &payload, const std::string &unquoted_payload);
std::string unescapeBrokenUtf8(const std::string &text);
bool containsCspReportPolicy(const std::string &payload);
bool testUrlBareUtf8Evasion(const std::string &line);
bool testUrlBadUtf8Evasion(const std::string &line);

1
components/security_apps/waap/waap_component.cc
View File

@@ -52,7 +52,6 @@ WaapComponent::preload()
// TODO:: call stuff like registerExpectedCofiguration here..
registerExpectedConfiguration<WaapConfigApplication>("WAAP", "WebApplicationSecurity");
registerExpectedConfiguration<WaapConfigAPI>("WAAP", "WebAPISecurity");
registerExpectedConfiguration<std::string>("WAAP", "Sigs score file path");
registerExpectedConfiguration<std::string>("WAAP", "Sigs file path");
registerExpectedConfigFile("waap", Config::ConfigFileType::Policy);
registerConfigLoadCb(

24
components/security_apps/waap/waap_component_impl.cc
View File

@@ -60,18 +60,12 @@ WaapComponent::Impl::~Impl()
void
WaapComponent::Impl::init()
{
std::string sigs_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/1.data",
std::string waapDataFileName = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/waap.data",
"WAAP",
"Sigs file path"
);
std::string sigs_score_file_path = getConfigurationWithDefault<string>(
"/etc/cp/conf/waap/2.data",
"WAAP",
"Sigs score file path"
);
assets_metric.init(
"Assets Count",
ReportIS::AudienceTeam::AGENT_CORE,
@@ -84,18 +78,20 @@ WaapComponent::Impl::init()
registerListener();
waap_metric.registerListener();
init(sigs_file_path, sigs_score_file_path);
init(waapDataFileName);
}
void
WaapComponent::Impl::init(const std::string &sigs_file_path, const std::string &sigs_scores_file_path)
WaapComponent::Impl::init(const std::string &waapDataFileName)
{
//waf2_set_log_target(WAF2_LOGTARGET_STDERR);
dbgTrace(D_WAAP) << "WaapComponent::Impl::init() ...";
reputationAggregator.init();
bool success = waf2_proc_start(sigs_file_path, sigs_scores_file_path);
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
bool success = waf2_proc_start(waapDataFileName);
if (!success) {
dbgWarning(D_WAAP) << "WAF2 engine FAILED to initialize (probably failed to load signatures). Aborting!";
waf2_proc_exit();
@@ -107,8 +103,6 @@ WaapComponent::Impl::init(const std::string &sigs_file_path, const std::string &
I_StaticResourcesHandler *static_resources = Singleton::Consume<I_StaticResourcesHandler>::by<WaapComponent>();
static_resources->registerStaticResource("cp-ab.js", "/etc/cp/conf/waap/cp-ab.js");
static_resources->registerStaticResource("cp-csrf.js", "/etc/cp/conf/waap/cp-csrf.js");
waapStateTable = Singleton::Consume<I_Table>::by<WaapComponent>();
}
// Called when component is shut down
@@ -730,7 +724,7 @@ void WaapComponent::Impl::sendNotificationForFirstRequest(
}
bool
WaapComponent::Impl::waf2_proc_start(const std::string& sigsFname, const std::string& scoresFname)
WaapComponent::Impl::waf2_proc_start(const std::string& waapDataFileName)
{
// WAAP uses libxml library, which requires process-level initialization when process starts
#if 0 // TODO:: silence the error messages printed by libxml2
@@ -740,7 +734,7 @@ WaapComponent::Impl::waf2_proc_start(const std::string& sigsFname, const std::st
::xmlInitParser();
return
Singleton::Consume<I_WaapAssetStatesManager>::by<WaapComponent>()->initBasicWaapSigs(sigsFname, scoresFname);
Singleton::Consume<I_WaapAssetStatesManager>::by<WaapComponent>()->initBasicWaapSigs(waapDataFileName);
}
void

4
components/security_apps/waap/waap_component_impl.h
View File

@@ -55,13 +55,13 @@ public:
EventVerdict respond(const EndTransactionEvent &) override;
private:
void init(const std::string &sigs_file_path, const std::string &sigs_scores_file_path);
void init(const std::string &waapDataFileName);
EventVerdict waapDecisionAfterHeaders(IWaf2Transaction& waf2Transaction);
EventVerdict waapDecision(IWaf2Transaction& waf2Transaction);
void finishTransaction(IWaf2Transaction& waf2Transaction);
bool waf2_proc_start(const std::string& sigsFname, const std::string& scoresFname);
bool waf2_proc_start(const std::string& waapDataFileName);
void waf2_proc_exit();
void validateFirstRequestForAsset(const ReportIS::Severity severity);
void sendNotificationForFirstRequest(