github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Nov_12_2023-Dev

Browse Source
This commit is contained in:
Ned Wright
2023-11-12 18:50:17 +00:00
parent 0869b8f24d
commit 3061342b45
114 changed files with 3627 additions and 1305 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -84,6 +84,7 @@ add_library(waap_clib
WaapSampleValue.cc
ParserGql.cc
ParserPercentEncode.cc
ParserPairs.cc
)
add_definitions("-Wno-unused-function")

12
components/security_apps/waap/waap_clib/ContentTypeParser.cc
View File

@@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_CONTENT_TYPE);
const std::string ContentTypeParser::m_parserName = "contentTypeParser";
int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth)
{
dbgTrace(D_WAAP_PARSER_CONTENT_TYPE) << "ContentTypeParser::onKv(): " << std::string(v, v_len);
assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH);
@@ -42,10 +42,12 @@ int ContentTypeParser::onKv(const char *k, size_t k_len, const char *v, size_t v
return 0; // ok
}
ContentTypeParser::ContentTypeParser()
:ctParserState(CTP_STATE_CONTENT_TYPE), m_rcvr(*this), m_hvp(m_rcvr), m_error(false)
{
}
ContentTypeParser::ContentTypeParser() :
ctParserState(CTP_STATE_CONTENT_TYPE),
m_rcvr(*this),
m_hvp(m_rcvr),
m_error(false)
{}
size_t ContentTypeParser::push(const char *data, size_t data_len)
{

2
components/security_apps/waap/waap_clib/ContentTypeParser.h
View File

@@ -25,7 +25,7 @@ class ContentTypeParser : public ParserBase, private IParserReceiver {
CTP_STATE_CONTENT_TYPE_PARAMS
} ctParserState;
private:
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
public:
ContentTypeParser();

2
components/security_apps/waap/waap_clib/DeepAnalyzer.cc
View File

@@ -107,7 +107,7 @@ AnalysisResult DeepAnalyzer::Impl::analyzeData(const D2InputData& data, const IW
(shouldExcept ? "true" : "false");
analysis.threatLevel = threat;
analysis.shouldBlock = shouldBlock && !shouldExcept;
analysis.shouldBlock = shouldBlock;
return analysis;
}

1454
components/security_apps/waap/waap_clib/DeepParser.cc
View File

File diff suppressed because it is too large Load Diff

59
components/security_apps/waap/waap_clib/DeepParser.h
View File

@@ -31,7 +31,7 @@ public:
void setWaapAssetState(std::shared_ptr<WaapAssetState> pWaapAssetState);
// This callback receives input key/value pairs, dissects, decodes and deep-scans these, recursively
// finally, it calls onDetected() on each detected parameter.
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
void clear();
void showStats(std::string& buff, const ValueStatsAnalyzer& valueStats);
@@ -44,7 +44,7 @@ public:
void setMultipartBoundary(const std::string &boundary);
const std::string &getMultipartBoundary() const;
bool isBinaryData() const;
const std::string getLastParser() const;
const std::string getActualParser(size_t parser_depth) const;
bool isWBXmlData() const;
Maybe<std::string> getSplitType() const;
std::vector<std::pair<std::string, std::string> > kv_pairs;
@@ -94,6 +94,7 @@ public:
std::vector<KeywordInfo> m_keywordInfo;
KeyStack m_key;
int getShiftInUrlEncodedBuffer(const ValueStatsAnalyzer &valueStats, std::string &cur_val);
private:
class Ref
@@ -115,18 +116,60 @@ private:
// Split a value by given regexp. Return true if split, false otherwise.
// note: This function calls onKv(), and the call can be recursive!
// TODO:: maybe convert this splitter to Parser-derived class?!
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix);
void createInternalParser(const char *k, size_t k_len, std::string& cur_val,
bool splitByRegex(const std::string &val, const Regex &r, const char *keyPrefix, size_t parser_depth);
int createInternalParser(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags);
int pushValueToTopParser(std::string& cur_val, int flags, bool base64ParamFound);
int parseBuffer(ValueStatsAnalyzer& valueStats, const std::string &cur_val, bool base64ParamFound,
bool shouldUpdateKeyStack);
int flags,
size_t parser_depth
);
int createUrlParserForJson(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags,
size_t parser_depth
);
void printParserDeque();
int parseAfterMisleadingMultipartBoundaryCleaned(
const char *k,
size_t k_len,
std::string &cur_val,
const ValueStatsAnalyzer &valueStats,
bool isBodyPayload,
bool isRefererPayload,
bool isRefererParamPayload,
bool isUrlPayload,
bool isUrlParamPayload,
int flags,
size_t parser_depth,
bool base64ParamFound
);
int pushValueToTopParser(std::string &cur_val, int flags, bool base64ParamFound, int offset, size_t parser_depth);
int parseBuffer(
ValueStatsAnalyzer &valueStats,
const std::string &cur_val,
bool base64ParamFound,
bool shouldUpdateKeyStack,
size_t parser_depth
);
bool shouldEnforceDepthLimit(const std::shared_ptr<ParserBase>& parser) const;
void setLocalMaxObjectDepth(size_t depth) { m_localMaxObjectDepth = depth; }
void setGlobalMaxObjectDepthReached() { m_globalMaxObjectDepthReached = true; }

12
components/security_apps/waap/waap_clib/PHPSerializedDataParser.cc
View File

@@ -19,9 +19,15 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PHPSERIALIZE);
const std::string PHPSerializedDataParser::m_parserName = "PHPSerializedDataParser";
PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver)
: m_state(), m_outReceiver(outReceiver), m_keyStack("php_serialized")
PHPSerializedDataParser::PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth) :
m_state(),
m_outReceiver(outReceiver),
m_keyStack("php_serialized"),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_PHPSERIALIZE)
<< "parser_depth="
<< parser_depth;
}
size_t
@@ -344,7 +350,7 @@ size_t PHPSerializedDataParser::handleValue (const char &c)
// else will parse it normaly.
dbgTrace(D_WAAP_PARSER_PHPSERIALIZE) << "PHPSerializedDataParser::push(): End of Class object" <<
" sending class object data to PHPSerializedDataParser";
PHPSerializedDataParser psdp(m_outReceiver);
PHPSerializedDataParser psdp(m_outReceiver, m_parser_depth);
psdp.push(m_value.c_str(), m_value.length());
if(psdp.error())
{

4
components/security_apps/waap/waap_clib/PHPSerializedDataParser.h
View File

@@ -20,7 +20,7 @@
class PHPSerializedDataParser : public ParserBase {
public:
PHPSerializedDataParser(IParserStreamReceiver &outReceiver);
PHPSerializedDataParser(IParserStreamReceiver &outReceiver, size_t parser_depth);
size_t push(const char* buf, size_t len);
void finish();
virtual const std::string &name() const;
@@ -84,6 +84,6 @@ private:
IParserStreamReceiver &m_outReceiver;
KeyStack m_keyStack;
std::stack <State> m_stack;
size_t m_parser_depth;
static const std::string m_parserName;
};

60
components/security_apps/waap/waap_clib/ParserBase.cc
View File

@@ -13,19 +13,38 @@
#include "ParserBase.h"
#include <string.h>
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER);
// Max size for key and value that can be stored in memory (per thread)
#define MAX_KEY_SIZE 64*1024
#define MAX_VALUE_SIZE 64*1024
BufferedReceiver::BufferedReceiver(IParserReceiver &receiver)
:m_receiver(receiver),
m_flags(BUFFERED_RECEIVER_F_FIRST)
BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_flags(BUFFERED_RECEIVER_F_FIRST),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER)
<< "parser_depth="
<< parser_depth;
}
int BufferedReceiver::onKey(const char *k, size_t k_len)
{
dbgTrace(D_WAAP_PARSER)
<< "Entering BufferedReceiver::onKey with values: \n"
<< "\tkey = "
<< k
<< "\n\tk_len = "
<< k_len
<< "\n\tm_key = "
<< m_key
<< "\n\t m_key_size = "
<< m_key.size()
<< "\n\t parser_depth = "
<< m_parser_depth;
if (m_key.size() + k_len < MAX_KEY_SIZE) {
m_key += std::string(k, k_len);
}
@@ -36,7 +55,18 @@ int BufferedReceiver::onKey(const char *k, size_t k_len)
int BufferedReceiver::onValue(const char *v, size_t v_len)
{
int rc = 0;
dbgTrace(D_WAAP_PARSER)
<< "Entering BufferedReceiver::onValue with values: \n"
<< "\tvalue = "
<< v
<< "\n\tv_len = "
<< v_len
<< "\n\tm_value = "
<< m_value
<< "\n\t m_value_size = "
<< m_value.size()
<< "\n\t parser_depth = "
<< m_parser_depth;
while (v_len > 0) {
// Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size
size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size());
@@ -48,7 +78,18 @@ int BufferedReceiver::onValue(const char *v, size_t v_len)
// Only push full buffers to the m_receiver
if (m_value.size() == MAX_VALUE_SIZE) {
// The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag
int tempRc= m_receiver.onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags);
dbgTrace(D_WAAP_PARSER)
<< " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag"
<< "calling onKv() while m_flags = "
<< m_flags;
int tempRc= m_receiver.onKv(
m_key.data(),
m_key.size(),
m_value.data(),
m_value.size(),
m_flags,
m_parser_depth
);
if (tempRc != 0) {
rc = tempRc;
}
@@ -68,16 +109,19 @@ int BufferedReceiver::onKvDone()
// Call onKv on the remainder of the buffer not yet pushed to the receiver
// This must be called even if m_value is empty in order to signal the BUFFERED_RECEIVER_F_LAST flag to the
// receiver!
int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags);
dbgTrace(D_WAAP_PARSER)
<< " Call onKv on the remainder of the buffer not yet pushed to the receiver "
<< "calling onKv()";
int rc = onKv(m_key.data(), m_key.size(), m_value.data(), m_value.size(), m_flags, m_parser_depth);
// Reset the object's state to allow reuse for other parsers
clear();
return rc;
}
int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags)
int BufferedReceiver::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth)
{
return m_receiver.onKv(k, k_len, v, v_len, flags);
return m_receiver.onKv(k, k_len, v, v_len, flags, parser_depth);
}
void BufferedReceiver::clear()

15
components/security_apps/waap/waap_clib/ParserBase.h
View File

@@ -33,7 +33,7 @@
// Interface for receiver classes that accept full key/value pairs
struct IParserReceiver {
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags) = 0;
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth) = 0;
};
struct IParserReceiver2 {
@@ -80,11 +80,11 @@ struct IParserStreamReceiver : public IParserReceiver {
// This will in many cases cause sub-parsers to also work in zero-copy style too!
class BufferedReceiver : public IParserStreamReceiver {
public:
BufferedReceiver(IParserReceiver &receiver);
BufferedReceiver(IParserReceiver &receiver, size_t parser_depth=0);
virtual int onKey(const char *k, size_t k_len);
virtual int onValue(const char *v, size_t v_len);
virtual int onKvDone();
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
virtual void clear();
// Helper methods to access accumulated key and value (read-only)
@@ -97,6 +97,8 @@ private:
// Accumulated key/value pair
std::string m_key;
std::string m_value;
size_t m_parser_depth;
};
// Base class for various streaming parsers that accept data stream in multiple pieces through
@@ -123,10 +125,11 @@ class BufferedParser : public ParserBase
{
public:
template<typename ..._Args>
explicit BufferedParser(IParserReceiver &receiver, _Args... _args)
explicit BufferedParser(IParserReceiver &receiver, size_t parser_depth, _Args... _args)
:
m_bufferedReceiver(receiver),
m_parser(m_bufferedReceiver, _args...) // pass any extra arguments to specific parser's constructor
m_bufferedReceiver(receiver, parser_depth),
// pass any extra arguments to specific parser's constructor
m_parser(m_bufferedReceiver, parser_depth, _args...)
{}
virtual ~BufferedParser() {}
virtual size_t push(const char *data, size_t data_len) { return m_parser.push(data, data_len); }

8
components/security_apps/waap/waap_clib/ParserBinary.cc
View File

@@ -18,13 +18,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_BINARY);
#define MIN_TEXT_SIZE 10
ParserBinary::ParserBinary(IParserStreamReceiver& receiver) :
ParserBinary::ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth) :
m_parserName("binary"),
m_receiver(receiver),
m_state(s_start),
m_textFromLastBuffer(),
m_textCharCount(0)
m_textCharCount(0),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_BINARY)
<< "parser_depth="
<< parser_depth;
}
ParserBinary::~ParserBinary()

4
components/security_apps/waap/waap_clib/ParserBinary.h
View File

@@ -19,7 +19,7 @@
class ParserBinary : public ParserBase
{
public:
ParserBinary(IParserStreamReceiver& receiver);
ParserBinary(IParserStreamReceiver& receiver, size_t parser_depth);
virtual ~ParserBinary();
virtual size_t push(const char* data, size_t data_len);
virtual void finish();
@@ -39,7 +39,7 @@ private:
state m_state;
std::string m_textFromLastBuffer;
size_t m_textCharCount;
size_t m_parser_depth;
void flush();
};

9
components/security_apps/waap/waap_clib/ParserConfluence.cc
View File

@@ -16,12 +16,17 @@
USE_DEBUG_FLAG(D_WAAP_PARSER_CONFLUENCE);
ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver) :
ParserConfluence::ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth) :
m_parserName("confluence"),
m_state(s_start),
m_receiver(receiver),
m_name()
m_name(),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_CONFLUENCE)
<< "parser_depth="
<< parser_depth;
}
ParserConfluence::~ParserConfluence()

3
components/security_apps/waap/waap_clib/ParserConfluence.h
View File

@@ -19,7 +19,7 @@
class ParserConfluence : public ParserBase
{
public:
ParserConfluence(IParserStreamReceiver& receiver);
ParserConfluence(IParserStreamReceiver& receiver, size_t parser_depth);
virtual ~ParserConfluence();
virtual size_t push(const char* data, size_t data_len);
@@ -43,6 +43,7 @@ private:
state m_state;
IParserStreamReceiver& m_receiver;
std::string m_name;
size_t m_parser_depth;
};
#endif

15
components/security_apps/waap/waap_clib/ParserDelimiter.cc
View File

@@ -16,15 +16,22 @@
USE_DEBUG_FLAG(D_WAAP_PARSER_DELIMITER);
ParserDelimiter::ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName)
: ParserBase(),
ParserDelimiter::ParserDelimiter(
IParserStreamReceiver& receiver,
size_t parser_depth,
char delim,
const std::string& delimName
) : ParserBase(),
m_state(s_start),
m_receiver(receiver),
m_delim(delim),
m_delim_name(delimName),
m_found_delim(false)
m_found_delim(false),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_DELIMITER)
<< "parsing delimiter: parser depth="
<< parser_depth;
}
ParserDelimiter::~ParserDelimiter()

3
components/security_apps/waap/waap_clib/ParserDelimiter.h
View File

@@ -19,7 +19,7 @@
class ParserDelimiter : public ParserBase
{
public:
ParserDelimiter(IParserStreamReceiver& receiver, char delim, const std::string& delimName);
ParserDelimiter(IParserStreamReceiver& receiver, size_t parser_depth, char delim, const std::string& delimName);
virtual ~ParserDelimiter();
virtual size_t push(const char* data, size_t data_len);
@@ -45,6 +45,7 @@ private:
char m_delim;
std::string m_delim_name;
bool m_found_delim;
size_t m_parser_depth;
};

18
components/security_apps/waap/waap_clib/ParserGql.cc
View File

@@ -21,12 +21,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_GQL);
const std::string ParserGql::m_parserName = "gqlParser";
ParserGql::ParserGql(IParserReceiver& receiver) :
ParserGql::ParserGql(IParserReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_error(false),
m_curNameValues(0)
m_curNameValues(0),
m_parser_depth(parser_depth)
{
dbgFlow(D_WAAP_PARSER_GQL);
dbgTrace(D_WAAP_PARSER_GQL) << "parser_depth=" << parser_depth;
}
ParserGql::~ParserGql() {
@@ -56,7 +58,9 @@ size_t ParserGql::push(const char* buf, size_t len) {
// Handle corner case of last name visited without value: don't forget to output that name too
if (m_curNameValues == 0 && !m_curNodeName.empty()) {
dbgTrace(D_WAAP_PARSER_GQL) << "handle last name: '" << m_curNodeName << "'";
if (m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH) != 0) {
if (m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
) != 0) {
m_error = true;
}
}
@@ -81,7 +85,9 @@ bool ParserGql::visitValue(const char *value)
{
dbgTrace(D_WAAP_PARSER_GQL) << "'" << value << "'";
m_curNameValues++;
return m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH);
return m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), value, strlen(value), BUFFERED_RECEIVER_F_BOTH, m_parser_depth
);
}
bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
@@ -89,7 +95,9 @@ bool ParserGql::visitName(const facebook::graphql::ast::Name &node)
dbgTrace(D_WAAP_PARSER_GQL) << node.getValue() << "'";
bool ret = true;
if (m_curNameValues == 0 && !m_curNodeName.empty()) {
ret = m_receiver.onKv(m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH);
ret = m_receiver.onKv(
m_curNodeName.data(), m_curNodeName.size(), "", 0, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
);
}
// wait for next name
m_curNodeName = std::string(node.getValue());

3
components/security_apps/waap/waap_clib/ParserGql.h
View File

@@ -25,7 +25,7 @@
class ParserGql : public ParserBase, public facebook::graphql::ast::visitor::AstVisitor {
public:
ParserGql(IParserReceiver &receiver);
ParserGql(IParserReceiver &receiver, size_t parser_depth);
virtual ~ParserGql();
size_t push(const char *data, size_t data_len);
void finish();
@@ -51,6 +51,7 @@ private:
bool visitEnumValue(const facebook::graphql::ast::EnumValue &node) override;
public:
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_JQL_H

148
components/security_apps/waap/waap_clib/ParserHTML.cc
View File

@@ -38,16 +38,24 @@ void ParserHTML::onStartElement(
if (attr_value == NULL) {
attr_value = (const xmlChar*)"";
}
dbgTrace(D_WAAP_PARSER_HTML) << "\tHTML ATTR: elem='" << (char*)localname << "', " << attr_localname <<
"='" << std::string((char*)attr_value) << "'";
p->m_key.push((const char*)attr_localname, xmlStrlen(attr_localname));
dbgTrace(D_WAAP_PARSER_HTML)
<< "\tHTML ATTR: elem='"
<< (char *)localname
<< "', "
<< attr_localname
<< "='"
<< std::string((char *)attr_value)
<< "'";
p->m_key.push((const char *)attr_localname, xmlStrlen(attr_localname));
if (p->m_receiver.onKv(
p->m_key.first().c_str(),
p->m_key.first().size(),
(const char*)attr_value, strlen((const char*)attr_value),
BUFFERED_RECEIVER_F_BOTH
) != 0) {
p->m_key.first().c_str(),
p->m_key.first().size(),
(const char *)attr_value,
strlen((const char *)attr_value),
BUFFERED_RECEIVER_F_BOTH,
p->m_parser_depth
) != 0) {
p->m_state = s_error;
}
p->m_key.pop("HTML end attribute");
@@ -73,8 +81,8 @@ ParserHTML::onEndElement(
dbgTrace(D_WAAP_PARSER_HTML) << "HTML CLOSE: '" << localname << "'";
if (p->m_elemTrackStack.empty()) {
dbgWarning(D_WAAP_PARSER_HTML) <<
"HTML closing tag and elem track stack is empty. This is probably sign of a bug!";
dbgWarning(D_WAAP_PARSER_HTML)
<< "HTML closing tag and elem track stack is empty. This is probably sign of a bug!";
return;
}
@@ -111,8 +119,10 @@ ParserHTML::onEndElement(
p->m_key.pop("HTML end element");
}
void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) {
ParserHTML* p = (ParserHTML*)ctx;
void
ParserHTML::onCharacters(void *ctx, const xmlChar *ch, int len)
{
ParserHTML *p = (ParserHTML *)ctx;
if (p->m_elemTrackStack.empty()) {
dbgWarning(D_WAAP_PARSER_HTML) << "HTML text and elem track stack is empty. This is probably sign of a bug!";
@@ -141,7 +151,9 @@ void ParserHTML::onCharacters(void* ctx, const xmlChar* ch, int len) {
elemTrackInfo.value += val;
}
static void onError(void* ctx, const char* msg, ...) {
static void
onError(void *ctx, const char *msg, ...)
{
static const size_t TMP_BUF_SIZE = 4096;
char string[TMP_BUF_SIZE];
va_list arg_ptr;
@@ -152,9 +164,19 @@ static void onError(void* ctx, const char* msg, ...) {
dbgTrace(D_WAAP_PARSER_HTML) << "LIBXML (html) onError: " << std::string(string);
}
ParserHTML::ParserHTML(IParserStreamReceiver& receiver)
:m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("html_parser"), m_pushParserCtxPtr(NULL) {
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::ParserHTML()";
ParserHTML::ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_bufLen(0),
m_key("html_parser"),
m_pushParserCtxPtr(NULL),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::ParserHTML()"
<< "parser_depth="
<< parser_depth;
// TODO:: is zeroing this really needed?
memset(m_buf, 0, sizeof(m_buf));
@@ -173,7 +195,8 @@ ParserHTML::ParserHTML(IParserStreamReceiver& receiver)
m_key.push("html", 4);
}
ParserHTML::~ParserHTML() {
ParserHTML::~ParserHTML()
{
// Cleanup HTML
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::~ParserHTML()";
@@ -182,9 +205,15 @@ ParserHTML::~ParserHTML() {
}
}
bool ParserHTML::filterErrors(xmlErrorPtr xmlError) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): xmlError " << xmlError->code << ": '" <<
xmlError->message << "'";
bool
ParserHTML::filterErrors(xmlErrorPtr xmlError)
{
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::filterErrors(): xmlError "
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
// Ignore specific error: "HTML declaration allowed only at the start of the document".
// This includes the case of "multiple HTML declarations" we've seen sent by some SOAP clients.
@@ -193,15 +222,21 @@ bool ParserHTML::filterErrors(xmlErrorPtr xmlError) {
// Ignoring this error prevents the WAAP code from thinking the HTML is "broken" and from scanning the HTML
// source as-is, in effect preventing false alarm on that HTML source.
if (xmlError->code == XML_ERR_RESERVED_XML_NAME || xmlError->code == XML_ERR_UNDECLARED_ENTITY) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::filterErrors(): ignoring the '" << xmlError->code << ": " <<
xmlError->message << "' html parser error.";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::filterErrors(): ignoring the '"
<< xmlError->code
<< ": "
<< xmlError->message
<< "' html parser error.";
return false;
}
return true;
}
size_t ParserHTML::push(const char* data, size_t data_len) {
size_t
ParserHTML::push(const char *data, size_t data_len)
{
size_t i = 0;
char c;
@@ -212,8 +247,12 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
if (htmlParseChunk(m_pushParserCtxPtr, m_buf, 0, 1)) {
xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr);
if (xmlError && filterErrors(xmlError)) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code << ": '" <<
xmlError->message << "'";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): xmlError: code="
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
m_state = s_error; // error
return -1;
}
@@ -232,8 +271,13 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
// fall through //
CP_FALL_THROUGH;
case s_accumulate_first_bytes:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_accumulate_first_bytes. c='" << data[i] <<
"'; m_bufLen=" << m_bufLen << "; i=" << i;
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_accumulate_first_bytes. c='"
<< data[i]
<< "'; m_bufLen="
<< m_bufLen
<< "; i="
<< i;
m_buf[m_bufLen] = c;
m_bufLen++;
if (c == '?') {
@@ -244,13 +288,19 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
}
break;
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_start_parsing. sending len=" << m_bufLen << ": '" <<
std::string(m_buf, m_bufLen) << "'; i=" << i;
// Create HTML SAX (push parser) context
// It is important to buffer at least first 4 bytes of input stream so libxml can determine text encoding!
m_pushParserCtxPtr = htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL,
XML_CHAR_ENCODING_UTF8);
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_start_parsing. sending len="
<< m_bufLen
<< ": '"
<< std::string(m_buf, m_bufLen)
<< "'; i="
<< i;
// Create HTML SAX (push parser) context
// It is important to buffer at least first 4 bytes of input stream so libxml can determine text
// encoding!
m_pushParserCtxPtr =
htmlCreatePushParserCtxt(&m_saxHandler, this, m_buf, m_bufLen, NULL, XML_CHAR_ENCODING_UTF8);
// Enable "permissive mode" for HTML SAX parser.
// In this mode, the libxml parser doesn't stop on errors, but still reports them!
@@ -261,14 +311,23 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
// fall through //
CP_FALL_THROUGH;
case s_parsing:
dbgTrace(D_WAAP_PARSER_HTML) << "ParserHTML::push(): s_parsing. sending len=" << (int)(data_len - i) <<
": '" << std::string(data + i, data_len - i) << "'; i=" << i;
dbgTrace(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): s_parsing. sending len="
<< (int)(data_len - i)
<< ": '"
<< std::string(data + i, data_len - i)
<< "'; i="
<< i;
if (m_pushParserCtxPtr) {
if (htmlParseChunk(m_pushParserCtxPtr, data + i, data_len - i, 0)) {
xmlErrorPtr xmlError = xmlCtxtGetLastError(m_pushParserCtxPtr);
if (xmlError && filterErrors(xmlError)) {
dbgDebug(D_WAAP_PARSER_HTML) << "ParserHTML::push(): xmlError: code=" << xmlError->code <<
": '" << xmlError->message << "'";
dbgDebug(D_WAAP_PARSER_HTML)
<< "ParserHTML::push(): xmlError: code="
<< xmlError->code
<< ": '"
<< xmlError->message
<< "'";
m_state = s_error; // error
return 0;
}
@@ -290,15 +349,20 @@ size_t ParserHTML::push(const char* data, size_t data_len) {
return i;
}
void ParserHTML::finish() {
void
ParserHTML::finish()
{
push(NULL, 0);
}
const std::string &
ParserHTML::name() const {
ParserHTML::name() const
{
return m_parserName;
}
bool ParserHTML::error() const {
bool
ParserHTML::error() const
{
return m_state == s_error;
}

3
components/security_apps/waap/waap_clib/ParserHTML.h
View File

@@ -24,7 +24,7 @@
class ParserHTML : public ParserBase {
public:
ParserHTML(IParserStreamReceiver &receiver);
ParserHTML(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserHTML();
size_t push(const char *data, size_t data_len);
void finish();
@@ -81,4 +81,5 @@ private:
htmlParserCtxtPtr m_pushParserCtxPtr;
static const std::string m_parserName;
size_t m_parser_depth;
};

154
components/security_apps/waap/waap_clib/ParserJson.cc
View File

@@ -20,17 +20,21 @@
#include <assert.h>
USE_DEBUG_FLAG(D_WAAP_PARSER_JSON);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
const std::string ParserJson::m_parserName = "jsonParser";
int ParserJson::cb_null() {
int
ParserJson::cb_null()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_null():";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), "null", 4, DataType::EMPTY);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "null", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
@@ -41,20 +45,22 @@ int ParserJson::cb_null() {
return 1;
}
int ParserJson::cb_boolean(int boolean) {
int
ParserJson::cb_boolean(int boolean)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_boolean(): " << boolean;
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), NULL, boolean, DataType::BOOLEAN);
}
if (boolean) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "true", 4, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
}
else {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH)) {
} else {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), "false", 5, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
}
@@ -65,14 +71,16 @@ int ParserJson::cb_boolean(int boolean) {
return 1;
}
int ParserJson::cb_number(const char* s, yajl_size_t slen) {
int
ParserJson::cb_number(const char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_number(): '" << std::string(s, slen) << "'";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), s, slen, DataType::NUMBER);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(m_key.c_str(), m_key.size(), s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth)) {
return 0;
}
@@ -82,14 +90,19 @@ int ParserJson::cb_number(const char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char*)s, slen) << "'";
int
ParserJson::cb_string(const unsigned char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_string(): '" << std::string((const char *)s, slen) << "'";
if (m_receiver2) {
m_receiver2->onKvt(m_key.c_str(), m_key.size(), (const char*)s, slen, DataType::STRING);
}
if (m_receiver.onKv(m_key.c_str(), m_key.size(), (const char*)s, slen, BUFFERED_RECEIVER_F_BOTH)) {
if (m_receiver.onKv(
m_key.c_str(), m_key.size(), (const char *)s, slen, BUFFERED_RECEIVER_F_BOTH, m_parser_depth
)) {
return 0;
}
@@ -99,8 +112,10 @@ int ParserJson::cb_string(const unsigned char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char*)s, slen) << "'";
int
ParserJson::cb_map_key(const unsigned char *s, yajl_size_t slen)
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_map_key(): '" << std::string((const char *)s, slen) << "'";
m_key.push((char*)s, slen);
@@ -111,7 +126,9 @@ int ParserJson::cb_map_key(const unsigned char* s, yajl_size_t slen) {
return 1;
}
int ParserJson::cb_start_map() {
int
ParserJson::cb_start_map()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_map():";
if (m_receiver2) {
@@ -122,7 +139,9 @@ int ParserJson::cb_start_map() {
return 1;
}
int ParserJson::cb_end_map() {
int
ParserJson::cb_end_map()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_map():";
if (m_receiver2) {
@@ -140,7 +159,9 @@ int ParserJson::cb_end_map() {
return 1;
}
int ParserJson::cb_start_array() {
int
ParserJson::cb_start_array()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_start_array():";
if (m_receiver2) {
@@ -151,7 +172,9 @@ int ParserJson::cb_start_array() {
return 1;
}
int ParserJson::cb_end_array() {
int
ParserJson::cb_end_array()
{
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::cb_end_array():";
if (m_receiver2) {
@@ -162,6 +185,7 @@ int ParserJson::cb_end_array() {
m_depthStack.pop_back();
}
if (!m_depthStack.empty() && m_depthStack.back() == js_map) {
m_key.pop("json end array");
}
@@ -169,51 +193,78 @@ int ParserJson::cb_end_array() {
}
// Static functions to be called from C and forward the calls to respective class cb_* methods
int ParserJson::p_null(void* ctx)
int
ParserJson::p_null(void *ctx)
{
return ((ParserJson*)ctx)->cb_null();
}
int ParserJson::p_boolean(void* ctx, int boolean)
int
ParserJson::p_boolean(void *ctx, int boolean)
{
return ((ParserJson*)ctx)->cb_boolean(boolean);
}
int ParserJson::p_number(void* ctx, const char* s, yajl_size_t slen)
int
ParserJson::p_number(void *ctx, const char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_number(s, slen);
}
int ParserJson::p_string(void* ctx, const unsigned char* s, yajl_size_t slen)
int
ParserJson::p_string(void *ctx, const unsigned char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_string(s, slen);
}
int ParserJson::p_map_key(void* ctx, const unsigned char* s, yajl_size_t slen)
int
ParserJson::p_map_key(void *ctx, const unsigned char *s, yajl_size_t slen)
{
return ((ParserJson*)ctx)->cb_map_key(s, slen);
}
int ParserJson::p_start_map(void* ctx)
int
ParserJson::p_start_map(void *ctx)
{
return ((ParserJson*)ctx)->cb_start_map();
}
int ParserJson::p_end_map(void* ctx)
int
ParserJson::p_end_map(void *ctx)
{
return ((ParserJson*)ctx)->cb_end_map();
}
int ParserJson::p_start_array(void* ctx)
int
ParserJson::p_start_array(void *ctx)
{
return ((ParserJson*)ctx)->cb_start_array();
}
int ParserJson::p_end_array(void* ctx)
int
ParserJson::p_end_array(void *ctx)
{
return ((ParserJson*)ctx)->cb_end_array();
}
ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) :
ParserJson::ParserJson(
IParserReceiver &receiver,
bool should_collect_oas,
size_t parser_depth,
IParserReceiver2 *receiver2)
:
m_receiver(receiver),
m_receiver2(receiver2),
m_state(s_start),
m_bufLen(0),
m_key("json_parser"),
m_jsonHandler(NULL)
m_jsonHandler(NULL),
is_map_empty(false),
should_collect_for_oa_schema_updater(should_collect_oas),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_JSON) << "parser_depth= " << parser_depth;
// TODO:: do we really want to clear this?
memset(m_buf, 0, sizeof(m_buf));
@@ -232,7 +283,7 @@ ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) :
};
m_jsonHandler = yajl_alloc(&callbacks, NULL, this);
if (m_jsonHandler == NULL) {
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::ParserJson(): yajl_alloc() failed. Switching to s_error state.";
m_state = s_error;
@@ -249,7 +300,8 @@ ParserJson::ParserJson(IParserReceiver& receiver, IParserReceiver2* receiver2) :
m_key.push("json", 4);
}
ParserJson::~ParserJson() {
ParserJson::~ParserJson()
{
// Cleanup JSON
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::~ParserJson():";
@@ -258,7 +310,9 @@ ParserJson::~ParserJson() {
}
}
size_t ParserJson::push(const char* buf, size_t len) {
size_t
ParserJson::push(const char *buf, size_t len)
{
size_t i = 0;
char c;
@@ -280,6 +334,7 @@ size_t ParserJson::push(const char* buf, size_t len) {
while (i < len) {
c = buf[i];
switch (m_state) {
case s_start:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start";
@@ -288,8 +343,12 @@ size_t ParserJson::push(const char* buf, size_t len) {
// fallthrough //
CP_FALL_THROUGH;
case s_accumulate_first_bytes:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_accumulate_first_bytes. i=" << i <<
" c='" << buf[i] << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_accumulate_first_bytes. i="
<< i
<< " c='"
<< buf[i]
<< "'";
m_buf[m_bufLen] = c;
m_bufLen++;
if (m_bufLen == FIRST_JSON_BUFFER_SIZE) {
@@ -298,15 +357,23 @@ size_t ParserJson::push(const char* buf, size_t len) {
break;
case s_start_parsing:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_start_parsing. sending len=" <<
(int)m_bufLen << ": '" << std::string(m_buf, m_bufLen) << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_start_parsing. sending len="
<< (int)m_bufLen
<< ": '"
<< std::string(m_buf, m_bufLen)
<< "'";
m_state = s_parsing;
// fallthrough //
CP_FALL_THROUGH;
case s_parsing:
dbgTrace(D_WAAP_PARSER_JSON) << "ParserJson::push(): s_parsing. sending len=" << (int)(len - i) << ": '" <<
std::string(buf + i, len - i) << "'";
dbgTrace(D_WAAP_PARSER_JSON)
<< "ParserJson::push(): s_parsing. sending len="
<< (int)(len - i)
<< ": '"
<< std::string(buf + i, len - i)
<< "'";
if (m_bufLen > 0) {
// Send accumulated bytes (if any)
if (yajl_parse(m_jsonHandler, (unsigned char*)m_buf, m_bufLen) != yajl_status_ok) {
@@ -333,15 +400,20 @@ size_t ParserJson::push(const char* buf, size_t len) {
return len;
}
void ParserJson::finish() {
void
ParserJson::finish()
{
push(NULL, 0);
}
const std::string &
ParserJson::name() const {
ParserJson::name() const
{
return m_parserName;
}
bool ParserJson::error() const {
bool
ParserJson::error() const
{
return m_state == s_error;
}

10
components/security_apps/waap/waap_clib/ParserJson.h
View File

@@ -27,7 +27,11 @@ typedef size_t yajl_size_t;
class ParserJson : public ParserBase {
public:
ParserJson(IParserReceiver &receiver, IParserReceiver2 *receiver2=NULL);
ParserJson(
IParserReceiver &receiver,
bool should_collect_for_oa_schema_updater=false,
size_t parser_depth=0,
IParserReceiver2 *receiver2=NULL);
virtual ~ParserJson();
size_t push(const char *data, size_t data_len);
void finish();
@@ -80,6 +84,10 @@ private:
KeyStack m_key;
std::vector<enum js_state> m_depthStack;
yajl_handle m_jsonHandler;
bool is_map_empty;
bool should_collect_for_oa_schema_updater;
size_t m_parser_depth;
public:
static const std::string m_parserName;
};

22
components/security_apps/waap/waap_clib/ParserMultipartForm.cc
View File

@@ -26,7 +26,14 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_MULTIPART_FORM);
const std::string ParserMultipartForm::m_parserName = "ParserMultipartForm";
int ParserMultipartForm::HdrValueAnalyzer::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags)
int ParserMultipartForm::HdrValueAnalyzer::onKv(
const char* k,
size_t k_len,
const char* v,
size_t v_len,
int flags,
size_t parser_depth
)
{
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "HdrValueAnalyzer::onKv(): k='%.*s' v='%.*s'" << (int)k_len << v;
assert((flags & BUFFERED_RECEIVER_F_BOTH) == BUFFERED_RECEIVER_F_BOTH);
@@ -43,10 +50,8 @@ void ParserMultipartForm::HdrValueAnalyzer::clear() {
}
ParserMultipartForm::ParserMultipartForm(
IParserStreamReceiver& receiver,
const char* boundary,
size_t boundary_len)
:
IParserStreamReceiver &receiver, size_t parser_depth, const char *boundary, size_t boundary_len
) :
m_receiver(receiver),
m_partIdx(0),
state(s_start),
@@ -55,9 +60,12 @@ ParserMultipartForm::ParserMultipartForm(
lookbehind(NULL),
multipart_boundary(NULL),
m_headerValueParser(NULL),
m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer)
m_hdrValueAnalyzerBufferedReceiver(m_hdrValueAnalyzer),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM) << "ParserMultipartForm::ParserMultipartForm()";
dbgTrace(D_WAAP_PARSER_MULTIPART_FORM)
<< "ParserMultipartForm::ParserMultipartForm() parser_depth="
<< parser_depth;
boundary_len += 2; // two hyphens will be prepended to boundary string provided
multipart_boundary = (char*)malloc(boundary_len + boundary_len + 9);

19
components/security_apps/waap/waap_clib/ParserMultipartForm.h
View File

@@ -18,18 +18,25 @@
#include "ParserHdrValue.h"
#include <boost/core/noncopyable.hpp>
class ParserMultipartForm : public ParserBase, boost::noncopyable {
class ParserMultipartForm : public ParserBase, boost::noncopyable
{
public:
class HdrValueAnalyzer : public IParserReceiver {
class HdrValueAnalyzer : public IParserReceiver
{
public:
int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags, size_t parser_depth);
void clear();
const std::string &getPartName() const { return m_partName; }
private:
std::string m_partName;
};
ParserMultipartForm(IParserStreamReceiver &receiver, const char *boundary, size_t boundary_len);
ParserMultipartForm(
IParserStreamReceiver &receiver,
size_t parser_depth,
const char *boundary,
size_t boundary_len
);
virtual ~ParserMultipartForm();
size_t push(const char *buf, size_t len);
void finish();
@@ -37,7 +44,8 @@ public:
virtual bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
enum state
{
s_start,
s_start_boundary,
s_key_start,
@@ -88,6 +96,7 @@ private:
std::string m_partName; // Part name
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_MULTIPART_FORM_H__1c7eb4fa

479
components/security_apps/waap/waap_clib/ParserPairs.cc Normal file
View File

@@ -0,0 +1,479 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserPairs.h"
#include "Waf2Util.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_PAIRS);
USE_DEBUG_FLAG(D_WAAP);
const std::string ParserPairs::m_parserName = "ParserPairs";
ParserPairs::ParserPairs(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar,
bool should_decode_per,
bool should_decode_plus_sign
) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_separatorChar(separatorChar),
m_escapedCharCandidate(0),
should_decode_percent(should_decode_per),
should_decode_plus(should_decode_plus_sign),
m_parser_depth(parser_depth),
m_bracket_counter(0)
{
dbgTrace(D_WAAP)
<< "should_decode_percent="
<< should_decode_per
<< "parser_depth="
<< parser_depth;
// TODO:: is there a need for this?
memset(m_escaped, 0, sizeof(m_escaped));
}
ParserPairs::~ParserPairs()
{}
size_t
ParserPairs::push(const char *buf, size_t len)
{
size_t i = 0;
size_t mark = 0;
char c;
int is_last = 0;
dbgTrace(D_WAAP) << "ParserPairs::push(): starting (len=" << len << ")";
if (len == 0) {
dbgTrace(D_WAAP) << "ParserPairs::push(): end of data signal! m_state=" << m_state;
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_state == s_key_start) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
} else if (m_state == s_value_start) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
}
m_escapedLen = 0;
}
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
return i;
}
return 0;
}
while (i < len) {
c = buf[i];
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32) {
// Bitwise operation checking since `char` can be either signed on unsigned depending on arch
if (!isspace(c) && ((c & 0x80) == 0x00)) {
dbgDebug(D_WAAP_PARSER_PAIRS)
<< "invalid URL encoding character: "
<< c
<< " decimal value is "
<< c + 0;
m_state = s_error;
return i;
}
}
dbgTrace(D_WAAP_PARSER_PAIRS)
<< "ParserPairs::push(): state="
<< m_state
<< "; ch='"
<< c
<< "' m_bracket_counter = "
<< m_bracket_counter;
switch (m_state) {
case s_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_start";
// m_state = s_key_start;
// fallthrough //
CP_FALL_THROUGH;
}
case s_key_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_start";
mark = i;
m_state = s_key;
// fallthrough //
CP_FALL_THROUGH;
}
case s_key: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key";
// skip leading spaces in the key
if (isspace(c)) {
m_state = s_key_start; // skip the space character without including it in the output
break;
}
if (c == '%' && should_decode_percent) {
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_state = s_key_escaped1;
break;
} else if (c == '+' && should_decode_plus) {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
mark = i;
}
m_escaped[m_escapedLen] = ' ';
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_key_start;
break;
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
}
if (c == m_separatorChar) {
// this happens when there is a key without value. Example: ?p&a=b&k&%61&blah
// in this case we emit the key, but not the value, and send onKvDone to cause
// the receiver to process the pair: key will be provided with no value.
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
if (m_receiver.onKvDone() != 0) {
m_state = s_error;
return i;
}
m_state = s_key_start;
break;
}
if (c == '=') {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
m_state = s_value_start;
break;
}
if (is_last) {
if (m_receiver.onKey(buf + mark, (i - mark) + 1) != 0) {
m_state = s_error;
return i;
}
}
break;
}
case s_key_escaped1: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped1";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) { // character right after the '%' is not a valid hex char.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onKey("%", 1) != 0) {
return i;
}
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onKey(&c, 1) != 0) {
return i;
}
// otherwise (the character is not '%'), switch back to the s_key state
m_state = s_key_start;
}
break;
}
m_escapedCharCandidate = c;
m_escaped[m_escapedLen] = v << 4;
m_state = s_key_escaped2;
break;
}
case s_key_escaped2: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_key_escaped2";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) {
// This situation (2nd character is not valid hex) is not treated right now.
// In this case, v will be equal to 0 and output character will be invalid one.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onKey("%", 1) != 0) {
return i;
}
// add the character that was thought to be escaped value
if (m_receiver.onKey(&m_escapedCharCandidate, 1)) {
return i;
}
// re parse the character as a key (i is incremented back to current value)
i--;
m_state = s_key_start;
break;
}
m_escapedCharCandidate = 0;
m_escaped[m_escapedLen] |= v;
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_key_start;
break;
}
case s_value_start: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_start";
mark = i;
m_state = s_value;
// fallthrough //
CP_FALL_THROUGH;
}
case s_value: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value";
if (c == '%' && should_decode_percent) {
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_state = s_value_escaped1;
break;
} else if (c == '+' && should_decode_plus) {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
m_state = s_error;
return i;
}
}
m_escaped[m_escapedLen] = ' ';
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_value_start;
break;
} else {
// flush unescaped data collected (if any)
if (c == '{' || c == '[') m_bracket_counter++;
if (c == '}' || c == ']') m_bracket_counter--;
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
}
if (c == m_separatorChar) {
if (m_bracket_counter) {
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
mark = i;
}
} else {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() s_value : failed on value";
m_state = s_error;
return i;
}
if (m_receiver.onKvDone() != 0) {
dbgWarning(D_WAAP_PARSER_PAIRS) << "ParserPairs::push() : s_value : failed on KV";
m_state = s_error;
return i;
}
m_state = s_key_start;
break;
}
}
if (is_last) {
if (m_receiver.onValue(buf + mark, (i - mark) + 1) != 0) {
m_state = s_error;
return i;
}
}
break;
}
case s_value_escaped1: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped1";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) { // character right after the '%' is not a valid hex char.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onValue("%", 1) != 0) {
return i;
}
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences)
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onValue(&c, 1) != 0) {
return i;
}
// otherwise (the character is not '%'), switch back to the s_value state
m_state = s_value_start;
}
break;
}
m_escapedCharCandidate = c;
m_escaped[m_escapedLen] = v << 4;
m_state = s_value_escaped2;
break;
}
case s_value_escaped2: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_value_escaped2";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid) {
// This situation (2nd character is not valid hex) is not treated right now.
// In this case, v will be equal to 0 and output character will be invalid one.
// dump escaped chars
if (m_escapedLen > 0 && m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
if (m_receiver.onValue("%", 1) != 0) {
return i;
}
// add the character that was thought to be escaped value
if (m_receiver.onValue(&m_escapedCharCandidate, 1)) {
return i;
}
// re parse the character as a key (i is incremented back to current value)
i--;
m_state = s_value_start;
break;
}
m_escapedCharCandidate = 0;
m_escaped[m_escapedLen] |= v;
m_escapedLen++;
if (m_escapedLen >= MAX_PAIRS_ESCAPED_SIZE) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_value_start;
break;
}
case s_error: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): s_error";
return 0;
}
default: {
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): URL parser unrecoverable error";
m_state = s_error;
return 0;
}
} // end of switch()
++i;
}
dbgTrace(D_WAAP_PARSER_PAIRS) << "ParserPairs::push(): finished: len=" << len;
return len;
}
void
ParserPairs::finish()
{
push(NULL, 0);
}
const std::string &
ParserPairs::name() const
{
return m_parserName;
}
bool
ParserPairs::error() const
{
return m_state == s_error;
}

65
components/security_apps/waap/waap_clib/ParserPairs.h Executable file
View File

@@ -0,0 +1,65 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_PAIRS_H__
#define __PARSER_PAIRS_H__
#include "ParserBase.h"
#include <string.h>
#define MAX_PAIRS_ESCAPED_SIZE 16
class ParserPairs : public ParserBase {
public:
ParserPairs(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar = '&',
bool should_decode_per = false,
bool should_decode_plus_sign = false);
virtual ~ParserPairs();
size_t push(const char *data, size_t data_len);
void finish();
virtual const std::string &name() const;
bool error() const;
virtual size_t depth() { return 1; }
private:
enum state {
s_start,
s_key_start,
s_key,
s_key_escaped1,
s_key_escaped2,
s_value_start,
s_value,
s_value_escaped1,
s_value_escaped2,
s_end,
s_error
};
IParserStreamReceiver &m_receiver;
enum state m_state;
unsigned char m_escapedLen; // count of characters loaded in m_escaped[] buffer
char m_escaped[MAX_PAIRS_ESCAPED_SIZE];
char m_separatorChar;
char m_escapedCharCandidate;
bool should_decode_percent;
bool should_decode_plus;
static const std::string m_parserName;
size_t m_parser_depth;
int m_bracket_counter;
};
#endif // __PARSER_PAIRS_H__

42
components/security_apps/waap/waap_clib/ParserPercentEncode.cc
View File

@@ -19,12 +19,17 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_PERCENT);
const std::string ParserPercentEncode::m_parserName = "ParserPercentEncode";
ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver) :
ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_escapedCharCandidate(0)
m_escapedCharCandidate(0),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "parser_depth="
<< parser_depth;
memset(m_escaped, 0, sizeof(m_escaped));
}
@@ -51,7 +56,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
@@ -97,7 +103,6 @@ ParserPercentEncode::push(const char *buf, size_t len)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_start";
// fallthrough //
CP_FALL_THROUGH;
}
@@ -107,7 +112,6 @@ ParserPercentEncode::push(const char *buf, size_t len)
<< "ParserPercentEncode::push(): s_value_start";
pointer_in_buffer = i;
m_state = s_value;
// fallthrough //
CP_FALL_THROUGH;
}
@@ -120,9 +124,10 @@ ParserPercentEncode::push(const char *buf, size_t len)
if (i - pointer_in_buffer > 0)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with buffer = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
<< "<<< of size "
<< i - pointer_in_buffer;
if (m_receiver.onValue(buf + pointer_in_buffer, i - pointer_in_buffer) != 0)
{
m_state = s_error;
@@ -140,7 +145,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
@@ -155,7 +161,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
<< "<<< of size "
<< (i - pointer_in_buffer) + 1;
if (m_receiver.onValue(buf + pointer_in_buffer, (i - pointer_in_buffer) + 1) != 0)
{
m_state = s_error;
@@ -177,7 +184,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
@@ -186,7 +194,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
}
m_escapedLen = 0;
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with \"%\" = >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
@@ -199,7 +207,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
{
// pass the non-hex character back to the output too.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with current char = >>>"
<< c
<< "<<<";
if (m_receiver.onValue(&c, 1) != 0)
@@ -232,7 +240,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
@@ -243,7 +252,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with \"%\" >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
@@ -252,7 +261,7 @@ ParserPercentEncode::push(const char *buf, size_t len)
}
// add the character that was thought to be escaped value
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "ParserPercentEncode::push(): call onValue with m_escapedCharCandicate = >>>"
<< m_escapedCharCandidate
<< "<<<";
if (m_receiver.onValue(&m_escapedCharCandidate, 1))
@@ -273,7 +282,8 @@ ParserPercentEncode::push(const char *buf, size_t len)
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
<< "<<< and m_escapedLen = "
<< m_escapedLen;
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;

3
components/security_apps/waap/waap_clib/ParserPercentEncode.h
View File

@@ -22,7 +22,7 @@
class ParserPercentEncode : public ParserBase {
public:
ParserPercentEncode(IParserStreamReceiver &receiver);
ParserPercentEncode(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserPercentEncode();
size_t push(const char *data, size_t data_len);
void finish();
@@ -53,6 +53,7 @@ private:
char m_escaped[MAX_PERCENT_ENCODED_SIZE];
char m_escapedCharCandidate;
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif

2
components/security_apps/waap/waap_clib/ParserRaw.cc
View File

@@ -18,7 +18,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_RAW);
const std::string ParserRaw::m_parserName = "ParserRaw";
ParserRaw::ParserRaw(IParserStreamReceiver &receiver, const std::string &key)
ParserRaw::ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key)
:m_receiver(receiver), m_key(key), m_state(s_start) {
}

2
components/security_apps/waap/waap_clib/ParserRaw.h
View File

@@ -19,7 +19,7 @@
class ParserRaw : public ParserBase {
public:
ParserRaw(IParserStreamReceiver &receiver, const std::string &key);
ParserRaw(IParserStreamReceiver &receiver, size_t parser_depth, const std::string &key);
virtual ~ParserRaw();
size_t push(const char *data, size_t data_len);
void finish();

54
components/security_apps/waap/waap_clib/ParserUrlEncode.cc
View File

@@ -16,27 +16,37 @@
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_URLENCODE);
USE_DEBUG_FLAG(D_WAAP);
const std::string ParserUrlEncode::m_parserName = "ParserUrlEncode";
ParserUrlEncode::ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar, bool should_decode_per)
:
ParserUrlEncode::ParserUrlEncode(
IParserStreamReceiver &receiver, size_t parser_depth, char separatorChar, bool should_decode_per
) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_separatorChar(separatorChar),
m_escapedCharCandidate(0),
should_decode_percent(should_decode_per)
should_decode_percent(should_decode_per),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_URLENCODE) << "should_decode_per=" << should_decode_per;
dbgTrace(D_WAAP)
<< "should_decode_percent="
<< should_decode_per
<< "parser_depth="
<< parser_depth;
// TODO:: is there a need for this?
memset(m_escaped, 0, sizeof(m_escaped));
}
ParserUrlEncode::~ParserUrlEncode() {
}
ParserUrlEncode::~ParserUrlEncode()
{}
size_t ParserUrlEncode::push(const char *buf, size_t len) {
size_t
ParserUrlEncode::push(const char *buf, size_t len)
{
size_t i = 0;
size_t mark = 0;
char c;
@@ -53,8 +63,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
m_state = s_error;
return i;
}
}
else if (m_state == s_value_start) {
} else if (m_state == s_value_start) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
@@ -76,8 +85,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < 32)
{
if (c < 32) {
dbgDebug(D_WAAP_PARSER_URLENCODE) << "invalid URL encoding character: " << c;
m_state = s_error;
return i;
@@ -119,8 +127,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_key_escaped1;
break;
}
else if (c == '+') {
} else if (c == '+') {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onKey(buf + mark, i - mark) != 0) {
@@ -140,8 +147,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_key_start;
break;
}
else {
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onKey(m_escaped, m_escapedLen) != 0) {
@@ -201,7 +207,6 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences
if (c != '%') {
// pass the non-hex character back to the output too.
if (m_receiver.onKey(&c, 1) != 0) {
return i;
@@ -279,8 +284,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_value_escaped1;
break;
}
else if (c == '+') {
} else if (c == '+') {
// convert plus character to space
if (i - mark > 0) {
if (m_receiver.onValue(buf + mark, i - mark) != 0) {
@@ -299,8 +303,7 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
}
m_state = s_value_start;
break;
}
else {
} else {
// flush unescaped data collected (if any)
if (m_escapedLen > 0) {
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
@@ -425,15 +428,20 @@ size_t ParserUrlEncode::push(const char *buf, size_t len) {
return len;
}
void ParserUrlEncode::finish() {
void
ParserUrlEncode::finish()
{
push(NULL, 0);
}
const std::string &
ParserUrlEncode::name() const {
ParserUrlEncode::name() const
{
return m_parserName;
}
bool ParserUrlEncode::error() const {
bool
ParserUrlEncode::error() const
{
return m_state == s_error;
}

7
components/security_apps/waap/waap_clib/ParserUrlEncode.h
View File

@@ -21,7 +21,11 @@
class ParserUrlEncode : public ParserBase {
public:
ParserUrlEncode(IParserStreamReceiver &receiver, char separatorChar = '&', bool should_decode_per = true);
ParserUrlEncode(
IParserStreamReceiver &receiver,
size_t parser_depth,
char separatorChar = '&',
bool should_decode_per = true);
virtual ~ParserUrlEncode();
size_t push(const char *data, size_t data_len);
void finish();
@@ -52,6 +56,7 @@ private:
char m_escapedCharCandidate;
bool should_decode_percent;
static const std::string m_parserName;
size_t m_parser_depth;
};
#endif // __PARSER_URL_ENCODE_H__29ebe806

22
components/security_apps/waap/waap_clib/ParserXML.cc
View File

@@ -60,9 +60,11 @@ void ParserXML::onStartElementNs(
if (p->m_receiver.onKv(
p->m_key.c_str(),
p->m_key.size(),
(const char*)attr_value_begin, attr_value_end - attr_value_begin,
BUFFERED_RECEIVER_F_BOTH
) != 0) {
(const char *)attr_value_begin,
attr_value_end - attr_value_begin,
BUFFERED_RECEIVER_F_BOTH,
p->m_parser_depth
) != 0) {
p->m_state = s_error;
}
p->m_key.pop("XML end attribute");
@@ -195,9 +197,17 @@ static void onError(void* ctx, const char* msg, ...) {
dbgTrace(D_WAAP_PARSER_XML) << "LIBXML (xml) onError: " << std::string(string);
}
ParserXML::ParserXML(IParserStreamReceiver& receiver)
:m_receiver(receiver), m_state(s_start), m_bufLen(0), m_key("xml_parser"), m_pushParserCtxPtr(NULL) {
dbgTrace(D_WAAP_PARSER_XML) << "ParserXML::ParserXML()";
ParserXML::ParserXML(IParserStreamReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
m_state(s_start),
m_bufLen(0),
m_key("xml_parser"),
m_pushParserCtxPtr(NULL),
m_parser_depth(parser_depth)
{
dbgTrace(D_WAAP_PARSER_XML)
<< "ParserXML::ParserXML() parser_depth="
<< parser_depth;
// TODO:: is zeroing this really needed?
memset(m_buf, 0, sizeof(m_buf));

3
components/security_apps/waap/waap_clib/ParserXML.h
View File

@@ -24,7 +24,7 @@
class ParserXML : public ParserBase {
public:
ParserXML(IParserStreamReceiver &receiver);
ParserXML(IParserStreamReceiver &receiver, size_t parser_depth);
virtual ~ParserXML();
size_t push(const char *data, size_t data_len);
void finish();
@@ -94,6 +94,7 @@ private:
std::vector<ElemTrackInfo> m_elemTrackStack;
xmlSAXHandler m_saxHandler;
xmlParserCtxtPtr m_pushParserCtxPtr;
size_t m_parser_depth;
public:
static const std::string m_parserName;
};

49
components/security_apps/waap/waap_clib/Serializator.cc
View File

@@ -96,7 +96,6 @@ Maybe<string> RestGetFile::genJson() const
return genError("Failed to compress data");
}
data = string((const char *)res.output, res.num_output_bytes);
json = data;
if (res.output) free(res.output);
@@ -179,10 +178,54 @@ void SerializeToFileBase::saveData()
serialize(ss);
string data = ss.str();
auto compression_stream = initCompressionStream();
CompressionResult res = compressData(
compression_stream,
CompressionType::GZIP,
data.size(),
reinterpret_cast<const unsigned char *>(data.c_str()),
true
);
finiCompressionStream(compression_stream);
if (!res.ok) {
dbgWarning(D_WAAP_CONFIDENCE_CALCULATOR) << "Failed to gzip data";
} else {
ss.str(string((const char *)res.output, res.num_output_bytes));
}
filestream << ss.str();
filestream.close();
}
string decompress(string fileContent) {
if (!isGZipped(fileContent)) {
dbgTrace(D_WAAP) << "file note zipped";
return fileContent;
}
auto compression_stream = initCompressionStream();
DecompressionResult res = decompressData(
compression_stream,
fileContent.size(),
reinterpret_cast<const unsigned char *>(fileContent.c_str())
);
finiCompressionStream(compression_stream);
if (res.ok) {
string decompressedData = string((const char *)res.output, res.num_output_bytes);
if (res.output) free(res.output);
res.output = nullptr;
res.num_output_bytes = 0;
return decompressedData;
}
return fileContent;
}
void SerializeToFileBase::loadFromFile(string filePath)
{
dbgTrace(D_WAAP_CONFIDENCE_CALCULATOR) << "loadFromFile() file: " << filePath;
@@ -239,8 +282,8 @@ void SerializeToFileBase::loadFromFile(string filePath)
delete[] buffer;
stringstream ss(dataObfuscated);
stringstream ss;
ss << decompress(dataObfuscated);
try
{

12
components/security_apps/waap/waap_clib/Signatures.cc
View File

@@ -85,13 +85,6 @@ static filtered_parameters_t to_filtermap(const picojson::value::object& JsObj)
return result;
}
std::string genDelimitedKeyValPattern(const std::string& delim)
{
std::string pattern = "^([^" + delim + "]+?=[^" + delim + "]+?" + delim + ")+"
"([^" + delim + "]+?=[^" + delim + "]+?)" + delim + "?$";
return pattern;
}
Signatures::Signatures(const std::string& filepath) :
sigsSource(loadSource(filepath)),
error(false),
@@ -207,11 +200,6 @@ Signatures::Signatures(const std::string& filepath) :
html_regex("(<(?>body|head)\\b.*>(?>.|[\\r\\n]){0,400}){2}|<html", error, "htmlRegex"),
uri_parser_regex("(http|https)://([^/ :]+):?([^/ ]*)(/?[^ #?]*)", error, "uriParserRegex"),
confluence_macro_re("{[^\"]+:(?>.+\\|)+.+}"),
pipes_delimited_key_val_re(genDelimitedKeyValPattern("\\|")),
semicolon_delimited_key_val_re(genDelimitedKeyValPattern(";")),
asterisk_delimited_key_val_re(genDelimitedKeyValPattern("\\*")),
comma_delimited_key_val_re(genDelimitedKeyValPattern(",")),
ampersand_delimited_key_val_re(genDelimitedKeyValPattern("&")),
headers_re(to_regexmap(sigsSource["headers_re"].get<JsObj>(), error)),
format_magic_binary_re(sigsSource["format_magic_binary_re"].get<std::string>(), error, "format_magic_binary_re"),
params_type_re(to_regexmap(sigsSource["format_types_regex_list"].get<JsObj>(), error)),

5
components/security_apps/waap/waap_clib/Signatures.h
View File

@@ -61,11 +61,6 @@ public:
const Regex html_regex;
const Regex uri_parser_regex;
const boost::regex confluence_macro_re;
const boost::regex pipes_delimited_key_val_re;
const boost::regex semicolon_delimited_key_val_re;
const boost::regex asterisk_delimited_key_val_re;
const boost::regex comma_delimited_key_val_re;
const boost::regex ampersand_delimited_key_val_re;
#if 0 // Removed by Pavel's request. Leaving here in case he'll want to add this back...
const std::set<std::string> cookie_ignored_keywords;
const std::set<std::string> cookie_ignored_patterns;

1
components/security_apps/waap/waap_clib/WaapConfigApi.cc
View File

@@ -59,7 +59,6 @@ WaapConfigAPI::clearAssetsCount()
void WaapConfigAPI::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
WaapConfigBase::load(ar);
assets_ids_aggregation.insert(m_assetId);
}

1
components/security_apps/waap/waap_clib/WaapConfigApplication.cc
View File

@@ -74,7 +74,6 @@ void WaapConfigApplication::load(cereal::JSONInputArchive& ar)
{
// order has affect - we need to call base last because of triggers and overrides
loadOpenRedirectPolicy(ar);
loadErrorDisclosurePolicy(ar);
loadCsrfPolicy(ar);

5
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -24,6 +24,8 @@
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
using boost::algorithm::to_lower_copy;
using namespace std;
@@ -253,12 +255,12 @@ void WaapConfigBase::loadOpenRedirectPolicy(cereal::JSONInputArchive& ar)
}
const std::vector<std::string> &
WaapConfigBase::get_applicationUrls() const
{
return m_applicationUrls;
}
void WaapConfigBase::loadErrorDisclosurePolicy(cereal::JSONInputArchive& ar)
{
std::string failMessage = "Failed to load the WAAP Information Disclosure policy";
@@ -432,6 +434,7 @@ const std::shared_ptr<Waap::OpenRedirect::Policy>& WaapConfigBase::get_OpenRedir
}
const std::shared_ptr<Waap::ErrorDisclosure::Policy>& WaapConfigBase::get_ErrorDisclosurePolicy() const
{
return m_errorDisclosurePolicy;

1
components/security_apps/waap/waap_clib/WaapConfigBase.h
View File

@@ -99,6 +99,7 @@ private:
std::vector<std::string> m_applicationUrls;
std::shared_ptr<Waap::ErrorDisclosure::Policy> m_errorDisclosurePolicy;
std::string m_schemaValidationPoicyStatusMessage;
std::string m_schemaUpdaterPoicyStatusMessage;
std::shared_ptr<Waap::Csrf::Policy> m_csrfPolicy;
std::shared_ptr<Waap::RateLimiting::Policy> m_rateLimitingPolicy;
std::shared_ptr<Waap::RateLimiting::Policy> m_errorLimitingPolicy;

14
components/security_apps/waap/waap_clib/WaapOverride.cc
View File

@@ -31,7 +31,7 @@ bool Match::operator==(const Match &other) const
}
Behavior::Behavior()
: m_action(""), m_log(""), m_sourceIdentifier("")
: m_id(""), m_action(""), m_log(""), m_sourceIdentifier("")
{
}
@@ -40,6 +40,11 @@ bool Behavior::operator==(const Behavior &other) const
return (m_action == other.m_action) && (m_log == other.m_log) && (m_sourceIdentifier == other.m_sourceIdentifier);
}
const std::string & Behavior::getParentId() const
{
return m_id;
}
const std::string & Behavior::getAction() const
{
return m_action;
@@ -55,6 +60,11 @@ const std::string& Behavior::getSourceIdentifier() const
return m_sourceIdentifier;
}
void Behavior::setParentId(const std::string& id)
{
m_id = id;
}
bool Rule::operator==(const Rule &other) const
{
return (m_match == other.m_match) &&
@@ -70,7 +80,9 @@ bool Policy::operator==(const Policy &other) const
State::State() :
bForceBlock(false),
forceBlockIds(),
bForceException(false),
forceExceptionIds(),
bIgnoreLog(false),
bSourceIdentifierOverride(false),
sSourceIdentifierMatch("")

18
components/security_apps/waap/waap_clib/WaapOverride.h
View File

@@ -175,10 +175,13 @@ public:
}
}
const std::string &getParentId() const;
const std::string &getAction() const;
const std::string &getLog() const;
const std::string &getSourceIdentifier() const;
void setParentId(const std::string& id);
private:
std::string m_id;
std::string m_action;
std::string m_log;
std::string m_sourceIdentifier;
@@ -195,20 +198,20 @@ public:
}
catch (const cereal::Exception &e)
{
dbgTrace(D_WAAP_OVERRIDE) << "An override rule has no id.";
dbgDebug(D_WAAP_OVERRIDE) << "An override rule has no id.";
m_id.clear();
}
ar(cereal::make_nvp("parsedMatch", m_match));
ar(cereal::make_nvp("parsedBehavior", m_behaviors));
m_isChangingRequestData = false;
for (std::vector<Waap::Override::Behavior>::const_iterator it = m_behaviors.begin();
for (std::vector<Waap::Override::Behavior>::iterator it = m_behaviors.begin();
it != m_behaviors.end();
++it)
{
const Behavior& behavior = *it;
Behavior& behavior = *it;
behavior.setParentId(m_id);
if (!behavior.getSourceIdentifier().empty()) // this rule changes data in request itself
{
m_isChangingRequestData = true;
@@ -223,8 +226,9 @@ public:
{
if (m_match.match(testFunctor)) {
// extend matchedBehaviors list with all behaviors on this rule
dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched. Adding " << m_behaviors.size() << " new behaviors:";
std::string overrideId = getId();
dbgTrace(D_WAAP_OVERRIDE) << "Override rule matched id: " << overrideId <<
". Adding " << m_behaviors.size() << " new behaviors:";
if (!overrideId.empty()) {
matchedOverrideIds.insert(overrideId);
}
@@ -308,8 +312,10 @@ private:
struct State {
// whether to force block regardless of stage2 response (and even if bSendRequest and/or bSendResponse are false)
bool bForceBlock;
std::set<std::string> forceBlockIds;
// exception (allow) was matched, so this request won't be blocked.
bool bForceException;
std::set<std::string> forceExceptionIds;
// overrides decision in case log should be ignored
bool bIgnoreLog;
// user identfier override to be applied
@@ -335,10 +341,12 @@ struct State {
if (matchedBehavior.getAction() == "accept") {
dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceException due to override behavior.";
bForceException = true;
forceExceptionIds.insert(matchedBehavior.getParentId());
}
else if (matchedBehavior.getAction() == "reject") {
dbgTrace(D_WAAP_OVERRIDE) << "applyOverride(): setting bForceBlock due to override behavior.";
bForceBlock = true;
forceBlockIds.insert(matchedBehavior.getParentId());
}
if (matchedBehavior.getLog() == "ignore")

135
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.cc
View File

@@ -15,81 +15,86 @@
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
namespace Waap {
ResponseInspectReasons::ResponseInspectReasons()
:
openRedirect(false),
errorDisclosure(false),
errorLimiter(false),
rateLimiting(false),
collectResponseForLog(false),
applyOverride(false)
{
}
ResponseInspectReasons::ResponseInspectReasons()
:
openRedirect(false),
errorDisclosure(false),
errorLimiter(false),
rateLimiting(false),
collectResponseForLog(false),
applyOverride(false)
{
}
bool
ResponseInspectReasons::shouldInspect() const
{
dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" <<
" OpenRedirect=" << openRedirect <<
" ErrorDisclosure=" << errorDisclosure <<
" RateLimiting=" << rateLimiting <<
" ErrorLimiter=" << errorLimiter <<
" collectResponseForLog=" << collectResponseForLog <<
" applyOverride=" << applyOverride;
return openRedirect || errorDisclosure || rateLimiting || errorLimiter || collectResponseForLog || applyOverride;
}
bool
ResponseInspectReasons::shouldInspect() const
{
dbgTrace(D_WAAP) << "ResponseInspectReasons::shouldInspect():" <<
" OpenRedirect=" << openRedirect <<
" ErrorDisclosure=" << errorDisclosure <<
" RateLimiting=" << rateLimiting <<
" ErrorLimiter=" << errorLimiter <<
" collectResponseForLog=" << collectResponseForLog <<
" applyOverride=" << applyOverride;
void
ResponseInspectReasons::setOpenRedirect(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag;
openRedirect = flag;
}
return
openRedirect || errorDisclosure || rateLimiting || errorLimiter ||
collectResponseForLog || applyOverride;
}
void
ResponseInspectReasons::setErrorDisclosure(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag;
errorDisclosure = flag;
}
void
ResponseInspectReasons::setOpenRedirect(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(OpenRedirect) " << openRedirect << " to " << flag;
openRedirect = flag;
}
void
ResponseInspectReasons::setRateLimiting(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag;
rateLimiting = flag;
}
void
ResponseInspectReasons::setErrorLimiter(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag;
errorLimiter = flag;
}
void
ResponseInspectReasons::setErrorDisclosure(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorDisclosure) " << errorDisclosure << " to " << flag;
errorDisclosure = flag;
}
void
ResponseInspectReasons::setCollectResponseForLog(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog << " to " <<
flag;
collectResponseForLog = flag;
}
void
ResponseInspectReasons::setRateLimiting(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(RateLimiting) " << rateLimiting << " to " << flag;
rateLimiting = flag;
}
void
ResponseInspectReasons::setApplyOverride(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " <<
flag;
applyOverride = flag;
}
void
ResponseInspectReasons::setErrorLimiter(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(ErrorLimiter) " << errorLimiter << " to " << flag;
errorLimiter = flag;
}
bool
ResponseInspectReasons::getApplyOverride(void)
{
return applyOverride;
}
void
ResponseInspectReasons::setCollectResponseForLog(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(collectResponseForLog) " << collectResponseForLog <<
" to " << flag;
collectResponseForLog = flag;
}
void
ResponseInspectReasons::setApplyOverride(bool flag)
{
dbgTrace(D_WAAP) << "Change ResponseInspectReasons(setApplyOverride) " << applyOverride << " to " <<
flag;
applyOverride = flag;
}
bool
ResponseInspectReasons::getApplyOverride(void)
{
return applyOverride;
}
}

1
components/security_apps/waap/waap_clib/WaapResponseInspectReasons.h
View File

@@ -25,6 +25,7 @@ public:
void setErrorLimiter(bool flag);
void setCollectResponseForLog(bool flag);
void setApplyOverride(bool flag);
bool getApplyOverride(void);
private:
bool openRedirect;

14
components/security_apps/waap/waap_clib/WaapScanner.cc
View File

@@ -17,8 +17,13 @@
#include <string>
#include "debug.h"
#include "reputation_features_events.h"
#include <boost/algorithm/string.hpp>
USE_DEBUG_FLAG(D_WAAP_SCANNER);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
// id generated by xml parser for an entity attribute
const std::string Waap::Scanner::xmlEntityAttributeId = "08a80340-06d3-11ea-9f87-0242ac11000f";
double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolName)
{
@@ -117,7 +122,7 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN
// Ignore scan results from specific fields on csp-report json in case those are not filtered by learning
bool Waap::Scanner::isKeyCspReport(const std::string &key, Waf2ScanResult &res, DeepParser &dp)
{
if (res.score < 8.0f && res.location == "body" && dp.getLastParser() == "jsonParser") {
if (res.score < 8.0f && res.location == "body" && dp.getActualParser(0) == "jsonParser") {
if (key == "csp-report.blocked-uri" || key == "csp-report.script-sample" ||
(key == "csp-report.original-policy" && Waap::Util::containsCspReportPolicy(res.unescaped_line)) ) {
dbgTrace(D_WAAP_SCANNER) << "CSP report detected, ignoring.";
@@ -169,11 +174,14 @@ bool Waap::Scanner::suspiciousHit(Waf2ScanResult& res, DeepParser &dp,
return m_transaction->reportScanResult(res);
}
int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) {
int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) {
Waf2ScanResult& res = m_lastScanResult;
DeepParser &dp = m_transaction->getDeepParser();
std::string key = std::string(k, k_len);
std::string value = std::string(v, v_len);
res.clear();
dbgTrace(D_WAAP_SCANNER) << "Waap::Scanner::onKv: k='" << key <<
"' v='" << value << "'";
@@ -266,7 +274,7 @@ int Waap::Scanner::onKv(const char* k, size_t k_len, const char* v, size_t v_len
}
// Special value only matched when XML <!ENTITY> atribute is found.
if (v_len == 36) {
if (value == "08a80340-06d3-11ea-9f87-0242ac11000f" && !m_transaction->shouldIgnoreOverride(res)) {
if (value == Waap::Scanner::xmlEntityAttributeId && !m_transaction->shouldIgnoreOverride(res)) {
// Always return max score when <!ENTITY tag is encoutered during XML parsing.
res.score = 10.0;
res.unescaped_line = "<!ENTITY";

6
components/security_apps/waap/waap_clib/WaapScanner.h
View File

@@ -32,13 +32,17 @@ namespace Waap {
m_bIgnoreOverride(false)
{
}
bool suspiciousHit(Waf2ScanResult &res, DeepParser &dp,
const std::string &location, const std::string &param_name, const std::string &key);
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags) override;
int onKv(const char* k, size_t k_len, const char* v, size_t v_len, int flags, size_t parser_depth) override;
const std::string &getAntibotCookie() const { return m_antibotCookie; }
bool getIgnoreOverride() { return m_bIgnoreOverride; };
const Waf2ScanResult &getLastScanResult() const { return m_lastScanResult; }
static const std::string xmlEntityAttributeId;
private:
double getScoreData(Waf2ScanResult& res, const std::string &poolName);
bool shouldIgnoreOverride(const Waf2ScanResult &res);

13
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc
View File

@@ -94,7 +94,8 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
canSplitSemicolon(true),
canSplitPipe(true),
hasSpace(false),
isUrlEncoded(false)
isUrlEncoded(false),
hasCharLess(false)
{
unsigned int zerosSeq[2] = {0};
bool lastNul = false; // whether last processed character was ASCII NUL
@@ -139,6 +140,12 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
case '|':
hasCharPipe = true;
break;
case '<':
hasCharLess = true;
break;
case '\"':
hasDoubleQuote = true;
break;
}
if (isspace(ch)) {
@@ -259,4 +266,8 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
textual +=(hasSpace ? "true" : "false");
textual.append("\nisUrlEncoded = ");
textual +=(isUrlEncoded ? "true" : "false");
textual.append("\nhasCharLess = ");
textual +=(hasCharLess ? "true" : "false");
textual.append("\nhasDoubleQuote = ");
textual +=(hasDoubleQuote ? "true" : "false");
}

2
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h
View File

@@ -34,6 +34,8 @@ struct ValueStatsAnalyzer
bool canSplitPipe;
bool hasSpace;
bool isUrlEncoded;
bool hasCharLess;
bool hasDoubleQuote;
std::string textual;
};

75
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -55,6 +55,8 @@
USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_ULIMITS);
USE_DEBUG_FLAG(D_WAAP_BOT_PROTECTION);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
using namespace ReportIS;
#define MAX_REQUEST_BODY_SIZE (2*1024)
@@ -529,6 +531,7 @@ void Waf2Transaction::set_method(const char* method) {
m_methodStr = method;
}
bool Waf2Transaction::checkIsScanningRequired()
{
bool result = false;
@@ -606,7 +609,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
bool firstPush = true;
// Parse URL
ParserRaw urlParser(m_deepParserReceiver, scanStage);
ParserRaw urlParser(m_deepParserReceiver, 0, scanStage);
// Scan the uri until '?' or ';' character found, whichever comes first (or until end of the uri string),
// Do not account for last character as valid separator
@@ -654,7 +657,6 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
pushed = true;
std::string url(uri);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
// We found no '?' character so set p to NULL to prevent parameters scan below.
@@ -677,7 +679,6 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
pushed = true;
std::string url(p, q-p);
Waap::Util::decodePercentEncoding(url);
urlParser.push(url.data(), url.size());
}
@@ -704,7 +705,7 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
bool ignoreScore = m_ignoreScore;
m_ignoreScore = true;
m_deepParser.m_key.push(scanStage.c_str(), scanStage.size());
ParserDelimiter uriSegmentsParser(m_deepParserReceiver, '/', scanStage);
ParserDelimiter uriSegmentsParser(m_deepParserReceiver, 0, '/', scanStage);
std::string baseUriUnescaped(baseUri);
Waap::Util::decodePercentEncoding(baseUriUnescaped);
uriSegmentsParser.push(baseUriUnescaped.c_str(), baseUriUnescaped.length());
@@ -749,7 +750,8 @@ void Waf2Transaction::processUri(const std::string &uri, const std::string& scan
std::string tag = scanStage + "_param";
m_deepParser.m_key.push(tag.data(), tag.size());
size_t buff_len = uriEnd - p;
ParserUrlEncode up(m_deepParserReceiver, paramSep, checkUrlEncoded(p, buff_len));
dbgTrace(D_WAAP) << "% will be encoded?'" << checkUrlEncoded(p, buff_len) << "'";
ParserUrlEncode up(m_deepParserReceiver, 0, paramSep, checkUrlEncoded(p, buff_len));
up.push(p, buff_len);
up.finish();
m_deepParser.m_key.pop(tag.c_str());
@@ -796,7 +798,7 @@ void Waf2Transaction::parseCookie(const char* value, int value_len)
if (value_len > 0) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the cookie value";
m_deepParser.m_key.push("cookie", 6);
ParserUrlEncode cookieValueParser(m_deepParserReceiver, ';');
ParserUrlEncode cookieValueParser(m_deepParserReceiver, 0, ';');
cookieValueParser.push(value, value_len);
cookieValueParser.finish();
m_deepParser.m_key.pop("cookie");
@@ -838,7 +840,7 @@ void Waf2Transaction::parseUnknownHeaderName(const char* name, int name_len)
!m_pWaapAssetState->getSignatures()->good_header_name_re.hasMatch(std::string(name, name_len))) {
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header name";
m_deepParser.m_key.push("header", 6);
ParserRaw headerNameParser(m_deepParserReceiver, std::string(name, name_len));
ParserRaw headerNameParser(m_deepParserReceiver, 0, std::string(name, name_len));
headerNameParser.push(name, name_len);
headerNameParser.finish();
m_deepParser.m_key.pop("header name");
@@ -857,7 +859,7 @@ void Waf2Transaction::parseGenericHeaderValue(const std::string &headerName, con
dbgTrace(D_WAAP) << "[transaction:" << this << "] scanning the header value";
m_deepParser.m_key.push("header", 6);
ParserRaw headerValueParser(m_deepParserReceiver, headerName);
ParserRaw headerValueParser(m_deepParserReceiver, 0, headerName);
headerValueParser.push(value, value_len);
headerValueParser.finish();
m_deepParser.m_key.pop("header value");
@@ -1077,7 +1079,7 @@ void Waf2Transaction::start_request_body() {
clearRequestParserState();
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, "body");
m_requestBodyParser = new ParserRaw(m_deepParserReceiver, 0, "body");
m_request_body_bytes_received = 0;
m_request_body.clear();
@@ -1188,6 +1190,7 @@ void Waf2Transaction::end_request() {
dbgTrace(D_WAAP) << "(Waf2Engine::end_request): Security Headers State was created";
}
// Enable response headers processing if response scanning is enabled in policy
auto errorDisclosurePolicy = m_siteConfig ? m_siteConfig->get_ErrorDisclosurePolicy() : NULL;
m_responseInspectReasons.setErrorDisclosure(errorDisclosurePolicy && errorDisclosurePolicy->enable);
@@ -1642,6 +1645,11 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog,
std::vector<std::string> vOverrideIds(m_matchedOverrideIds.size());
std::copy(m_matchedOverrideIds.begin(), m_matchedOverrideIds.end(), vOverrideIds.begin());
waapLog.addToOrigin(LogField("exceptionIdList", vOverrideIds));
if (!m_effectiveOverrideIds.empty()) {
std::vector<std::string> vEffectiveOverrideIds(m_effectiveOverrideIds.size());
std::copy(m_effectiveOverrideIds.begin(), m_effectiveOverrideIds.end(), vEffectiveOverrideIds.begin());
waapLog.addToOrigin(LogField("effectiveExceptionIdList", vEffectiveOverrideIds));
}
}
}
@@ -1697,13 +1705,18 @@ Waf2Transaction::sendLog()
return;
}
dbgTrace(D_WAAP) << "force exception: " << m_overrideState.bForceException <<
" force block: " << m_overrideState.bForceBlock <<
" matched overrides count: " << m_matchedOverrideIds.size() <<
" effective overrides count: " << m_effectiveOverrideIds.size();
bool shouldBlock = false;
if (m_overrideState.bForceBlock) {
// If override forces "reject" decision, mention it in the "override" log field.
logOverride = OVERRIDE_DROP;
shouldBlock = true;
}
else if (m_overrideState.bForceException) {
} else if (m_overrideState.bForceException) {
// If override forces "allow" decision, mention it in the "override" log field.
logOverride = OVERRIDE_ACCEPT;
} else if (m_scanner.getIgnoreOverride()) {
@@ -2024,6 +2037,9 @@ Waf2Transaction::decideAutonomousSecurity(
if (m_overrideState.bForceBlock) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces REJECT ...";
if (!decision->shouldBlock()) {
m_effectiveOverrideIds.insert(m_overrideState.forceBlockIds.begin(), m_overrideState.forceBlockIds.end());
}
decision->setBlock(true);
if (!m_overrideState.bIgnoreLog)
{
@@ -2033,6 +2049,17 @@ Waf2Transaction::decideAutonomousSecurity(
else if (m_overrideState.bForceException) {
dbgTrace(D_WAAP) << "decideAutonomousSecurity(): decision was " << decision->shouldBlock() <<
" and override forces ALLOW ...";
if (m_scanResult) {
// on accept exception the decision is not set and needs to be calculated to determine effectivness
ThreatLevel threat = Waap::Conversions::convertFinalScoreToThreatLevel(m_scanResult->score);
bool shouldBlock = Waap::Conversions::shouldDoWafBlocking(&sitePolicy, threat);
if (shouldBlock) {
m_effectiveOverrideIds.insert(
m_overrideState.forceExceptionIds.begin(), m_overrideState.forceExceptionIds.end()
);
}
}
decision->setBlock(false);
if (!m_overrideState.bIgnoreLog)
{
@@ -2041,8 +2068,13 @@ Waf2Transaction::decideAutonomousSecurity(
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO) {
bool log_all = false;
const std::shared_ptr<Waap::Trigger::Policy> triggerPolicy = sitePolicy.get_TriggerPolicy();
if (triggerPolicy) {
const std::shared_ptr<Waap::Trigger::Log> triggerLog = getTriggerLog(triggerPolicy);
if (triggerLog && triggerLog->webRequests) log_all = true;
}
if(decision->getThreatLevel() <= ThreatLevel::THREAT_INFO && !log_all) {
decision->setLog(false);
} else {
decision->setLog(true);
@@ -2171,8 +2203,10 @@ Waf2Transaction::reportScanResult(const Waf2ScanResult &res) {
bool
Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
auto exceptions = getConfiguration<ParameterException>("rulebase", "exception");
if (!exceptions.ok()) return false;
if (!exceptions.ok()) {
dbgInfo(D_WAAP_OVERRIDE) << "matching exceptions error:" << exceptions.getErr();
return false;
}
dbgTrace(D_WAAP_OVERRIDE) << "matching exceptions";
std::unordered_map<std::string, std::set<std::string>> exceptions_dict;
@@ -2212,13 +2246,20 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) {
// calling behavior and check if there is a behavior that match to this specific param name.
auto behaviors = exceptions.unpack().getBehavior(exceptions_dict,
getAssetState()->m_filtersMngr->getMatchedOverrideKeywords());
for (auto const &behavior : behaviors) {
if (behavior == action_ignore) {
for (const auto &behavior : behaviors) {
if (behavior == action_ignore)
{
dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore.";
std::string overrideId = behavior.getId();
if (!overrideId.empty()) {
m_matchedOverrideIds.insert(overrideId);
}
if (!res.keyword_matches.empty() || res.unescaped_line == Waap::Scanner::xmlEntityAttributeId)
{
if (!overrideId.empty()) {
m_effectiveOverrideIds.insert(overrideId);
}
}
return true;
}
}

2
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@@ -283,6 +283,7 @@ private:
// Matched override IDs
std::set<std::string> m_matchedOverrideIds;
std::set<std::string> m_effectiveOverrideIds;
//csrf state
Waap::CSRF::State m_csrfState;
@@ -344,7 +345,6 @@ private:
// Cached pointer to const triggerLog (hence mutable)
mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
Waf2TransactionFlags m_waf2TransactionFlags;
// Grace period for logging

4
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@@ -42,6 +42,7 @@ USE_DEBUG_FLAG(D_WAAP);
USE_DEBUG_FLAG(D_WAAP_EVASIONS);
USE_DEBUG_FLAG(D_WAAP_BASE64);
USE_DEBUG_FLAG(D_WAAP_JSON);
USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
#define MIN_HEX_LENGTH 6
#define charToDigit(c) (c - '0')
@@ -1186,7 +1187,7 @@ static const SingleRegex base64_key_value_detector_re(
err,
"base64_key_value");
static const SingleRegex json_key_value_detector_re(
"^[^<>{};,&\\?|=\\s]+={.+(?s):.+(?s)}\\z",
"\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]",
err,
"json_key_value");
static const SingleRegex base64_key_detector_re(
@@ -1444,6 +1445,7 @@ base64_variants b64Test (
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: FINAL key = '" << key << "'";
}
retVal = decodeBase64Chunk(s, start, s.end(), value);
dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After testing and conversion value = "
<< value << "retVal = '" << retVal <<"'";
if (!retVal) {