github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-09-29 19:24:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

Aug_23_2023-Dev

Browse Source
This commit is contained in:
Ned Wright
2023-08-23 14:15:32 +00:00
parent 702c1184ea
commit b25fd8def5
115 changed files with 8292 additions and 1189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@@ -83,6 +83,7 @@ add_library(waap_clib
LogGenWrapper.cc
WaapSampleValue.cc
ParserGql.cc
ParserPercentEncode.cc
)
add_definitions("-Wno-unused-function")

58
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@@ -23,6 +23,7 @@
#include "ParserHTML.h"
#include "ParserBinary.h"
#include "ParserMultipartForm.h"
#include "ParserPercentEncode.h"
#include "ParserDelimiter.h"
#include "WaapAssetState.h"
#include "Waf2Regex.h"
@@ -232,16 +233,16 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
bool base64ParamFound = false;
dbgTrace(D_WAAP_DEEP_PARSER) << " ===Processing potential base64===";
std::string decoded_val, key;
std::string decoded_val, decoded_key;
base64_variants base64_status = Waap::Util::b64Test (cur_val,
key,
decoded_key,
decoded_val);
dbgTrace(D_WAAP_DEEP_PARSER)
<< " status = "
<< base64_status
<< " key = "
<< key
<< decoded_key
<< " value = "
<< decoded_val;
@@ -255,7 +256,7 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
if (decoded_val.size() > 0) {
cur_val = decoded_val;
base64ParamFound = true;
rc = onKv(key.c_str(), key.size(), cur_val.data(), cur_val.size(), flags);
rc = onKv(decoded_key.c_str(), decoded_key.size(), cur_val.data(), cur_val.size(), flags);
dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc;
if (rc != CONTINUE_PARSING) {
return rc;
@@ -284,11 +285,6 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
// Calculate various statistics over currently processed value
ValueStatsAnalyzer valueStats(cur_val_html_escaped);
if (valueStats.isUrlEncoded && !Waap::Util::testUrlBareUtf8Evasion(cur_val) &&
!Waap::Util::testUrlBadUtf8Evasion(cur_val)) {
Waap::Util::decodePercentEncoding(cur_val);
}
if (valueStats.canSplitPipe || valueStats.canSplitSemicolon)
{
std::string key = IndicatorsFiltersManager::generateKey(m_key.first(), m_key.str(), m_pTransaction);
@@ -373,14 +369,14 @@ int DeepParser::onKv(const char* k, size_t k_len, const char* v, size_t v_len, i
return rc;
}
if (Waap::Util::detectJSONasParameter(cur_val, key, decoded_val)) {
if (Waap::Util::detectJSONasParameter(cur_val, decoded_key, decoded_val)) {
dbgTrace(D_WAAP_DEEP_PARSER)
<< " detectJSONasParameter was true: key = "
<< key
<< decoded_key
<< " value = "
<< decoded_val;
rc = onKv(key.c_str(), key.size(), decoded_val.data(), decoded_val.size(), flags);
rc = onKv(decoded_key.c_str(), decoded_key.size(), decoded_val.data(), decoded_val.size(), flags);
dbgTrace(D_WAAP_DEEP_PARSER) << " After processing potential JSON rc = " << rc;
if (rc != CONTINUE_PARSING) {
@@ -746,6 +742,13 @@ void DeepParser::createInternalParser(const char *k, size_t k_len, std::string&
bool isUrlParamPayload,
int flags)
{
dbgTrace(D_WAAP_DEEP_PARSER)
<< "Starting create parsers for value: >>>"
<< cur_val
<< "<<<";
dbgTrace(D_WAAP_DEEP_PARSER)
<< "Stats:\n "
<< valueStats.textual;
bool isPipesType = false, isSemicolonType = false, isAsteriskType = false,
isCommaType = false, isAmperType = false;
bool isKeyValDelimited = false;
@@ -887,27 +890,36 @@ void DeepParser::createInternalParser(const char *k, size_t k_len, std::string&
// Note that this function must not add more than one parser
// because only the topmost parser will run on the value.
// Normally, DeepParser will take care of recursively run other parsers.
if (isHtmlType &&
if (valueStats.isUrlEncoded &&
!Waap::Util::testUrlBareUtf8Evasion(cur_val)) {
if (!valueStats.hasSpace &&
valueStats.hasCharAmpersand &&
valueStats.hasTwoCharsEqual &&
!isBinaryData()) {
dbgTrace(D_WAAP_DEEP_PARSER) << " Starting to parse an Url-encoded data";
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserUrlEncode>>(*this));
} else if (!Waap::Util::testUrlBadUtf8Evasion(cur_val)) {
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an percent decoding";
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserPercentEncode>>(*this));
}
} else if (isHtmlType &&
!isRefererPayload &&
!isUrlPayload)
{
!isUrlPayload) {
// HTML detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse an HTML file";
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserHTML>>(*this));
}
else if (cur_val.size() > 0 && signatures->php_serialize_identifier.hasMatch(cur_val))
{
} else if (cur_val.size() > 0 &&
signatures->php_serialize_identifier.hasMatch(cur_val)) {
// PHP value detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse phpSerializedData";
m_parsersDeque.push_front(std::make_shared<BufferedParser<PHPSerializedDataParser>>(*this));
}
else if (isPotentialGqlQuery && cur_val.size() > 0 && !validateJson(cur_val.data(), cur_val.size())) {
} else if (isPotentialGqlQuery &&
cur_val.size() > 0 &&
!validateJson(cur_val.data(), cur_val.size())) {
// Graphql value detected
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse graphql";
m_parsersDeque.push_front(std::make_shared<BufferedParser<ParserGql>>(*this));
}
else if (cur_val.length() > 0 && (cur_val[0] == '[' || cur_val[0] == '{'))
{
} else if (cur_val.length() > 0 && (cur_val[0] == '[' || cur_val[0] == '{')) {
boost::smatch confulence_match;
if (NGEN::Regex::regexMatch(__FILE__, __LINE__, cur_val, confulence_match, signatures->confluence_macro_re))

1
components/security_apps/waap/waap_clib/DeepParser.h
View File

@@ -34,6 +34,7 @@ public:
virtual int onKv(const char *k, size_t k_len, const char *v, size_t v_len, int flags);
void clear();
void showStats(std::string& buff, const ValueStatsAnalyzer& valueStats);
void apiProcessKey(const char *v, size_t v_len);
size_t depth() const;
void setGlobalMaxObjectDepth(size_t depth) { m_globalMaxObjectDepth = depth; }

326
components/security_apps/waap/waap_clib/ParserPercentEncode.cc Normal file
View File

@@ -0,0 +1,326 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "ParserPercentEncode.h"
#include "Waf2Util.h"
#include "debug.h"
USE_DEBUG_FLAG(D_WAAP_PARSER_PERCENT);
const std::string ParserPercentEncode::m_parserName = "ParserPercentEncode";
ParserPercentEncode::ParserPercentEncode(IParserStreamReceiver &receiver) :
m_receiver(receiver),
m_state(s_start),
m_escapedLen(0),
m_escapedCharCandidate(0)
{
memset(m_escaped, 0, sizeof(m_escaped));
}
ParserPercentEncode::~ParserPercentEncode()
{}
size_t
ParserPercentEncode::push(const char *buf, size_t len)
{
size_t i = 0;
size_t pointer_in_buffer = 0;
char c;
int is_last = 0;
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): starting (len=" << len << ")";
if (len == 0) {
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): end of data signal! m_state=" << m_state;
// flush unescaped data collected (if any)
if (m_escapedLen > 0)
{
if (m_state == s_value_start)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0) {
m_state = s_error;
return i;
}
}
m_escapedLen = 0;
}
if (m_receiver.onKvDone() != 0)
{
m_state = s_error;
return i;
}
return 0;
}
while (i < len)
{
c = buf[i];
is_last = (i == (len - 1));
// Checking valid char urlencode
if (c < VALID_URL_CODE_START)
{
dbgDebug(D_WAAP_PARSER_PERCENT)
<< "invalid URL encoding character: "
<< c;
m_state = s_error;
return i;
}
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): state="
<< m_state
<< "; ch='"
<< c
<< "'";
switch (m_state)
{
case s_start:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_start";
// fallthrough //
CP_FALL_THROUGH;
}
case s_value_start:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_value_start";
pointer_in_buffer = i;
m_state = s_value;
// fallthrough //
CP_FALL_THROUGH;
}
case s_value:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_value";
if (c == '%')
{
if (i - pointer_in_buffer > 0)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
if (m_receiver.onValue(buf + pointer_in_buffer, i - pointer_in_buffer) != 0)
{
m_state = s_error;
return i;
}
}
m_state = s_value_escaped1;
break;
}
else
{
// flush unescaped data collected (if any)
if (m_escapedLen > 0)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
return i;
}
m_escapedLen = 0;
pointer_in_buffer = i;
}
}
if (is_last)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< (buf + pointer_in_buffer)
<< "<<<";
if (m_receiver.onValue(buf + pointer_in_buffer, (i - pointer_in_buffer) + 1) != 0)
{
m_state = s_error;
return i;
}
}
break;
}
case s_value_escaped1:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_value_escaped1";
bool valid;
unsigned char v = from_hex(c, valid);
// character right after the '%' is not a valid hex char.
if (!valid)
{
// dump escaped chars
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT) << "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
{
return i;
}
// If the character is '%' - stay in the same state (correctly treat '%%%%hhh' sequences)
if (c != '%')
{
// pass the non-hex character back to the output too.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< c
<< "<<<";
if (m_receiver.onValue(&c, 1) != 0)
{
return i;
}
// otherwise (the character is not '%'), switch back to the s_value state
m_state = s_value_start;
}
break;
}
m_escapedCharCandidate = c;
m_escaped[m_escapedLen] = v << 4;
m_state = s_value_escaped2;
break;
}
case s_value_escaped2:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_value_escaped2";
bool valid;
unsigned char v = from_hex(c, valid);
if (!valid)
{
// This situation (2nd character is not valid hex) is not treated right now.
// In this case, v will be equal to 0 and output character will be invalid one.
// dump escaped chars
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
if (m_escapedLen > 0
&& m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
return i;
}
m_escapedLen = 0;
// return the '%' character back to the output.
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< "%"
<< "<<<";
if (m_receiver.onValue("%", 1) != 0)
{
return i;
}
// add the character that was thought to be escaped value
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escapedCharCandidate
<< "<<<";
if (m_receiver.onValue(&m_escapedCharCandidate, 1))
{
return i;
}
// re parse the character as a key (i is incremented back to current value)
i--;
m_state = s_value_start;
break;
}
m_escapedCharCandidate = 0;
m_escaped[m_escapedLen] |= v;
m_escapedLen++;
if (m_escapedLen >= MAX_PERCENT_ENCODED_SIZE)
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): call onValue with m_escaped = >>>"
<< m_escaped
<< "<<<";
if (m_receiver.onValue(m_escaped, m_escapedLen) != 0)
{
m_state = s_error;
return i;
}
m_escapedLen = 0;
}
m_state = s_value_start;
break;
}
case s_error:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): s_error";
return 0;
}
default:
{
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): URL parser unrecoverable error";
m_state = s_error;
return 0;
}
}
++i;
}
dbgTrace(D_WAAP_PARSER_PERCENT)
<< "ParserPercentEncode::push(): finished: len="
<< len;
return len;
}
void
ParserPercentEncode::finish()
{
push(NULL, 0);
}
const std::string &
ParserPercentEncode::name() const
{
return m_parserName;
}
bool
ParserPercentEncode::error() const
{
return m_state == s_error;
}

58
components/security_apps/waap/waap_clib/ParserPercentEncode.h Normal file
View File

@@ -0,0 +1,58 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __PARSER_PERCENT_ENCODE_H_
#define __PARSER_PERCENT_ENCODE_H_
#include "ParserBase.h"
#include <string.h>
#define MAX_PERCENT_ENCODED_SIZE 255
#define VALID_URL_CODE_START 32
class ParserPercentEncode : public ParserBase {
public:
ParserPercentEncode(IParserStreamReceiver &receiver);
virtual ~ParserPercentEncode();
size_t push(const char *data, size_t data_len);
void finish();
virtual const std::string &name() const;
bool error() const;
virtual size_t
depth()
{
return 1;
}
private:
enum state
{
s_start,
s_value_start,
s_value,
s_value_escaped1,
s_value_escaped2,
s_end,
s_error
};
IParserStreamReceiver &m_receiver;
enum state m_state;
unsigned char m_escapedLen;
char m_escaped[MAX_PERCENT_ENCODED_SIZE];
char m_escapedCharCandidate;
static const std::string m_parserName;
};
#endif

6
components/security_apps/waap/waap_clib/ScoreBuilder.cc
View File

@@ -348,12 +348,12 @@ void ScoreBuilder::calcScore(const std::string &poolName)
void ScoreBuilder::snap()
{
// Copy data from all mutable score pools to "snapshot" keyword->scores map
for (const std::pair<std::string, KeywordsScorePool> &pool : m_keywordsScorePools) {
for (const auto &pool : m_keywordsScorePools) {
const std::string &poolName = pool.first;
const KeywordsScorePool& keywordScorePool = pool.second;
m_snapshotKwScoreMap[poolName];
for (const std::pair<std::string, KeywordData> &kwData : keywordScorePool.m_keywordsDataMap)
for (const auto &kwData : keywordScorePool.m_keywordsDataMap)
{
const std::string &kwName = kwData.first;
double kwScore = kwData.second.score;
@@ -408,7 +408,7 @@ unsigned int ScoreBuilder::getFpStoreCount()
void ScoreBuilder::mergeScores(const ScoreBuilder& baseScores)
{
for (const std::pair<std::string, KeywordsScorePool> &pool : baseScores.m_keywordsScorePools) {
for (const auto &pool : baseScores.m_keywordsScorePools) {
const std::string &poolName = pool.first;
if (m_keywordsScorePools.find(poolName) == m_keywordsScorePools.end()) {
m_keywordsScorePools[poolName];

9
components/security_apps/waap/waap_clib/WaapConfigBase.cc
View File

@@ -78,6 +78,15 @@ void WaapConfigBase::readJSONByCereal(cereal::JSONInputArchive& ar)
cereal::make_nvp("ruleName", m_ruleName)
);
try {
std::string application_urls;
ar(cereal::make_nvp("applicationUrls", application_urls));
m_applicationUrls = split(application_urls, ';');
} catch (std::runtime_error& e) {
dbgWarning(D_WAAP) << "Error to load applicationUrls field in policy" << e.what();
ar.setNextName(nullptr);
}
m_blockingLevel = blockingLevelBySensitivityStr(m_autonomousSecurityLevel);
}

1
components/security_apps/waap/waap_clib/WaapConfigBase.h
View File

@@ -95,6 +95,7 @@ private:
std::shared_ptr<Waap::TrustedSources::TrustedSourcesParameter> m_trustedSourcesPolicy;
std::shared_ptr<Waap::Parameters::WaapParameters> m_waapParameters;
std::shared_ptr<Waap::OpenRedirect::Policy> m_openRedirectPolicy;
std::vector<std::string> m_applicationUrls;
std::shared_ptr<Waap::ErrorDisclosure::Policy> m_errorDisclosurePolicy;
std::string m_schemaValidationPoicyStatusMessage;
std::shared_ptr<Waap::Csrf::Policy> m_csrfPolicy;

26
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.cc
View File

@@ -233,4 +233,30 @@ ValueStatsAnalyzer::ValueStatsAnalyzer(const std::string &cur_val)
}
// Detect URLEncode value
isUrlEncoded = checkUrlEncoded(cur_val.data(), cur_val.size());
textual.clear();
textual.append("hasCharSlash = ");
textual +=(hasCharSlash ? "true" : "false");
textual.append("\nhasCharColon = ");
textual +=(hasCharColon ? "true" : "false");
textual.append("\nhasCharAmpersand = ");
textual +=(hasCharAmpersand ? "true" : "false");
textual.append("\nhasCharEqual = ");
textual +=(hasCharEqual ? "true" : "false");
textual.append("\nhasTwoCharsEqual = ");
textual +=(hasTwoCharsEqual ? "true" : "false");
textual.append("\nhasCharSemicolon = ");
textual +=(hasCharSemicolon ? "true" : "false");
textual.append("\nhasCharPipe = ");
textual +=(hasCharPipe ? "true" : "false");
textual.append("\nisUTF16 = ");
textual +=(isUTF16 ? "true" : "false");
textual.append("\ncanSplitSemicolon = ");
textual +=(canSplitSemicolon ? "true" : "false");
textual.append("\ncanSplitPipe = ");
textual +=(canSplitPipe ? "true" : "false");
textual.append("\nhasSpace = ");
textual +=(hasSpace ? "true" : "false");
textual.append("\nisUrlEncoded = ");
textual +=(isUrlEncoded ? "true" : "false");
}

1
components/security_apps/waap/waap_clib/WaapValueStatsAnalyzer.h
View File

@@ -34,6 +34,7 @@ struct ValueStatsAnalyzer
bool canSplitPipe;
bool hasSpace;
bool isUrlEncoded;
std::string textual;
};

2
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@@ -2277,7 +2277,7 @@ void Waf2Transaction::collectFoundPatterns()
{
if (m_scanResult)
{
for (const std::pair<std::string, std::vector<std::string>> &found_pattern : m_scanResult->found_patterns)
for (const auto &found_pattern : m_scanResult->found_patterns)
{
const std::string &regex_name = found_pattern.first; // the regex name (key)
m_found_patterns.insert(regex_name);

4
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@@ -1044,14 +1044,14 @@ namespace Util {
// trim from start
static inline std::string &ltrim(std::string &s) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(),
std::not1(std::ptr_fun<int, int>(std::isspace))));
[] (char c) { return !std::isspace(c); }));
return s;
}
// trim from end
static inline std::string &rtrim(std::string &s) {
s.erase(std::find_if(s.rbegin(), s.rend(),
std::not1(std::ptr_fun<int, int>(std::isspace))).base(), s.end());
[] (char c) { return !std::isspace(c); }).base(), s.end());
return s;
}