From 2419a36593cd25f2a65ed0ba9f0de43ad30de5a4 Mon Sep 17 00:00:00 2001 From: Felipe Zimmerle Date: Sun, 26 Feb 2017 23:32:48 -0300 Subject: [PATCH] First version of the inline highlight calculation --- .../blocked_request.conf | 2 +- .../blocked_request_engine_on.conf | 2 +- .../reading_logs_via_rule_message/match.conf | 2 +- .../no_match.conf | 2 +- .../reading_logs_via_rule_message.h | 44 ++++++- .../simple_request.cc | 2 +- headers/modsecurity/modsecurity.h | 2 +- headers/modsecurity/rule_message.h | 63 ++++++---- headers/modsecurity/transaction.h | 1 + src/rule.cc | 6 +- src/rule_message.cc | 111 ++++++++++++++++++ src/transaction.cc | 39 ++++++ 12 files changed, 241 insertions(+), 35 deletions(-) diff --git a/examples/reading_logs_via_rule_message/blocked_request.conf b/examples/reading_logs_via_rule_message/blocked_request.conf index 596b5640..2c2c3a40 100644 --- a/examples/reading_logs_via_rule_message/blocked_request.conf +++ b/examples/reading_logs_via_rule_message/blocked_request.conf @@ -1,3 +1,3 @@ -SecRule ARGS:param1 "test" "id:1,deny,phase:2,chain,msg:'test'" +SecRule ARGS:param1 "test" "id:1,deny,phase:2,t:lowercase,chain,msg:'test'" SecRule ARGS:param1 "test" "log" diff --git a/examples/reading_logs_via_rule_message/blocked_request_engine_on.conf b/examples/reading_logs_via_rule_message/blocked_request_engine_on.conf index af376c89..bc4cf9f4 100644 --- a/examples/reading_logs_via_rule_message/blocked_request_engine_on.conf +++ b/examples/reading_logs_via_rule_message/blocked_request_engine_on.conf @@ -1,2 +1,2 @@ SecRuleEngine On -SecRule ARGS:param1 "test" "id:1,deny" +SecRule ARGS:param1 "test" "id:1,deny,t:lowercase" diff --git a/examples/reading_logs_via_rule_message/match.conf b/examples/reading_logs_via_rule_message/match.conf index 471c484a..fc8d1004 100644 --- a/examples/reading_logs_via_rule_message/match.conf +++ b/examples/reading_logs_via_rule_message/match.conf @@ -1 +1 @@ -SecRule ARGS:param1 "test" "id:1,deny,msg:'this',msg:'is',msg:'a',msg:'test'" +SecRule ARGS:param1 "test" "id:1,deny,msg:'this',t:replaceNulls,msg:'is',msg:'a',msg:'test',t:lowercase,t:trim" diff --git a/examples/reading_logs_via_rule_message/no_match.conf b/examples/reading_logs_via_rule_message/no_match.conf index 51f239fb..db7588a1 100644 --- a/examples/reading_logs_via_rule_message/no_match.conf +++ b/examples/reading_logs_via_rule_message/no_match.conf @@ -1 +1 @@ -SecRule ARGS:param1 "WHEEE" "id:1,phase:2,deny,msg:'this',msg:'is',msg:'a',msg:'test'" +SecRule ARGS:param1 "WHEEE" "id:1,phase:2,deny,msg:'this',msg:'is',msg:'a',msg:'test',t:lower" diff --git a/examples/reading_logs_via_rule_message/reading_logs_via_rule_message.h b/examples/reading_logs_via_rule_message/reading_logs_via_rule_message.h index 501004d7..50d6b352 100644 --- a/examples/reading_logs_via_rule_message/reading_logs_via_rule_message.h +++ b/examples/reading_logs_via_rule_message/reading_logs_via_rule_message.h @@ -161,7 +161,6 @@ class ReadingLogsViaRuleMessage { pthread_join(threads[i], &status); std::cout << "Main: completed thread id :" << i << std::endl; } - delete rules; delete modsec; pthread_exit(NULL); @@ -170,6 +169,38 @@ end: return -1; } + + static std::string highlightToText( + const modsecurity::RuleMessageHighlight &h) { + std::cout << " * ModSecurity variable to be highlighted" << std::endl; + + for (const auto &i : h.m_variable) { + std::cout << " - From: " << std::to_string(i.m_startingAt); + std::cout << " to: " << std::to_string(i.m_startingAt + i.m_size); + std::cout << std::endl; + } + std::cout << std::endl; + + std::cout << " * Variable's values "; + std::cout << "(may include transformations)" << std::endl; + for (const auto &i : h.m_value) { + std::cout << " - " << i.first << ": " << i.second << std::endl; + } + std::cout << std::endl; + + std::cout << " * Operators match to be highlight inside "; + std::cout << "the variables (after transformations)" << std::endl; + + for (const auto &i : h.m_op) { + std::cout << " - From: " << i.m_area.m_startingAt; + std::cout << " to: " << std::to_string(i.m_area.m_startingAt \ + + i.m_area.m_size); + std::cout << " [Value: " << i.m_value << "]" << std::endl; + } + std::cout << std::endl; + return ""; + } + static void logCb(void *data, const void *ruleMessagev) { if (ruleMessagev == NULL) { std::cout << "I've got a call but the message was null ;("; @@ -194,6 +225,17 @@ end: std::cout << modsecurity::RuleMessage::log(ruleMessage); std::cout << std::endl; } + std::cout << std::endl; + std::cout << "Verbose details on the match highlight" << std::endl; + std::cout << " Highlight reference string: "; + std::cout << ruleMessage->m_reference << std::endl; + std::cout << std::endl; + std::cout << "Details:" << std::endl; + modsecurity::RuleMessageHighlight h = + modsecurity::RuleMessage::computeHighlight(ruleMessage, + ruleMessage->m_buf); + highlightToText(h); + std::cout << std::endl; } protected: diff --git a/examples/reading_logs_via_rule_message/simple_request.cc b/examples/reading_logs_via_rule_message/simple_request.cc index 302a3874..04b6e7c4 100644 --- a/examples/reading_logs_via_rule_message/simple_request.cc +++ b/examples/reading_logs_via_rule_message/simple_request.cc @@ -33,7 +33,7 @@ int main(int argc, char **argv) { std::string rules(*argv); ReadingLogsViaRuleMessage rlvrm(request_header, request_uri, request_body, - response_headers, response_body, ip, rules); + "", response_body, ip, rules); rlvrm.process(); diff --git a/headers/modsecurity/modsecurity.h b/headers/modsecurity/modsecurity.h index c17a2071..a5380c52 100644 --- a/headers/modsecurity/modsecurity.h +++ b/headers/modsecurity/modsecurity.h @@ -301,11 +301,11 @@ class ModSecurity { collection::Collection *m_ip_collection; collection::Collection *m_session_collection; collection::Collection *m_user_collection; + int m_logProperties; private: std::string m_connector; ModSecLogCb m_logCb; - int m_logProperties; }; diff --git a/headers/modsecurity/rule_message.h b/headers/modsecurity/rule_message.h index 1c9968a0..888897f3 100644 --- a/headers/modsecurity/rule_message.h +++ b/headers/modsecurity/rule_message.h @@ -24,6 +24,11 @@ #ifndef HEADERS_MODSECURITY_RULE_MESSAGE_H_ #define HEADERS_MODSECURITY_RULE_MESSAGE_H_ +#ifdef __cplusplus +#include +#endif + +#include "modsecurity/modsecurity.h" #include "modsecurity/transaction.h" #include "modsecurity/rule.h" @@ -32,32 +37,36 @@ namespace modsecurity { +class RuleMessageHighlightArea { + public: + RuleMessageHighlightArea() + : m_startingAt(0), + m_size(0) { } + size_t m_startingAt; + size_t m_size; +}; + + +class RuleMessageHighlightOperator { + public: + RuleMessageHighlightOperator() + : m_value("") { } + RuleMessageHighlightArea m_area; + std::string m_value; +}; + + +class RuleMessageHighlight { + public: + std::list m_variable; + std::list> m_value; + std::list m_op; +}; + + class RuleMessage { public: - explicit RuleMessage(Rule *rule, Transaction *trans) : - m_accuracy(rule->m_accuracy), - m_clientIpAddress(trans->m_clientIpAddress), - m_data(""), - m_disruptiveMessage(""), - m_id(trans->m_id), - m_isDisruptive(false), - m_match(""), - m_maturity(rule->m_maturity), - m_message(""), - m_noAuditLog(false), - m_phase(rule->m_phase - 1), - m_reference(""), - m_rev(rule->m_rev), - m_rule(rule), - m_ruleFile(rule->m_fileName), - m_ruleId(rule->m_ruleId), - m_ruleLine(rule->m_lineNumber), - m_saveMessage(true), - m_serverIpAddress(trans->m_serverIpAddress), - m_severity(0), - m_uriNoQueryStringDecoded(trans->m_uri_no_query_string_decoded), - m_ver(rule->m_ver) - { } + RuleMessage(Rule *rule, Transaction *trans); std::string errorLog() { return RuleMessage::errorLog(this); @@ -79,8 +88,11 @@ class RuleMessage { static std::string errorLogTail(const RuleMessage *rm); static std::string errorLog(const RuleMessage *rm); static std::string log(const RuleMessage *rm); + static RuleMessageHighlight computeHighlight(const RuleMessage *rm, + const std::string buf); int m_accuracy; + std::string m_buf; std::string m_clientIpAddress; std::string m_data; std::string m_disruptiveMessage; @@ -90,6 +102,7 @@ class RuleMessage { int m_maturity; std::string m_message; bool m_noAuditLog; + std::string m_opValue; int m_phase; std::string m_reference; std::string m_rev; @@ -101,9 +114,11 @@ class RuleMessage { std::string m_serverIpAddress; int m_severity; std::string m_uriNoQueryStringDecoded; + std::string m_varValue; std::string m_ver; std::list m_tags; + RuleMessageHighlight m_highlight; }; diff --git a/headers/modsecurity/transaction.h b/headers/modsecurity/transaction.h index adf39689..94f75d2a 100644 --- a/headers/modsecurity/transaction.h +++ b/headers/modsecurity/transaction.h @@ -329,6 +329,7 @@ class Transaction : public TransactionAnchoredVariables { int getRuleEngineState(); std::string toJSON(int parts); + std::string toBuf(); std::string toOldAuditLogFormat(int parts, const std::string &trailer); std::string toOldAuditLogFormatIndex(const std::string &filename, double size, const std::string &md5); diff --git a/src/rule.cc b/src/rule.cc index 83005eeb..fd66bed0 100644 --- a/src/rule.cc +++ b/src/rule.cc @@ -356,8 +356,7 @@ std::list, if (multiMatch == true) { if (*newValue != *value) { ret.push_back(std::make_pair( - newValue, - transStr)); + newValue, transStr)); } } value = std::shared_ptr(newValue); @@ -386,8 +385,7 @@ std::list, if (multiMatch == true) { if (*value != *newValue) { ret.push_back(std::make_pair( - newValue, - transStr)); + newValue, transStr)); value = newValue; } } diff --git a/src/rule_message.cc b/src/rule_message.cc index 952c0ff4..42761787 100644 --- a/src/rule_message.cc +++ b/src/rule_message.cc @@ -20,9 +20,40 @@ #include "modsecurity/modsecurity.h" #include "modsecurity/transaction.h" #include "src/utils/string.h" +#include "src/utils/regex.h" +#include "modsecurity/actions/action.h" +#include "src/actions/transformations/transformation.h" namespace modsecurity { +RuleMessage::RuleMessage(Rule *rule, Transaction *trans) : + m_accuracy(rule->m_accuracy), + m_buf(""), + m_clientIpAddress(trans->m_clientIpAddress), + m_data(""), + m_disruptiveMessage(""), + m_id(trans->m_id), + m_isDisruptive(false), + m_match(""), + m_maturity(rule->m_maturity), + m_message(""), + m_noAuditLog(false), + m_opValue(""), + m_phase(rule->m_phase - 1), + m_reference(""), + m_rev(rule->m_rev), + m_rule(rule), + m_ruleFile(rule->m_fileName), + m_ruleId(rule->m_ruleId), + m_ruleLine(rule->m_lineNumber), + m_saveMessage(true), + m_serverIpAddress(trans->m_serverIpAddress), + m_severity(0), + m_uriNoQueryStringDecoded(trans->m_uri_no_query_string_decoded), + m_varValue(""), + m_ver(rule->m_ver) { } + + std::string RuleMessage::disruptiveErrorLog(const RuleMessage *rm) { std::string msg; @@ -78,6 +109,7 @@ std::string RuleMessage::noClientErrorLog(const RuleMessage *rm) { return modsecurity::utils::string::toHexIfNeeded(msg); } + std::string RuleMessage::errorLogTail(const RuleMessage *rm) { std::string msg; @@ -89,6 +121,7 @@ std::string RuleMessage::errorLogTail(const RuleMessage *rm) { return modsecurity::utils::string::toHexIfNeeded(msg); } + std::string RuleMessage::errorLog(const RuleMessage *rm) { std::string msg; @@ -99,6 +132,7 @@ std::string RuleMessage::errorLog(const RuleMessage *rm) { return msg; } + std::string RuleMessage::log(const RuleMessage *rm) { std::string msg(""); if (rm->m_isDisruptive) { @@ -110,4 +144,81 @@ std::string RuleMessage::log(const RuleMessage *rm) { return msg; } + +RuleMessageHighlight RuleMessage::computeHighlight(const RuleMessage *rm, + const std::string buf) { + RuleMessageHighlight ret; + Utils::Regex variables("v([0-9]+),([0-9]+)"); + Utils::Regex operators("o([0-9]+),([0-9]+)"); + Utils::Regex transformations("t:(?:(?!t:).)+"); + + std::string ref(rm->m_reference); + std::list vars = variables.searchAll(ref); + std::list ops = operators.searchAll(ref); + std::list trans = transformations.searchAll(ref); + + std::string varValue; + + while (vars.size() > 0) { + std::string value; + RuleMessageHighlightArea a; + vars.pop_back(); + std::string startingAt = vars.back().match; + vars.pop_back(); + std::string size = vars.back().match; + vars.pop_back(); + a.m_startingAt = std::stoi(startingAt); + a.m_size = std::stoi(size); + ret.m_variable.push_back(a); + + if ((stoi(startingAt) + stoi(size)) > buf.size()) { + return ret; + } + + value = std::string(buf, stoi(startingAt), stoi(size)); + if (varValue.size() > 0) { + varValue.append(" " + value); + } else { + varValue.append(value); + } + } + + ret.m_value.push_back(std::make_pair("original value", varValue)); + while (trans.size() > 0) { + modsecurity::actions::transformations::Transformation *t; + std::string varValueRes; + std::string transformation = trans.back().match.c_str(); + t = actions::transformations::Transformation::instantiate( + transformation); + + varValueRes = t->evaluate(varValue, NULL); + varValue.assign(varValueRes); + ret.m_value.push_back(std::make_pair(transformation, varValue)); + trans.pop_back(); + delete t; + } + + while (ops.size() > 0) { + RuleMessageHighlightOperator o; + ops.pop_back(); + std::string startingAt = ops.back().match; + ops.pop_back(); + std::string size = ops.back().match; + ops.pop_back(); + + if ((stoi(startingAt) + stoi(size)) > buf.size()) { + return ret; + } + + o.m_area.m_startingAt = std::stoi(startingAt); + o.m_area.m_size = std::stoi(size); + o.m_value.assign(std::string(varValue, o.m_area.m_startingAt, + o.m_area.m_size)); + ret.m_op.push_back(o); + } + + return ret; +} + + } // namespace modsecurity diff --git a/src/transaction.cc b/src/transaction.cc index e3e4f3a4..a45f4a94 100644 --- a/src/transaction.cc +++ b/src/transaction.cc @@ -1504,6 +1504,45 @@ std::string Transaction::toOldAuditLogFormat(int parts, } +std::string Transaction::toBuf() { + std::string a; + + a.append(*m_variableRequestMethod.evaluate()); + a.append(" "); + a.append(m_uri); + a.append(" HTTP/"); + a.append(m_httpVersion); + a.append("\n"); + std::vector l; + m_variableRequestHeaders.resolve(&l); + for (auto h : l) { + size_t pos = strlen("REQUEST_HEADERS:"); + a.append((h->m_key.c_str() + pos)); + a.append(": "); + a.append((h->m_value.c_str())); + } + + a.append("\n\n"); + if (this->m_requestBody.str().length() > 0) { + a.append(this->m_requestBody.str().c_str()); + a.append("\n\n"); + } +#if 0 + l.clear(); + m_variableResponseHeaders.resolve(&l); + for (auto h : l) { + size_t pos = strlen("RESPONSE_HEADERS:"); + a.append((h->m_key->c_str() + pos)); + a.append(": "); + a.append((h->m_value->c_str())); + } + a.append("\n\n"); + a.append(this->m_responseBody.str().c_str()); +#endif + return a; +} + + std::string Transaction::toJSON(int parts) { #ifdef WITH_YAJL const unsigned char *buf;