First version of the inline highlight calculation

examples/reading_logs_via_rule_message/blocked_request.conf

@@ -1,3 +1,3 @@
-SecRule ARGS:param1 "test" "id:1,deny,phase:2,chain,msg:'test'"
+SecRule ARGS:param1 "test" "id:1,deny,phase:2,t:lowercase,chain,msg:'test'"
 SecRule ARGS:param1 "test" "log"
 

examples/reading_logs_via_rule_message/blocked_request_engine_on.conf

@@ -1,2 +1,2 @@
 SecRuleEngine On
-SecRule ARGS:param1 "test" "id:1,deny"
+SecRule ARGS:param1 "test" "id:1,deny,t:lowercase"

examples/reading_logs_via_rule_message/match.conf

@@ -1 +1 @@
-SecRule ARGS:param1 "test" "id:1,deny,msg:'this',msg:'is',msg:'a',msg:'test'"
+SecRule ARGS:param1 "test" "id:1,deny,msg:'this',t:replaceNulls,msg:'is',msg:'a',msg:'test',t:lowercase,t:trim"

examples/reading_logs_via_rule_message/no_match.conf

@@ -1 +1 @@
-SecRule ARGS:param1 "WHEEE" "id:1,phase:2,deny,msg:'this',msg:'is',msg:'a',msg:'test'"
+SecRule ARGS:param1 "WHEEE" "id:1,phase:2,deny,msg:'this',msg:'is',msg:'a',msg:'test',t:lower"

examples/reading_logs_via_rule_message/reading_logs_via_rule_message.h

@@ -161,7 +161,6 @@ class ReadingLogsViaRuleMessage {
             pthread_join(threads[i], &status);
             std::cout << "Main: completed thread id :" << i << std::endl;
         }
-
         delete rules;
         delete modsec;
         pthread_exit(NULL);
@@ -170,6 +169,38 @@ end:
         return -1;
     }
 
+
+    static std::string highlightToText(
+        const modsecurity::RuleMessageHighlight &h) {
+        std::cout << " * ModSecurity variable to be highlighted" << std::endl;
+
+        for (const auto &i : h.m_variable) {
+            std::cout << "  - From: " << std::to_string(i.m_startingAt);
+            std::cout << " to: " << std::to_string(i.m_startingAt + i.m_size);
+            std::cout << std::endl;
+        }
+        std::cout << std::endl;
+
+        std::cout << " * Variable's values ";
+        std::cout << "(may include transformations)" << std::endl;
+        for (const auto &i : h.m_value) {
+            std::cout << "  - " << i.first << ": " << i.second << std::endl;
+        }
+        std::cout << std::endl;
+
+        std::cout << " * Operators match to be highlight inside ";
+        std::cout << "the variables (after transformations)" << std::endl;
+
+        for (const auto &i : h.m_op) {
+            std::cout << "  - From: " << i.m_area.m_startingAt;
+            std::cout << " to: " << std::to_string(i.m_area.m_startingAt \
+                + i.m_area.m_size);
+            std::cout << " [Value: " << i.m_value << "]" << std::endl;
+        }
+        std::cout << std::endl;
+        return "";
+    }
+
     static void logCb(void *data, const void *ruleMessagev) {
         if (ruleMessagev == NULL) {
             std::cout << "I've got a call but the message was null ;(";
@@ -194,6 +225,17 @@ end:
             std::cout << modsecurity::RuleMessage::log(ruleMessage);
             std::cout << std::endl;
         }
+        std::cout << std::endl;
+        std::cout << "Verbose details on the match highlight" << std::endl;
+        std::cout << "  Highlight reference string: ";
+        std::cout << ruleMessage->m_reference << std::endl;
+        std::cout << std::endl;
+        std::cout << "Details:" << std::endl;
+        modsecurity::RuleMessageHighlight h =
+            modsecurity::RuleMessage::computeHighlight(ruleMessage,
+                ruleMessage->m_buf);
+        highlightToText(h);
+        std::cout << std::endl;
     }
 
  protected:

examples/reading_logs_via_rule_message/simple_request.cc

@@ -33,7 +33,7 @@ int main(int argc, char **argv) {
 
     std::string rules(*argv);
     ReadingLogsViaRuleMessage rlvrm(request_header, request_uri, request_body,
-        response_headers, response_body, ip, rules);
+        "", response_body, ip, rules);
     rlvrm.process();
 
 

headers/modsecurity/modsecurity.h

@@ -301,11 +301,11 @@ class ModSecurity {
     collection::Collection *m_ip_collection;
     collection::Collection *m_session_collection;
     collection::Collection *m_user_collection;
+    int m_logProperties;
 
  private:
     std::string m_connector;
     ModSecLogCb m_logCb;
-    int m_logProperties;
 };
 
 

headers/modsecurity/rule_message.h

@@ -24,6 +24,11 @@
 #ifndef HEADERS_MODSECURITY_RULE_MESSAGE_H_
 #define HEADERS_MODSECURITY_RULE_MESSAGE_H_
 
+#ifdef __cplusplus
+#include <utility>
+#endif
+
+#include "modsecurity/modsecurity.h"
 #include "modsecurity/transaction.h"
 #include "modsecurity/rule.h"
 
@@ -32,32 +37,36 @@
 
 namespace modsecurity {
 
+class RuleMessageHighlightArea {
+ public:
+    RuleMessageHighlightArea()
+        : m_startingAt(0),
+        m_size(0) { }
+    size_t m_startingAt;
+    size_t m_size;
+};
+
+
+class RuleMessageHighlightOperator {
+ public:
+    RuleMessageHighlightOperator()
+        : m_value("") { }
+    RuleMessageHighlightArea m_area;
+    std::string m_value;
+};
+
+
+class RuleMessageHighlight {
+ public:
+    std::list<RuleMessageHighlightArea> m_variable;
+    std::list<std::pair<std::string, std::string>> m_value;
+    std::list<RuleMessageHighlightOperator> m_op;
+};
+
+
 class RuleMessage {
  public:
-    explicit RuleMessage(Rule *rule, Transaction *trans) :
-        m_accuracy(rule->m_accuracy),
-        m_clientIpAddress(trans->m_clientIpAddress),
-        m_data(""),
-        m_disruptiveMessage(""),
-        m_id(trans->m_id),
-        m_isDisruptive(false),
-        m_match(""),
-        m_maturity(rule->m_maturity),
-        m_message(""),
-        m_noAuditLog(false),
-        m_phase(rule->m_phase - 1),
-        m_reference(""),
-        m_rev(rule->m_rev),
-        m_rule(rule),
-        m_ruleFile(rule->m_fileName),
-        m_ruleId(rule->m_ruleId),
-        m_ruleLine(rule->m_lineNumber),
-        m_saveMessage(true),
-        m_serverIpAddress(trans->m_serverIpAddress),
-        m_severity(0),
-        m_uriNoQueryStringDecoded(trans->m_uri_no_query_string_decoded),
-        m_ver(rule->m_ver)
-    { }
+    RuleMessage(Rule *rule, Transaction *trans);
 
     std::string errorLog() {
         return RuleMessage::errorLog(this);
@@ -79,8 +88,11 @@ class RuleMessage {
     static std::string errorLogTail(const RuleMessage *rm);
     static std::string errorLog(const RuleMessage *rm);
     static std::string log(const RuleMessage *rm);
+    static RuleMessageHighlight computeHighlight(const RuleMessage *rm,
+        const std::string buf);
 
     int m_accuracy;
+    std::string m_buf;
     std::string m_clientIpAddress;
     std::string m_data;
     std::string m_disruptiveMessage;
@@ -90,6 +102,7 @@ class RuleMessage {
     int m_maturity;
     std::string m_message;
     bool m_noAuditLog;
+    std::string m_opValue;
     int m_phase;
     std::string m_reference;
     std::string m_rev;
@@ -101,9 +114,11 @@ class RuleMessage {
     std::string m_serverIpAddress;
     int m_severity;
     std::string m_uriNoQueryStringDecoded;
+    std::string m_varValue;
     std::string m_ver;
 
     std::list<std::string> m_tags;
+    RuleMessageHighlight m_highlight;
 };
 
 

headers/modsecurity/transaction.h

@@ -329,6 +329,7 @@ class Transaction : public TransactionAnchoredVariables {
     int getRuleEngineState();
 
     std::string toJSON(int parts);
+    std::string toBuf();
     std::string toOldAuditLogFormat(int parts, const std::string &trailer);
     std::string toOldAuditLogFormatIndex(const std::string &filename,
         double size, const std::string &md5);

src/rule.cc

@@ -356,8 +356,7 @@ std::list<std::pair<std::shared_ptr<std::string>,
                 if (multiMatch == true) {
                     if (*newValue != *value) {
                         ret.push_back(std::make_pair(
-                            newValue,
-                                          transStr));
+                            newValue, transStr));
                     }
                 }
                 value = std::shared_ptr<std::string>(newValue);
@@ -386,8 +385,7 @@ std::list<std::pair<std::shared_ptr<std::string>,
             if (multiMatch == true) {
                 if (*value != *newValue) {
                     ret.push_back(std::make_pair(
-                        newValue,
-                                      transStr));
+                        newValue, transStr));
                     value = newValue;
                 }
             }

src/rule_message.cc

@@ -20,9 +20,40 @@
 #include "modsecurity/modsecurity.h"
 #include "modsecurity/transaction.h"
 #include "src/utils/string.h"
+#include "src/utils/regex.h"
+#include "modsecurity/actions/action.h"
+#include "src/actions/transformations/transformation.h"
 
 namespace modsecurity {
 
+RuleMessage::RuleMessage(Rule *rule, Transaction *trans) :
+    m_accuracy(rule->m_accuracy),
+    m_buf(""),
+    m_clientIpAddress(trans->m_clientIpAddress),
+    m_data(""),
+    m_disruptiveMessage(""),
+    m_id(trans->m_id),
+    m_isDisruptive(false),
+    m_match(""),
+    m_maturity(rule->m_maturity),
+    m_message(""),
+    m_noAuditLog(false),
+    m_opValue(""),
+    m_phase(rule->m_phase - 1),
+    m_reference(""),
+    m_rev(rule->m_rev),
+    m_rule(rule),
+    m_ruleFile(rule->m_fileName),
+    m_ruleId(rule->m_ruleId),
+    m_ruleLine(rule->m_lineNumber),
+    m_saveMessage(true),
+    m_serverIpAddress(trans->m_serverIpAddress),
+    m_severity(0),
+    m_uriNoQueryStringDecoded(trans->m_uri_no_query_string_decoded),
+    m_varValue(""),
+    m_ver(rule->m_ver) { }
+
+
 std::string RuleMessage::disruptiveErrorLog(const RuleMessage *rm) {
     std::string msg;
 
@@ -78,6 +109,7 @@ std::string RuleMessage::noClientErrorLog(const RuleMessage *rm) {
     return modsecurity::utils::string::toHexIfNeeded(msg);
 }
 
+
 std::string RuleMessage::errorLogTail(const RuleMessage *rm) {
     std::string msg;
 
@@ -89,6 +121,7 @@ std::string RuleMessage::errorLogTail(const RuleMessage *rm) {
     return modsecurity::utils::string::toHexIfNeeded(msg);
 }
 
+
 std::string RuleMessage::errorLog(const RuleMessage *rm) {
     std::string msg;
 
@@ -99,6 +132,7 @@ std::string RuleMessage::errorLog(const RuleMessage *rm) {
     return msg;
 }
 
+
 std::string RuleMessage::log(const RuleMessage *rm) {
     std::string msg("");
     if (rm->m_isDisruptive) {
@@ -110,4 +144,81 @@ std::string RuleMessage::log(const RuleMessage *rm) {
     return msg;
 }
 
+
+RuleMessageHighlight RuleMessage::computeHighlight(const RuleMessage *rm,
+    const std::string buf) {
+    RuleMessageHighlight ret;
+    Utils::Regex variables("v([0-9]+),([0-9]+)");
+    Utils::Regex operators("o([0-9]+),([0-9]+)");
+    Utils::Regex transformations("t:(?:(?!t:).)+");
+
+    std::string ref(rm->m_reference);
+    std::list<Utils::SMatch> vars = variables.searchAll(ref);
+    std::list<Utils::SMatch> ops = operators.searchAll(ref);
+    std::list<Utils::SMatch> trans = transformations.searchAll(ref);
+
+    std::string varValue;
+
+    while (vars.size() > 0) {
+        std::string value;
+        RuleMessageHighlightArea a;
+        vars.pop_back();
+        std::string startingAt = vars.back().match;
+        vars.pop_back();
+        std::string size = vars.back().match;
+        vars.pop_back();
+        a.m_startingAt = std::stoi(startingAt);
+        a.m_size = std::stoi(size);
+        ret.m_variable.push_back(a);
+
+        if ((stoi(startingAt) + stoi(size)) > buf.size()) {
+            return ret;
+        }
+
+        value = std::string(buf, stoi(startingAt), stoi(size));
+        if (varValue.size() > 0) {
+            varValue.append(" " + value);
+        } else {
+            varValue.append(value);
+        }
+    }
+
+    ret.m_value.push_back(std::make_pair("original value", varValue));
+    while (trans.size() > 0) {
+        modsecurity::actions::transformations::Transformation *t;
+        std::string varValueRes;
+        std::string transformation = trans.back().match.c_str();
+        t = actions::transformations::Transformation::instantiate(
+            transformation);
+
+        varValueRes = t->evaluate(varValue, NULL);
+        varValue.assign(varValueRes);
+        ret.m_value.push_back(std::make_pair(transformation, varValue));
+        trans.pop_back();
+        delete t;
+    }
+
+    while (ops.size() > 0) {
+        RuleMessageHighlightOperator o;
+        ops.pop_back();
+        std::string startingAt = ops.back().match;
+        ops.pop_back();
+        std::string size = ops.back().match;
+        ops.pop_back();
+
+        if ((stoi(startingAt) + stoi(size)) > buf.size()) {
+            return ret;
+        }
+
+        o.m_area.m_startingAt = std::stoi(startingAt);
+        o.m_area.m_size = std::stoi(size);
+        o.m_value.assign(std::string(varValue, o.m_area.m_startingAt,
+            o.m_area.m_size));
+        ret.m_op.push_back(o);
+    }
+
+    return ret;
+}
+
+
 }  // namespace modsecurity

src/transaction.cc

@@ -1504,6 +1504,45 @@ std::string Transaction::toOldAuditLogFormat(int parts,
 }
 
 
+std::string Transaction::toBuf() {
+    std::string a;
+
+    a.append(*m_variableRequestMethod.evaluate());
+    a.append(" ");
+    a.append(m_uri);
+    a.append(" HTTP/");
+    a.append(m_httpVersion);
+    a.append("\n");
+    std::vector<const collection::Variable *> l;
+    m_variableRequestHeaders.resolve(&l);
+    for (auto h : l) {
+        size_t pos = strlen("REQUEST_HEADERS:");
+        a.append((h->m_key.c_str() + pos));
+        a.append(": ");
+        a.append((h->m_value.c_str()));
+    }
+
+    a.append("\n\n");
+    if (this->m_requestBody.str().length() > 0) {
+        a.append(this->m_requestBody.str().c_str());
+        a.append("\n\n");
+    }
+#if 0
+    l.clear();
+    m_variableResponseHeaders.resolve(&l);
+    for (auto h : l) {
+        size_t pos = strlen("RESPONSE_HEADERS:");
+        a.append((h->m_key->c_str() + pos));
+        a.append(": ");
+        a.append((h->m_value->c_str()));
+    }
+    a.append("\n\n");
+    a.append(this->m_responseBody.str().c_str());
+#endif
+    return a;
+}
+
+
 std::string Transaction::toJSON(int parts) {
 #ifdef WITH_YAJL
     const unsigned char *buf;