Adds offset regression tests and assorted fixes on var's offsets

Makefile.am

@@ -274,4 +274,5 @@ TESTS+=test/test-cases/secrules-language-tests/operators/ipMatch.json
 TESTS+=test/test-cases/secrules-language-tests/operators/strmatch.json
 TESTS+=test/test-cases/secrules-language-tests/operators/detectXSS.json
 TESTS+=test/test-cases/secrules-language-tests/operators/eq.json
+TESTS+=test/test-cases/regression/offset-variable.json
 

headers/modsecurity/anchored_set_variable.h

@@ -75,6 +75,10 @@ class AnchoredSetVariable : public std::unordered_multimap<std::string,
 
     void set(const std::string &key, const std::string &value,
         size_t offset);
+
+    void set(const std::string &key, const std::string &value,
+        size_t offset, size_t len);
+
     void setCopy(std::string key, std::string value, size_t offset);
 
     void resolve(std::vector<const collection::Variable *> *l);

headers/modsecurity/anchored_variable.h

@@ -47,8 +47,12 @@ class AnchoredVariable {
 
     void unset();
     void set(const std::string &a, size_t offset);
+    void set(const std::string &a, size_t offset, size_t offsetLen);
     void append(const std::string &a, size_t offset,
-            bool spaceSeparator = false);
+        bool spaceSeparator = false);
+    void append(const std::string &a, size_t offset,
+        bool spaceSeparator, int size);
+
     void evaluate(std::vector<const collection::Variable *> *l);
     std::string *  evaluate();
     std::unique_ptr<std::string> resolveFirst();

headers/modsecurity/transaction.h

@@ -352,11 +352,6 @@ class Transaction : public TransactionAnchoredVariables {
      */
     const char *m_httpVersion;
 
-    /**
-     * Holds the request method: GET, POST, HEAD ...
-     */
-    const char *m_method;
-
     /**
      * Holds the server IP Address
      */

headers/modsecurity/variable_origin.h

@@ -40,7 +40,7 @@ class VariableOrigin {
     std::string toText() {
         std::string offset = std::to_string(m_offset);
         std::string len = std::to_string(m_length);
-        return "rr:" + offset + "," + len;
+        return "v" + offset + "," + len;
     }
 
     int m_length;

src/anchored_set_variable.cc

@@ -49,6 +49,23 @@ void AnchoredSetVariable::unset() {
 }
 
 
+void AnchoredSetVariable::set(const std::string &key,
+    const std::string &value, size_t offset, size_t len) {
+    std::unique_ptr<VariableOrigin> origin(new VariableOrigin());
+    std::string *v = new std::string(value);
+    std::string *k = new std::string(m_name + ":" + key);
+    collection::Variable *var = new collection::Variable(k, v);
+
+    origin->m_offset = offset;
+    origin->m_length = len;
+
+    var->m_dynamic_value = true;
+    var->m_dynamic = false;
+    var->m_orign.push_back(std::move(origin));
+    emplace(key, var);
+}
+
+
 void AnchoredSetVariable::set(const std::string &key,
     const std::string &value, size_t offset) {
     std::unique_ptr<VariableOrigin> origin(new VariableOrigin());

src/anchored_variable.cc

@@ -56,6 +56,18 @@ void AnchoredVariable::unset() {
 }
 
 
+void AnchoredVariable::set(const std::string &a, size_t offset,
+    size_t offsetLen) {
+    std::unique_ptr<VariableOrigin> origin(new VariableOrigin());
+
+    m_offset = offset;
+    m_value.assign(a.c_str(), a.size());
+    origin->m_offset = offset;
+    origin->m_length = offsetLen;
+    m_var->m_orign.push_back(std::move(origin));
+}
+
+
 void AnchoredVariable::set(const std::string &a, size_t offset) {
     std::unique_ptr<VariableOrigin> origin(new VariableOrigin());
 
@@ -84,6 +96,23 @@ void AnchoredVariable::append(const std::string &a, size_t offset,
 }
 
 
+void AnchoredVariable::append(const std::string &a, size_t offset,
+    bool spaceSeparator, int size) {
+    std::unique_ptr<VariableOrigin> origin(
+        new VariableOrigin());
+
+    if (spaceSeparator && !m_value.empty()) {
+        m_value.append(" " + a);
+    } else {
+        m_value.append(a);
+    }
+    m_offset = offset;
+    origin->m_offset = offset;
+    origin->m_length = size;
+    m_var->m_orign.push_back(std::move(origin));
+}
+
+
 void AnchoredVariable::evaluate(std::vector<const collection::Variable *> *l) {
     if (m_name.empty() || m_var == NULL || m_var->m_key == NULL
             || m_var->m_value == NULL || m_var->m_key->empty()) {

src/audit_log/writer/parallel.cc

@@ -144,7 +144,7 @@ bool Parallel::write(Transaction *transaction, int parts, std::string *error) {
 
     std::ofstream myfile;
     std::string a(fileName.c_str());
-    myfile.open (a);
+    myfile.open(a);
     myfile << log;
     myfile.close();
 

src/operators/operator.h

@@ -78,10 +78,7 @@ class Operator {
 
     static void logOffset(RuleMessage *ruleMessage, int offset, int len) {
         if (ruleMessage) {
-            if (ruleMessage->m_reference.empty() == false) {
+            ruleMessage->m_reference.append("o"
-                ruleMessage->m_reference.append(";");
-            }
-            ruleMessage->m_reference.append("op:"
                 + std::to_string(offset) + ","
                 + std::to_string(len));
         }

src/request_body_processor/json.h

@@ -13,6 +13,9 @@
  *
  */
 
+#ifndef SRC_REQUEST_BODY_PROCESSOR_JSON_H_
+#define SRC_REQUEST_BODY_PROCESSOR_JSON_H_
+
 
 #ifdef WITH_YAJL
 
@@ -24,8 +27,6 @@
 #include "modsecurity/transaction.h"
 #include "modsecurity/rules.h"
 
-#ifndef SRC_REQUEST_BODY_PROCESSOR_JSON_H_
-#define SRC_REQUEST_BODY_PROCESSOR_JSON_H_
 
 
 namespace modsecurity {
@@ -82,7 +83,6 @@ class JSON {
 }  // namespace RequestBodyProcessor
 }  // namespace modsecurity
 
+#endif  // WITH_YAJL
 
 #endif  // SRC_REQUEST_BODY_PROCESSOR_JSON_H_
-
-#endif  // WITH_YAJL

src/request_body_processor/multipart.cc

@@ -25,6 +25,7 @@
 #include <list>
 #include <iostream>
 #include <string>
+#include <utility>
 
 #include "modsecurity/collection/collections.h"
 #include "modsecurity/rules.h"
@@ -212,7 +213,7 @@ void Multipart::validate_quotes(const char *data)  {
 }
 
 
-int Multipart::parse_content_disposition(const char *c_d_value) {
+int Multipart::parse_content_disposition(const char *c_d_value, int offset) {
     const char *p = NULL;
 
     /* accept only what we understand */
@@ -342,7 +343,8 @@ int Multipart::parse_content_disposition(const char *c_d_value) {
         if (name == "name") {
             validate_quotes(value.c_str());
 
-            m_transaction->m_variableMultiPartName.set(value, value, 0);
+            m_transaction->m_variableMultiPartName.set(value, value,
+                offset + ((p - c_d_value) - value.size()));
 
             if (!m_mpp->m_name.empty()) {
                 debug(4, "Multipart: Warning: Duplicate Content-Disposition " \
@@ -350,10 +352,12 @@ int Multipart::parse_content_disposition(const char *c_d_value) {
                 return -14;
             }
             m_mpp->m_name.assign(value);
+            m_mpp->m_nameOffset = offset + ((p - c_d_value) - value.size());
             debug(9, "Multipart: Content-Disposition name: " + value + ".");
         } else if (name == "filename") {
             validate_quotes(value.c_str());
-            m_transaction->m_variableMultiPartFileName.set(value, value, 0);
+            m_transaction->m_variableMultiPartFileName.set(value, value, \
+                offset + ((p - c_d_value) - value.size()));
 
             if (!m_mpp->m_filename.empty()) {
                 debug(4, "Multipart: Warning: Duplicate Content-Disposition " \
@@ -361,6 +365,7 @@ int Multipart::parse_content_disposition(const char *c_d_value) {
                 return -15;
             }
             m_mpp->m_filename.assign(value);
+            m_mpp->m_filenameOffset = offset + ((p - c_d_value) - value.size());
 
             debug(9, "Multipart: Content-Disposition filename: " \
                 + value + ".");
@@ -432,7 +437,7 @@ int Multipart::tmp_file_name(std::string *filename) const {
 }
 
 
-int Multipart::process_part_data(std::string *error) {
+int Multipart::process_part_data(std::string *error, size_t offset) {
     char *p = m_buf + (MULTIPART_BUF_SIZE - m_bufleft);
     char localreserve[2] = { '\0', '\0' }; /* initialized to quiet warning */
     int bytes_reserved = 0;
@@ -525,7 +530,11 @@ int Multipart::process_part_data(std::string *error) {
                     return -1;
                 }
 
-                m_mpp->m_tmp_file_size += m_reserve[0];
+                m_mpp->m_tmp_file_size.first += m_reserve[0];
+                if (m_mpp->m_tmp_file_size.second == 0) {
+                    m_mpp->m_tmp_file_size.second = offset \
+                        - m_mpp->m_tmp_file_size.first;
+                }
                 m_mpp->m_length += m_reserve[0];
             }
 
@@ -541,12 +550,26 @@ int Multipart::process_part_data(std::string *error) {
                 return -1;
             }
 
-            m_mpp->m_tmp_file_size += (MULTIPART_BUF_SIZE - m_bufleft);
+            m_mpp->m_value.append(std::string(m_buf,
+                    MULTIPART_BUF_SIZE - m_bufleft));
+            m_mpp->m_valueOffset = offset - (MULTIPART_BUF_SIZE - m_bufleft);
+
+            m_mpp->m_tmp_file_size.first += (MULTIPART_BUF_SIZE - m_bufleft);
+            if (m_mpp->m_tmp_file_size.second == 0) {
+                m_mpp->m_tmp_file_size.second = offset \
+                    - m_mpp->m_tmp_file_size.first;
+            }
+
             m_mpp->m_length += (MULTIPART_BUF_SIZE - m_bufleft);
         } else {
             /* just keep track of the file size */
-            m_mpp->m_tmp_file_size += (MULTIPART_BUF_SIZE - m_bufleft) \
+            m_mpp->m_tmp_file_size.first += (MULTIPART_BUF_SIZE - m_bufleft) \
                 + m_reserve[0];
+            if (m_mpp->m_tmp_file_size.second == 0) {
+                m_mpp->m_tmp_file_size.second = offset \
+                    - m_mpp->m_tmp_file_size.first;
+            }
+
             m_mpp->m_length += (MULTIPART_BUF_SIZE - m_bufleft) + m_reserve[0];
         }
     } else if (m_mpp->m_type == MULTIPART_FORMDATA) {
@@ -573,7 +596,7 @@ int Multipart::process_part_data(std::string *error) {
             m_mpp->m_length += d.size();
         }
 
-        m_mpp->m_value_parts.push_back(d);
+        m_mpp->m_value_parts.push_back(std::make_pair(d, m_buf_offset));
 
         debug(9, "Multipart: Added data to variable: " + d);
     } else {
@@ -601,7 +624,7 @@ int Multipart::process_part_data(std::string *error) {
 }
 
 
-int Multipart::process_part_header(std::string *error) {
+int Multipart::process_part_header(std::string *error, int offset) {
     int i, len;
 
     /* Check for nul bytes. */
@@ -614,6 +637,7 @@ int Multipart::process_part_header(std::string *error) {
         }
     }
 
+    i = 0;
     /* The buffer is data so increase the data length counter. */
     m_reqbody_no_files_length += (MULTIPART_BUF_SIZE - m_bufleft);
 
@@ -639,9 +663,10 @@ int Multipart::process_part_header(std::string *error) {
                 "Content-Disposition header.");
             return false;
         }
-        header_value = m_mpp->m_headers.at("Content-Disposition");
+        header_value = m_mpp->m_headers.at("Content-Disposition").second;
 
-        rc = parse_content_disposition(header_value.c_str());
+        rc = parse_content_disposition(header_value.c_str(),
+            m_mpp->m_headers.at("Content-Disposition").first);
         if (rc < 0) {
             debug(1, "Multipart: Invalid Content-Disposition header ("
                 + std::to_string(rc) + "): " + header_value);
@@ -714,15 +739,17 @@ int Multipart::process_part_header(std::string *error) {
                     m_flag_invalid_header_folding = 1;
                 }
                 data++;
+                i++;
             }
 
             new_value = std::string(data);
             utils::string::chomp(&new_value);
 
             /* update the header value in the table */
-            header_value = m_mpp->m_headers.at(m_mpp->m_last_header_name);
+            header_value = m_mpp->m_headers.at(
+                m_mpp->m_last_header_name).second;
             new_value = header_value + " " +  new_value;
-            m_mpp->m_headers.at(m_mpp->m_last_header_name) = new_value;
+            m_mpp->m_headers.at(m_mpp->m_last_header_name).second = new_value;
 
             debug(9, "Multipart: Continued folder header \"" \
                 + m_mpp->m_last_header_name + "\" with \"" \
@@ -742,6 +769,7 @@ int Multipart::process_part_header(std::string *error) {
             data = m_buf;
             while ((*data != ':') && (*data != '\0')) {
                 data++;
+                i++;
             }
             if (*data == '\0') {
                 debug(1, "Multipart: Invalid part header (colon missing): " \
@@ -763,8 +791,10 @@ int Multipart::process_part_header(std::string *error) {
 
             /* extract the value value */
             data++;
+            i++;
             while ((*data == '\t') || (*data == ' ')) {
                 data++;
+                i++;
             }
             header_value = std::string(data);
             utils::string::chomp(&header_value);
@@ -780,7 +810,8 @@ int Multipart::process_part_header(std::string *error) {
 
 
             m_mpp->m_headers.emplace(
-                std::string(header_name), std::string(header_value));
+                std::string(header_name), std::make_pair(offset - len + i,
+                    std::string(header_value)));
 
 
             debug(9, "Multipart: Added part header \"" + header_name \
@@ -805,8 +836,11 @@ int Multipart::process_boundary(int last_part) {
 
         if (m_mpp->m_type != MULTIPART_FILE) {
             /* now construct a single string out of the parts */
-            for (std::string &i : m_mpp->m_value_parts) {
+            for (const auto &i : m_mpp->m_value_parts) {
-                m_mpp->m_value.append(i);
+                if (m_mpp->m_valueOffset == 0) {
+                    m_mpp->m_valueOffset = i.second;
+                }
+                m_mpp->m_value.append(i.first);
             }
         }
 
@@ -986,6 +1020,7 @@ int Multipart::multipart_complete(std::string *error) {
         if (m->m_name.empty()) {
             continue;
         }
+        size_t offset = m_transaction->m_variableOffset + 1;
 
         if (m->m_type == MULTIPART_FILE) {
             std::string tmp_name;
@@ -996,25 +1031,36 @@ int Multipart::multipart_complete(std::string *error) {
             if (!m->m_filename.empty()) {
                 name.assign(m->m_filename);
             }
+
             m_transaction->m_variableFiles.set(m->m_filename,
-                m->m_filename, 0);
+                m->m_filename, m->m_filenameOffset);
-            m_transaction->m_variableFilesNames.set(m->m_name,
+
-                m->m_name, 0);
+            m_transaction->m_variableFilesNames.set(m->m_filename,
+                m->m_filename, m->m_filenameOffset);
+
             m_transaction->m_variableFilesSizes.set(m->m_name,
-                std::to_string(m->m_tmp_file_size), 0);
+                std::to_string(m->m_tmp_file_size.first),
-            m_transaction->m_variableFilesTmpContent.set(m->m_name,
+                m->m_tmp_file_size.second,
-               m->m_value, 0);
+                m->m_tmp_file_size.first);
-            m_transaction->m_variableFilesTmpContent.set(m->m_name,
+
-               m->m_value, 0);
+            m_transaction->m_variableFilesTmpContent.set(m->m_filename,
-            m_transaction->m_variableFilesTmpNames.set(m->m_name,
+               m->m_value, m->m_valueOffset);
-               m->m_value, 0);
+
-            file_combined_size = file_combined_size + m->m_tmp_file_size;
+            m_transaction->m_variableFilesTmpNames.set(m->m_filename,
+               m->m_filename, m->m_filenameOffset);
+
+            file_combined_size = file_combined_size + m->m_tmp_file_size.first;
+
+            m_transaction->m_variableFilesCombinedSize.set(
+               std::to_string(file_combined_size),
+               m->m_tmp_file_size.second, m->m_tmp_file_size.first);
         } else {
             debug(4, "Adding request argument (BODY): name \"" +
                 m->m_name + "\", value \"" + m->m_value + "\"");
             m_transaction->m_variableArgs.set(m->m_name, m->m_value,
-                m_transaction->m_variableOffset);
+                offset + m->m_valueOffset);
-            m_transaction->m_variableArgsPost.set(m->m_name, m->m_value, 0);
+            m_transaction->m_variableArgsPost.set(m->m_name, m->m_value,
+               offset + m->m_valueOffset);
         }
 #if 0
         if (m_transaction->m_namesArgs->empty()) {
@@ -1036,9 +1082,7 @@ int Multipart::multipart_complete(std::string *error) {
             std::to_string(m_transaction->->m_ARGScombinedSize));
 #endif
     }
-    m_transaction->m_variableFilesCombinedSize.set(
+
-        std::to_string(file_combined_size),
-        m_transaction->m_variableOffset);
 
     return true;
 }
@@ -1256,12 +1300,13 @@ bool Multipart::init(std::string *error) {
  * Assuming that all data is on data. We are not processing chunks.
  *
  */
-bool Multipart::process(const std::string& data, std::string *error) {
+bool Multipart::process(const std::string& data, std::string *error,
+    int offset) {
     const char *inptr = data.c_str();
     unsigned int inleft = data.size();
+    size_t z = 0;
 
     if (data.size() == 0) return true;
-
     m_seen_data = true;
 
     if (m_is_complete) {
@@ -1285,6 +1330,8 @@ bool Multipart::process(const std::string& data, std::string *error) {
         char c = *inptr;
         int process_buffer = 0;
 
+        z++;
+
         if ((c == '\r') && (m_bufleft == 1)) {
             /* we don't want to take \r as the last byte in the buffer */
             process_buffer = 1;
@@ -1443,13 +1490,13 @@ bool Multipart::process(const std::string& data, std::string *error) {
                             return false;
                         }
 
-                        if (process_part_header(error) < 0) {
+                        if (process_part_header(error, offset + z) < 0) {
                             m_flag_error = 1;
                             return false;
                         }
 
                     } else {
-                        if (process_part_data(error) < 0) {
+                        if (process_part_data(error, offset + z) < 0) {
                             m_flag_error = 1;
                             return false;
                         }

src/request_body_processor/multipart.h

@@ -17,6 +17,7 @@
 #include <iostream>
 #include <list>
 #include <unordered_map>
+#include <utility>
 
 #ifndef SRC_REQUEST_BODY_PROCESSOR_MULTIPART_H_
 #define SRC_REQUEST_BODY_PROCESSOR_MULTIPART_H_
@@ -58,9 +59,14 @@ class MultipartPart {
     MultipartPart()
      : m_type(MULTIPART_FORMDATA),
      m_tmp_file_fd(0),
-     m_tmp_file_size(0),
      m_offset(0),
-     m_length(0) { }
+     m_filenameOffset(0),
+     m_nameOffset(0),
+     m_valueOffset(0),
+     m_length(0) {
+         m_tmp_file_size.first = 0;
+         m_tmp_file_size.second = 0;
+    }
 
     ~MultipartPart() {
         m_headers.clear();
@@ -72,10 +78,13 @@ class MultipartPart {
 
     /* the name */
     std::string m_name;
+    size_t m_nameOffset;
 
     /* variables only, variable value */
     std::string m_value;
-    std::list<std::string> m_value_parts;
+    size_t m_valueOffset;
+
+    std::list<std::pair<std::string, int>> m_value_parts;
 
     /* files only, the content type (where available) */
     /* std::string m_content_type; */
@@ -83,13 +92,15 @@ class MultipartPart {
     /* files only, the name of the temporary file holding data */
     std::string m_tmp_file_name;
     int m_tmp_file_fd;
-    unsigned int m_tmp_file_size;
+    std::pair<size_t, size_t> m_tmp_file_size;
 
     /* files only, filename as supplied by the browser */
     std::string m_filename;
+    size_t m_filenameOffset;
 
     std::string m_last_header_name;
-    std::unordered_map<std::string, std::string, MyHash, MyEqual> m_headers;
+    std::unordered_map<std::string, std::pair<size_t, std::string>,
+        MyHash, MyEqual> m_headers;
 
     unsigned int m_offset;
     unsigned int m_length;
@@ -108,11 +119,11 @@ class Multipart {
     int is_token_char(unsigned char c);
     int multipart_complete(std::string *err);
 
-    int parse_content_disposition(const char *c_d_value);
+    int parse_content_disposition(const char *c_d_value, int offset);
-    bool process(const std::string& data, std::string *err);
+    bool process(const std::string& data, std::string *err, int offset);
     int process_boundary(int last_part);
-    int process_part_header(std::string *error);
+    int process_part_header(std::string *error, int offset);
-    int process_part_data(std::string *error);
+    int process_part_data(std::string *error, size_t offset);
 
     int tmp_file_name(std::string *filename) const;
 

src/rule.cc

@@ -651,13 +651,9 @@ bool Rule::evaluate(Transaction *trasn) {
             if (ret == true) {
                 ruleMessage.m_match = resolveMatchMessage(key, value);
                 for (auto &i : v->m_orign) {
-                    if (ruleMessage.m_reference.empty()) {
+                    ruleMessage.m_reference.append(i->toText());
-                        ruleMessage.m_reference.append(i->toText());
-                    } else {
-                        ruleMessage.m_reference.append(";" + i->toText());
-                    }
                 }
-                ruleMessage.m_reference.append("-" + *valueTemp.second);
+                ruleMessage.m_reference.append(*valueTemp.second);
                 updateMatchedVars(trasn, key, value);
                 executeActionsIndependentOfChainedRuleResult(trasn,
                     &containsDisruptive, &ruleMessage);

src/transaction.cc

@@ -105,7 +105,6 @@ Transaction::Transaction(ModSecurity *ms, Rules *rules, void *logCbData)
     m_serverPort(0),
     m_uri(""),
     m_uri_no_query_string_decoded(""),
-    m_method(""),
     m_httpVersion(""),
     m_rules(rules),
     m_timeStamp(std::time(NULL)),
@@ -301,17 +300,20 @@ bool Transaction::addArgument(const std::string& orig, const std::string& key,
 
     if (orig == "GET") {
         m_variableArgsGet.set(key, value, offset);
-        m_variableArgGetNames.append(key, offset, true);
+        m_variableArgGetNames.append(key, offset - key.size() - 1, true);
     } else if (orig == "POST") {
         m_variableArgsPost.set(key, value, offset);
-        m_variableArgPostNames.append(key, offset, true);
+        m_variableArgPostNames.append(key, offset - key.size() - 1, true);
     }
-    m_variableArgsNames.append(key, offset, true);
+    m_variableArgsNames.append(key, offset - key.size() - 1, true);
 
     m_ARGScombinedSizeDouble = m_ARGScombinedSizeDouble + \
         key.length() + value.length();
 
-    m_variableARGScombinedSize.set(std::to_string(m_ARGScombinedSizeDouble), 0);
+    m_variableARGScombinedSize.set(std::to_string(m_ARGScombinedSizeDouble),
+        offset - key.size() - 1, key.size());
+    m_variableARGScombinedSize.set(std::to_string(m_ARGScombinedSizeDouble),
+        offset, value.length());
 
     return true;
 }
@@ -347,7 +349,6 @@ int Transaction::processURI(const char *uri, const char *method,
     debug(4, "Starting phase URI. (SecRules 0 + 1/2)");
 #endif
 
-    m_method = method;
     m_httpVersion = http_version;
     m_uri = uri;
     std::string uri_s(uri);
@@ -355,10 +356,19 @@ int Transaction::processURI(const char *uri, const char *method,
 
     size_t pos = m_uri_decoded.find("?");
     size_t pos_raw = uri_s.find("?");
+    size_t var_size = pos_raw;
 
-    m_variableRequestLine.set(std::string(method) + " " + std::string(uri)
+    m_variableRequestMethod.set(method, 0);
+
+
+    std::string requestLine(std::string(method) + " " + std::string(uri));
+    m_variableRequestLine.set(requestLine \
         + " HTTP/" + std::string(http_version), m_variableOffset);
-    m_variableOffset = m_variableRequestLine.m_value.size();
+
+    m_variableRequestProtocol.set("HTTP/" + std::string(http_version),
+        m_variableOffset + requestLine.size() + 1);
+
+
 
     if (pos != std::string::npos) {
         m_uri_no_query_string_decoded = std::string(m_uri_decoded, 0, pos);
@@ -366,6 +376,7 @@ int Transaction::processURI(const char *uri, const char *method,
         m_uri_no_query_string_decoded = std::string(m_uri_decoded);
     }
 
+
     if (pos_raw != std::string::npos) {
         std::string qry = std::string(uri_s, pos_raw + 1,
             uri_s.length() - (pos_raw + 1));
@@ -379,18 +390,26 @@ int Transaction::processURI(const char *uri, const char *method,
     } else {
         path_info = std::string(m_uri_decoded, 0, pos);
     }
-    m_variablePathInfo.set(path_info, m_variableOffset);
+    if (var_size == std::string::npos) {
-    m_variableRequestFilename.set(path_info, m_variableOffset);
+        var_size = uri_s.size();
+    }
+
+    m_variablePathInfo.set(path_info, m_variableOffset + strlen(method) +
+        1, var_size);
+    m_variableRequestFilename.set(path_info,  m_variableOffset +
+        strlen(method) + 1, var_size);
+
+
 
     size_t offset = path_info.find_last_of("/\\");
     if (offset != std::string::npos && path_info.length() > offset + 1) {
         std::string basename = std::string(path_info, offset + 1,
             path_info.length() - (offset + 1));
-        m_variableRequestBasename.set(basename, m_variableOffset);
+        m_variableRequestBasename.set(basename, m_variableOffset +
+            strlen(method) + 1 + offset + 1);
     }
-    m_variableRequestMethod.set(method, 0);
+
-    m_variableRequestProtocol.set("HTTP/" + std::string(http_version),
+    m_variableOffset = m_variableRequestLine.m_value.size();
-        m_variableOffset);
 
     std::string parsedURI = m_uri_decoded;
     // The more popular case is without domain
@@ -416,8 +435,9 @@ int Transaction::processURI(const char *uri, const char *method,
         }
     }
 
-    m_variableRequestURI.set(parsedURI, m_variableOffset);
+    m_variableRequestURI.set(parsedURI, std::string(method).size() + 1,
-    m_variableRequestURIRaw.set(uri, m_variableOffset);
+        uri_s.size());
+    m_variableRequestURIRaw.set(uri, std::string(method).size() + 1);
 
     if (m_variableQueryString.m_value.empty() == false) {
         extractArguments("GET", m_variableQueryString.m_value,
@@ -480,11 +500,11 @@ int Transaction::processRequestHeaders() {
  */
 int Transaction::addRequestHeader(const std::string& key,
     const std::string& value) {
-    m_variableRequestHeadersNames.append(key, 0, true);
+    m_variableRequestHeadersNames.append(key, m_variableOffset, true,
+        key.size());
 
     m_variableOffset = m_variableOffset + key.size() + 2;
     m_variableRequestHeaders.set(key, value, m_variableOffset);
-    m_variableOffset = m_variableOffset + value.size() + 1;
 
 
     std::string keyl = utils::string::tolower(key);
@@ -494,19 +514,22 @@ int Transaction::addRequestHeader(const std::string& key,
     }
 
     if (keyl == "cookie") {
+        size_t localOffset = m_variableOffset;
         std::vector<std::string> cookies = utils::string::split(value, ';');
-        while (cookies.empty() == false) {
+        for (const std::string &c : cookies) {
-            std::vector<std::string> s = utils::string::split(cookies.back(),
+            std::vector<std::string> s = utils::string::split(c,
                '=');
             if (s.size() > 1) {
                 if (s[0].at(0) == ' ') {
                     s[0].erase(0, 1);
                 }
-                m_variableRequestCookies.set(s[0], s[1], m_variableOffset);
                 m_variableRequestCookiesNames.set(s[0],
-                    s[0], m_variableOffset);
+                    s[0], localOffset);
+
+                localOffset = localOffset + s[0].size() + 1;
+                m_variableRequestCookies.set(s[0], s[1], localOffset);
+                localOffset = localOffset + s[1].size() + 2;
             }
-            cookies.pop_back();
         }
     }
     /**
@@ -534,6 +557,8 @@ int Transaction::addRequestHeader(const std::string& key,
         std::vector<std::string> host = utils::string::split(value, ':');
         m_variableServerName.set(host[0], m_variableOffset);
     }
+    m_variableOffset = m_variableOffset + value.size() + 1;
+
     return 1;
 }
 
@@ -624,7 +649,7 @@ int Transaction::processRequestBody() {
     }
 
     if (m_variableInboundDataError.m_value.empty() == true) {
-        m_variableInboundDataError.set("0", m_variableOffset);
+        m_variableInboundDataError.set("0", 0);
     }
 
     /*
@@ -682,7 +707,7 @@ int Transaction::processRequestBody() {
         if (a != NULL) {
             Multipart m(*a, this);
             if (m.init(&error) == true) {
-                m.process(m_requestBody.str(), &error);
+                m.process(m_requestBody.str(), &error, m_variableOffset);
             }
             m.multipart_complete(&error);
         }
@@ -698,7 +723,8 @@ int Transaction::processRequestBody() {
             m_variableReqbodyProcessorError.set("0", m_variableOffset);
         }
     } else if (m_requestBodyType == WWWFormUrlEncoded) {
-        extractArguments("POST", m_requestBody.str(), 0);
+        m_variableOffset++;
+        extractArguments("POST", m_requestBody.str(), m_variableOffset);
     } else if (a != NULL) {
         std::string error;
         if (a != NULL && a->empty() == false) {
@@ -762,7 +788,7 @@ int Transaction::processRequestBody() {
         m_variableRequestBody.set(m_requestBody.str(), m_variableOffset);
         m_variableRequestBodyLength.set(std::to_string(
             m_requestBody.str().size()),
-            m_variableOffset);
+            m_variableOffset, m_requestBody.str().size());
     }
 
     this->m_rules->evaluate(modsecurity::RequestBodyPhase, this);
@@ -1310,7 +1336,8 @@ std::string Transaction::toOldAuditLogFormatIndex(const std::string &filename,
     ss << tstr << " ";
 
     ss << "\"";
-    ss << this->m_method << " ";
+    ss << utils::string::dash_if_empty(m_variableRequestMethod.evaluate());
+    ss << " ";
     ss << this->m_uri << " ";
     ss << "HTTP/" << m_httpVersion;
     ss << "\" ";
@@ -1360,7 +1387,9 @@ std::string Transaction::toOldAuditLogFormat(int parts,
     if (parts & audit_log::AuditLog::BAuditLogPart) {
         std::vector<const collection::Variable *> l;
         audit_log << "--" << trailer << "-" << "B--" << std::endl;
-        audit_log << this->m_method << " " << this->m_uri << " " << "HTTP/";
+        audit_log << utils::string::dash_if_empty(
+            m_variableRequestMethod.evaluate());
+        audit_log << " " << this->m_uri << " " << "HTTP/";
         audit_log << this->m_httpVersion << std::endl;
 
         m_variableRequestHeaders.resolve(&l);
@@ -1474,7 +1503,10 @@ std::string Transaction::toJSON(int parts) {
         strlen("request"));
     yajl_gen_map_open(g);
 
-    LOGFY_ADD("protocol", m_method);
+    LOGFY_ADD("protocol",
+        utils::string::dash_if_empty(
+            m_variableRequestMethod.evaluate()).c_str());
+
     LOGFY_ADD_INT("http_version", m_httpVersion);
     LOGFY_ADD("uri", this->m_uri);
 

src/variables/remote_user.cc

@@ -41,16 +41,18 @@ void RemoteUser::evaluate(Transaction *transaction,
     std::vector<const collection::Variable *> *l) {
     size_t pos;
     std::string base64;
+    collection::Variable *var;
 
-    std::unique_ptr<std::string> header = std::move(
+    transaction->m_variableRequestHeaders.resolve("authorization", l);
-	transaction->m_variableRequestHeaders.resolveFirst("Authorization"));
 
-    if (header == NULL) {
+    if (l->size() < 1) {
         return;
     }
 
-    if (header->compare(0, 6, "Basic ") == 0) {
+    std::string header(*l->at(0)->m_value);
-        base64 = std::string(*header, 6, header->length());
+
+    if (header.compare(0, 6, "Basic ") == 0) {
+        base64 = std::string(header, 6, header.length());
     }
 
     base64 = Utils::Base64::decode(base64);
@@ -61,8 +63,18 @@ void RemoteUser::evaluate(Transaction *transaction,
     }
     transaction->m_variableRemoteUser.assign(std::string(base64, 0, pos));
 
-    l->push_back(new collection::Variable(&m_retName,
+    var = new collection::Variable(l->at(0)->m_key,
-        &transaction->m_variableRemoteUser));
+        &transaction->m_variableRemoteUser);
+
+    for (auto &i : l->at(0)->m_orign) {
+        std::unique_ptr<VariableOrigin> origin(new VariableOrigin());
+        origin->m_offset = i->m_offset;
+        origin->m_length = i->m_length;
+        var->m_orign.push_back(std::move(origin));
+    }
+
+    l->clear();
+    l->push_back(var);
 }
 
 

test/coding_style_suppressions.txt

@@ -21,4 +21,12 @@
 ./test/fuzzer
 ./test/libfuzzer
 ./src/parser/seclang-parser.tab.cc
+./src/unique_id.cc:226
+./test/unit/unit.cc:84
+./test/unit/unit.cc:82
+./headers/modsecurity/rule.h:110
+./test/regression/regression.cc:44
+./test/benchmark/owasp-v3/util/av-scanning/runAV/common.h
+./src/audit_log/writer/parallel.cc:28
+./src/macro_expansion.c
 Total errors found

test/regression/custom_debug_log.h

@@ -31,6 +31,7 @@ class CustomDebugLog : public modsecurity::debug_log::DebugLog {
     void write(int level, const std::string& message) override;
     bool contains(const std::string& pattern);
     std::string log_messages();
+    std::string error_log_messages();
 
  private:
     std::stringstream m_log;

test/regression/regression.cc

@@ -475,7 +475,7 @@ int main(int argc, char **argv) {
         if (r->passed == true && r->skipped == false) {
             passed++;
         } else if (r->skipped == false) {
-            if (test.m_automake_output && 1 == 0) {
+            if (test.m_automake_output) {
                 // m_automake_output
             } else {
                 std::cout << KRED << "Test failed." << RESET << KWHT \

test/regression/regression_test.cc

@@ -66,15 +66,15 @@ inline std::vector<std::string> RegressionTest::yajl_array_to_vec_str(
 }
 
 
-inline std::unordered_map<std::string, std::string>
+inline std::vector<std::pair<std::string, std::string>>
     RegressionTest::yajl_array_to_map(const yajl_val &node) {
-    std::unordered_map<std::string, std::string> vec;
+    std::vector<std::pair<std::string, std::string>> vec;
     for (int z = 0; z < node->u.object.len; z++) {
         const char *key = node->u.object.keys[z];
         yajl_val val3 = node->u.object.values[z];
         const char *value = YAJL_GET_STRING(val3);
         std::pair<std::string, std::string> a(key, value);
-        vec.insert(a);
+        vec.push_back(a);
     }
     return vec;
 }

test/regression/regression_test.h

@@ -18,8 +18,10 @@
 #include <iostream>
 #include <sstream>
 #include <unordered_map>
+#include <map>
 #include <vector>
 #include <string>
+#include <utility>
 
 #ifndef TEST_REGRESSION_REGRESSION_TEST_H_
 #define TEST_REGRESSION_REGRESSION_TEST_H_
@@ -44,8 +46,8 @@ class RegressionTest {
     int version_max;
     int github_issue;
 
-    std::unordered_map<std::string, std::string> request_headers;
+    std::vector<std::pair<std::string, std::string>> request_headers;
-    std::unordered_map<std::string, std::string> response_headers;
+    std::vector<std::pair<std::string, std::string>> response_headers;
     std::string request_body;
     std::string response_body;
     std::string response_protocol;
@@ -68,8 +70,8 @@ class RegressionTest {
     static inline std::string yajl_array_to_str(const yajl_val &node);
     static inline std::vector<std::string> yajl_array_to_vec_str(
         const yajl_val &node);
-    static inline std::unordered_map<std::string,
+    static inline std::vector<std::pair<std::string, std::string>>
-        std::string> yajl_array_to_map(const yajl_val &node);
+        yajl_array_to_map(const yajl_val &node);
 
     int http_code;
     std::string redirect_url;

test/test-cases/regression/variable-FILES_NAMES.json

@@ -51,7 +51,7 @@
       ]
     },
     "expected":{  
-      "debug_log":"T \\(0\\) t:trim: \"filedata"
+      "debug_log":"T \\(0\\) t:trim: \"small_text"
     },
     "rules":[  
       "SecRuleEngine On",

test/test-cases/regression/variable-FULL_REQUEST.json

@@ -51,7 +51,7 @@
       ]
     },
     "expected":{  
-      "debug_log":"Content-Type: multipart/form-data; boundary=------------"
+      "debug_log":"Multipart: Boundary: --------------------------756b6d74fa1a8ee2"
     },
     "rules":[  
       "SecRuleEngine On",

test/test-cases/regression/variable-RESPONSE_HEADERS_NAMES.json

@@ -51,7 +51,7 @@
       ]
     },
     "expected":{  
-      "debug_log":"Target value: \"Content-Type Last-Modified Date\" \\(Variable: RESPONSE_HEADERS_NAMES\\)"
+      "debug_log":"Target value: \"Date Last-Modified Content-Type\" \\(Variable: RESPONSE_HEADERS_NAMES\\)"
     },
     "rules":[  
       "SecRuleEngine On",

test/unit/unit.cc

@@ -240,7 +240,7 @@ int main(int argc, char **argv) {
                 } else {
                     for (auto &i : r) {
                         if (i->skipped == true) {
-                            skp++; 
+                            skp++;
                         }
                     }
                     std::cout << KRED << r.size()-skp << " tests failed.";
@@ -275,7 +275,7 @@ int main(int argc, char **argv) {
             int skp = 0;
             for (auto &i : results) {
                 if (i->skipped == true) {
-                    skp++; 
+                    skp++;
                 }
             }
             std::cout << KRED << results.size()-skp << " failed.";