Huge improve in the variables resolution time

headers/modsecurity/transaction/collections.h

@@ -64,6 +64,23 @@ class Collections :
         std::list<transaction::Variable *> *l);
     std::list<transaction::Variable *> *resolve(const std::string& var);
 
+
+    void resolveSingleMatch(const std::string& var,
+        std::list<transaction::Variable *> *l);
+    void resolveSingleMatch(const std::string& var,
+        const std::string& collection,
+        std::list<transaction::Variable *> *l);
+    void resolveMultiMatches(const std::string& var,
+        std::list<transaction::Variable *> *l);
+    void resolveMultiMatches(const std::string& var,
+        const std::string& collection,
+        std::list<transaction::Variable *> *l);
+    void resolveRegularExpression(const std::string& var,
+        std::list<transaction::Variable *> *l);
+    void resolveRegularExpression(const std::string& var,
+        const std::string& collection,
+        std::list<transaction::Variable *> *l);
+
     /**
      * This is a special collection to host the transaction variables.
      *

headers/modsecurity/transaction/variables.h

@@ -57,6 +57,14 @@ class Variables :
 
     std::list<Variable *> resolve(const std::string& var,
         std::list<transaction::Variable *> *l);
+
+    void resolveSingleMatch(const std::string& var,
+        std::list<transaction::Variable *> *l);
+    void resolveMultiMatches(const std::string& var,
+        std::list<transaction::Variable *> *l);
+    void resolveRegularExpression(const std::string& var,
+        std::list<transaction::Variable *> *l);
+
 };
 
 }  // namespace transaction

src/collections.cc

@@ -137,8 +137,59 @@ std::list<transaction::Variable *> *
         new std::list<transaction::Variable *>();
 
     resolve(var, l);
+}
 
-    return l;
+
+void Collections::resolveSingleMatch(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+
+    m_transient.resolveSingleMatch(var, l);
+}
+
+
+void Collections::resolveSingleMatch(const std::string& var,
+    const std::string& collection,
+    std::list<transaction::Variable *> *l) {
+
+    try {
+        this->at(collection)->resolveSingleMatch(var, l);
+    } catch (...) {
+
+    }
+}
+
+void Collections::resolveMultiMatches(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+
+    m_transient.resolveMultiMatches(var, l);
+}
+
+
+void Collections::resolveMultiMatches(const std::string& var,
+    const std::string& collection,
+    std::list<transaction::Variable *> *l) {
+    try {
+        this->at(collection)->resolveMultiMatches(var, l);
+    } catch (...) {
+
+    }
+}
+
+void Collections::resolveRegularExpression(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+    m_transient.resolveRegularExpression(var, l);
+}
+
+
+void Collections::resolveRegularExpression(const std::string& var,
+    const std::string& collection,
+    std::list<transaction::Variable *> *l) {
+
+    try {
+        this->at(collection)->resolveRegularExpression(var, l);
+    } catch (...) {
+
+    }
 }
 
 }  // namespace transaction

src/parser/seclang-parser.yy

@@ -53,6 +53,7 @@ class Driver;
 #include "variables/time_sec.h"
 #include "variables/time_wday.h"
 #include "variables/time_year.h"
+#include "variables/tx.h"
 
 using ModSecurity::ModSecurity;
 
@@ -88,6 +89,7 @@ using ModSecurity::Variables::TimeSec;
 using ModSecurity::Variables::TimeWDay;
 using ModSecurity::Variables::TimeYear;
 using ModSecurity::Variables::Variable;
+using ModSecurity::Variables::Tx;
 
 
 #define CHECK_VARIATION_DECL \
@@ -196,6 +198,8 @@ using ModSecurity::Variables::Variable;
 %token <std::string> CONFIG_DIR_SEC_MARKER
 
 %token <std::string> VARIABLE
+%token <std::string> VARIABLE_TX
+%token <std::string> VARIABLE_COL
 %token <std::string> RUN_TIME_VAR_DUR
 %token <std::string> RUN_TIME_VAR_ENV
 %token <std::string> RUN_TIME_VAR_BLD
@@ -601,9 +605,27 @@ var:
       {
         std::string name($1);
         CHECK_VARIATION_DECL
-        CHECK_VARIATION(&) { var = new Count(new Variable(name)); }
+        CHECK_VARIATION(&) { var = new Count(new Variable(name, Variable::VariableKind::DirectVariable)); }
-        CHECK_VARIATION(!) { var = new Exclusion(new Variable(name)); }
+        CHECK_VARIATION(!) { var = new Exclusion(new Variable(name, Variable::VariableKind::DirectVariable)); }
-        if (!var) { var = new Variable(name); }
+        if (!var) { var = new Variable(name, Variable::VariableKind::DirectVariable); }
+        $$ = var;
+      }
+    | VARIABLE_COL
+      {
+        std::string name($1);
+        CHECK_VARIATION_DECL
+        CHECK_VARIATION(&) { var = new Count(new Variable(name, Variable::VariableKind::CollectionVarible)); }
+        CHECK_VARIATION(!) { var = new Exclusion(new Variable(name, Variable::VariableKind::CollectionVarible)); }
+        if (!var) { var = new Variable(name, Variable::VariableKind::CollectionVarible); }
+        $$ = var;
+      }
+    | VARIABLE_TX
+      {
+        std::string name($1);
+        CHECK_VARIATION_DECL
+        CHECK_VARIATION(&) { var = new Count(new Tx(name)); }
+        CHECK_VARIATION(!) { var = new Exclusion(new Tx(name)); }
+        if (!var) { var = new Tx(name); }
         $$ = var;
       }
     | RUN_TIME_VAR_DUR

src/parser/seclang-scanner.ll

@@ -108,10 +108,11 @@ OPERATORNOARG	(?i:@detectSQLi|@detectXSS|@geoLookup|@validateUrlEncoding|@valida
 
 TRANSFORMATION  t:(sha1|hexEncode|lowercase|urlDecodeUni|urlDecode|none|compressWhitespace|removeWhitespace|replaceNulls|removeNulls|htmlEntityDecode|jsDecode|cssDecode|trim|normalizePathWin|normalisePath|length|utf8toUnicode|urldecode|removeComments|replaceComments)
 
-VARIABLE (?i:(ARGS_COMBINED_SIZE|ARGS_GET_NAMES|ARGS_POST_NAMES|FILES_COMBINED_SIZE|FULL_REQUEST_LENGTH|FILES_SIZES|FILES_NAMES|FILES_TMP_CONTENT|REQUEST_BODY_LENGTH|REQUEST_URI_RAW|UNIQUE_ID|SERVER_PORT|SERVER_ADDR|REMOTE_PORT|REMOTE_HOST|MULTIPART_STRICT_ERROR|PATH_INFO|MULTIPART_NAME|MULTIPART_FILENAME|MULTIPART_CRLF_LF_LINES|MATCHED_VAR_NAME|MATCHED_VARS_NAMES|MATCHED_VAR|MATCHED_VARS|INBOUND_DATA_ERROR|OUTBOUND_DATA_ERROR|FULL_REQUEST|FILES|AUTH_TYPE|ARGS_NAMES|ARGS|QUERY_STRING|REMOTE_ADDR|REQUEST_BASENAME|REQUEST_BODY|REQUEST_COOKIES_NAMES|REQUEST_COOKIES|REQUEST_FILENAME|REQUEST_HEADERS_NAMES|REQUEST_HEADERS|REQUEST_METHOD|REQUEST_PROTOCOL|REQUEST_URI|RESPONSE_BODY|RESPONSE_CONTENT_LENGTH|RESPONSE_CONTENT_TYPE|RESPONSE_HEADERS_NAMES|RESPONSE_HEADERS|RESPONSE_PROTOCOL|RESPONSE_STATUS|TX|GEO|REQBODY_PROCESSOR|IP|XML))
-
 
+VARIABLE (?i:(ARGS_COMBINED_SIZE|ARGS_GET_NAMES|ARGS_POST_NAMES|FILES_COMBINED_SIZE|FULL_REQUEST_LENGTH|REQUEST_BODY_LENGTH|REQUEST_URI_RAW|UNIQUE_ID|SERVER_PORT|SERVER_ADDR|REMOTE_PORT|REMOTE_HOST|MULTIPART_STRICT_ERROR|PATH_INFO|MULTIPART_CRLF_LF_LINES|MATCHED_VAR_NAME|MATCHED_VAR|INBOUND_DATA_ERROR|OUTBOUND_DATA_ERROR|FULL_REQUEST|AUTH_TYPE|ARGS_NAMES|REMOTE_ADDR|REQUEST_BASENAME|REQUEST_BODY|REQUEST_FILENAME|REQUEST_HEADERS_NAMES|REQUEST_METHOD|REQUEST_PROTOCOL|REQUEST_URI|RESPONSE_BODY|RESPONSE_CONTENT_LENGTH|RESPONSE_CONTENT_TYPE|RESPONSE_HEADERS_NAMES|RESPONSE_PROTOCOL|RESPONSE_STATUS|REQBODY_PROCESSOR))
+VARIABLE_COL (?i:(ARGS|FILES_SIZES|FILES_NAMES|FILES_TMP_CONTENT|MULTIPART_FILENAME|MULTIPART_NAME|MATCHED_VARS_NAMES|MATCHED_VARS|FILES|QUERY_STRING|REQUEST_COOKIES|REQUEST_HEADERS|RESPONSE_HEADERS|GEO|IP|XML|REQUEST_COOKIES_NAMES))
 
+VARIABLE_TX (?i:TX)
 RUN_TIME_VAR_DUR  (?i:DURATION)
 RUN_TIME_VAR_ENV  (?i:ENV)
 RUN_TIME_VAR_BLD  (?i:MODSEC_BUILD)
@@ -204,14 +205,20 @@ CONFIG_DIR_UNICODE_MAP_FILE (?i:SecUnicodeMapFile)
 
 <INITIAL,EXPECTING_OPERATOR>{
 %{ /* Variables */ %}
-[!&]?{VARIABLE}(\:{DICT_ELEMENT})?             { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE(yytext, *driver.loc.back()); }
+[!&]?{VARIABLE}(\:{DICT_ELEMENT})?        { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE(yytext, *driver.loc.back()); }
+[!&]?{VARIABLE_COL}(\:{DICT_ELEMENT})?    { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE_COL(yytext, *driver.loc.back()); }
+[!&]?{VARIABLE_TX}(\:{DICT_ELEMENT})?     { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE_TX(yytext, *driver.loc.back()); }
 [!&]?{RUN_TIME_VAR_DUR}                      { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_DUR(yytext, *driver.loc.back()); }
 [!&]?{RUN_TIME_VAR_ENV}(\:{DICT_ELEMENT})?     { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_ENV(yytext, *driver.loc.back()); }
 [!&]?{RUN_TIME_VAR_BLD}                      { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_BLD(yytext, *driver.loc.back()); }
 [!&]?{RUN_TIME_VAR_HSV}                      { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_HSV(yytext, *driver.loc.back()); }
 [!&]?{VARIABLENOCOLON}                       { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE(yytext, *driver.loc.back()); }
 
-["][!&]?{VARIABLE}(\:{DICT_ELEMENT})?             { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE(yytext, *driver.loc.back()); }
+
+["][!&]?{VARIABLE}(\:{DICT_ELEMENT})?     { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE(yytext, *driver.loc.back()); }
+["][!&]?{VARIABLE_TX}(\:{DICT_ELEMENT})?  { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE_TX(yytext, *driver.loc.back()); }
+["][!&]?{VARIABLE_COL}(\:{DICT_ELEMENT})? { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_VARIABLE_COL(yytext, *driver.loc.back()); }
+
 ["][!&]?{RUN_TIME_VAR_DUR}["]                      { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_DUR(yytext, *driver.loc.back()); }
 ["][!&]?{RUN_TIME_VAR_ENV}(\:{DICT_ELEMENT})?["]     { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_ENV(yytext, *driver.loc.back()); }
 ["][!&]?{RUN_TIME_VAR_BLD}["]                      { BEGIN(EXPECTING_OPERATOR); return yy::seclang_parser::make_RUN_TIME_VAR_BLD(yytext, *driver.loc.back()); }

src/rule.cc

@@ -289,7 +289,7 @@ bool Rule::evaluate(Assay *assay) {
             for (auto &y : *z) {
                 exclusions.push_back(y->m_key);
             }
-            exclusions.push_back(variable->name);
+            exclusions.push_back(variable->m_name);
             delete z;
         }
     }

src/variables.cc

@@ -65,6 +65,48 @@ void Variables::del(const std::string& key) {
 }
 
 
+
+void Variables::resolveSingleMatch(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+    auto range = this->equal_range(var);
+
+    for (auto it = range.first; it != range.second; ++it) {
+        l->push_back(new transaction::Variable(var, it->second));
+    }
+}
+
+
+void Variables::resolveMultiMatches(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+    size_t keySize = var.size();
+
+    auto range = this->equal_range(var);
+
+    for (auto it = range.first; it != range.second; ++it) {
+        l->push_back(new transaction::Variable(var, it->second));
+    }
+
+    for (auto& x : *this) {
+        if (x.first.size() <= keySize + 1) {
+            continue;
+        }
+        if (x.first.at(keySize) != ':') {
+            continue;
+        }
+        if (x.first.compare(0, keySize, var) != 0) {
+            continue;
+        }
+        l->push_back(new transaction::Variable(x.first, x.second));
+    }
+}
+
+
+void Variables::resolveRegularExpression(const std::string& var,
+    std::list<transaction::Variable *> *l) {
+    /* Not ready */
+}
+
+
 std::list<transaction::Variable *> Variables::resolve(const std::string& key,
     std::list<transaction::Variable *> *l) {
     auto range = this->equal_range(key);

src/variables/env.cc

@@ -49,15 +49,15 @@ std::list<transaction::Variable *> *
         std::string value = std::string(env, pos+1, env.length() - (pos + 1));
 
         envs.insert(std::pair<std::string, std::string>("ENV:" + key, value));
-        if ("env:" + key == name) {
+        if ("env:" + key == m_name) {
-            resl->push_back(new transaction::Variable(name, value));
+            resl->push_back(new transaction::Variable(m_name, value));
             return resl;
         }
     }
 
     for (auto& x : envs) {
-        if ((x.first.substr(0, name.size() + 1).compare(name + ":") != 0)
+        if ((x.first.substr(0, m_name.size() + 1).compare(m_name + ":") != 0)
-            && (x.first != name)) {
+            && (x.first != m_name)) {
             continue;
         }
         resl->push_back(new transaction::Variable(x.first, x.second));

src/variables/tx.cc

@@ -38,7 +38,13 @@ std::list<transaction::Variable *> *
     std::list<transaction::Variable *> *resl =
         new std::list<transaction::Variable *>();
 
-    resl->push_back(new transaction::Variable("TX:0", "teste"));
+    if (m_type == SingleMatch) {
+        assay->m_collections.resolveSingleMatch(m_name, "TX", resl);
+    } else if (m_type == MultipleMatches) {
+        assay->m_collections.resolveMultiMatches(m_name, "TX", resl);
+    } else if (m_type == RegularExpression) {
+        assay->m_collections.resolveRegularExpression(m_name, "TX", resl);
+    }
 
     return resl;
 }

src/variables/variable.cc

@@ -28,20 +28,80 @@ using ModSecurity::Variables::Variations::Exclusion;
 namespace ModSecurity {
 namespace Variables {
 
+
+Variable::Variable(std::string name)
+    : m_name(name),
+    m_collectionName("") {
+
+    if (m_name.at(0) == '\\') {
+        m_type = RegularExpression;
+    } else if (m_name.find(":") != std::string::npos) {
+        m_type = SingleMatch;
+    } else {
+        m_type = MultipleMatches;
+    }
+
+    if (m_name.find(".") != std::string::npos) {
+        m_kind = CollectionVarible;
+        m_collectionName = std::string(m_name, 0, m_name.find("."));
+    } else {
+        m_kind = DirectVariable;
+    }
+}
+
+
+Variable::Variable(std::string name, VariableKind kind)
+    : m_name(name),
+    m_collectionName(""),
+    m_kind(kind) {
+
+    if (m_name.at(0) == '\\') {
+        m_type = RegularExpression;
+    } else if (m_name.find(":") != std::string::npos) {
+        m_type = SingleMatch;
+    } else {
+        m_type = MultipleMatches;
+    }
+
+    if (m_name.find(".") != std::string::npos) {
+        m_collectionName = std::string(m_name, 0, m_name.find("."));
+    }
+}
+
+
 std::list<transaction::Variable *> *
     Variable::evaluate(Assay *assay) {
     std::list<transaction::Variable *> *l =
         new std::list<transaction::Variable *>();
-    assay->m_collections.resolve(this->name, l);
+
+    if (m_collectionName.empty() == false) {
+        if (m_kind == CollectionVarible && m_type == MultipleMatches) {
+            assay->m_collections.resolveMultiMatches(m_name, m_collectionName, l);
+        } if (m_kind == CollectionVarible && m_type == RegularExpression) {
+            assay->m_collections.resolveRegularExpression(m_name, m_collectionName, l);
+        } else {
+            assay->m_collections.resolveSingleMatch(m_name, m_collectionName, l);
+        }
+    } else {
+        if (m_kind == CollectionVarible && m_type == MultipleMatches) {
+            assay->m_collections.resolveMultiMatches(m_name, l);
+        } if (m_kind == CollectionVarible && m_type == RegularExpression) {
+            assay->m_collections.resolveRegularExpression(m_name, l);
+        } else {
+            assay->m_collections.resolveSingleMatch(m_name, l);
+        }
+    }
+
     return l;
 }
 
+
 std::string Variable::to_s(
     std::vector<Variable *> *variables) {
     std::string ret;
     std::string except("");
     for (int i = 0; i < variables->size() ; i++) {
-        std::string name = variables->at(i)->name;
+        std::string name = variables->at(i)->m_name;
         Exclusion *e =  dynamic_cast<Exclusion *>(variables->at(i));
         if (e != NULL) {
             if (except.empty()) {

src/variables/variable.h

@@ -29,13 +29,51 @@ namespace Variables {
 
 class Variable {
  public:
-    explicit Variable(std::string _name)
+    /**
-        : name(_name) { }
+     *
+     */
+    enum VariableType {
+      /**
+       *
+       */
+      SingleMatch,
+      /**
+       *
+       */
+      MultipleMatches,
+      /**
+       *
+       */
+      RegularExpression
+    };
+
+    /**
+     *
+     */
+    enum VariableKind {
+      /**
+       *
+       */
+      DirectVariable,
+      /**
+       *
+       */
+      CollectionVarible,
+    };
+
+    explicit Variable(std::string _name);
+    Variable(std::string name, VariableKind kind);
 
     static std::string to_s(std::vector<Variable *> *variables);
-    virtual std::list<transaction::Variable *> *
+
-        evaluate(Assay *assay);
+    virtual std::list<transaction::Variable *> *evaluate(Assay *assay);
-    std::string name;
+    //virtual std::list<transaction::Variable *> *eval_int(Assay *assay);
+
+    std::string m_name;
+    std::string m_collectionName;
+
+    VariableType m_type;
+    VariableKind m_kind;
 };
 
 

src/variables/variations/count.cc

@@ -49,7 +49,7 @@ std::list<transaction::Variable *> *
 
     std::string res = std::to_string(count);
 
-    reslOut->push_back(new transaction::Variable(std::string(var->name),
+    reslOut->push_back(new transaction::Variable(std::string(var->m_name),
         std::string(res)));
 
     return reslOut;

src/variables/variations/count.h

@@ -32,7 +32,7 @@ namespace Variations {
 class Count : public Variable {
  public:
     explicit Count(Variable *v)
-        : Variable("count(" + v->name + ")"),
+        : Variable("count(" + v->m_name + ")"),
         var(v) { }
 
     std::list<transaction::Variable *> *

src/variables/variations/exclusion.cc

@@ -33,7 +33,7 @@ std::list<transaction::Variable *> *
     Exclusion::evaluate(Assay *assay) {
     std::list<transaction::Variable *> *l =
         new std::list<transaction::Variable *>();
-    assay->m_collections.resolve(this->name, l);
+    assay->m_collections.resolve(this->m_name, l);
     return l;
 }
 

src/variables/variations/exclusion.h

@@ -33,7 +33,7 @@ namespace Variations {
 class Exclusion : public Variable {
  public:
     explicit Exclusion(Variable *v)
-        : Variable(v->name),
+        : Variable(v->m_name),
         var(v) { }
 
     std::list<transaction::Variable *> *