Makes regular expression selection on collections key case insensitive

This issue was initially reported by @michaelgranzow-avi on #2296.

@airween made an initial attempt to provide a fixed at #2107; As a
consequence of the pull request review - provided by @victorhora,
@zimmerle, and @michaelgranzow-avi - @airween made a second attempt
at #2297. After reviewing by @martinhsv, @zimmerle, I have absorbed
the essential pieces from @airween patch into this one.

This patch differs from @airween's because @airween's patches were
partially working: Key exclusions with regex weren't covered, same
for anchored variables (e.g. ARGS). During the review, I have
highlighted the importance of having elementary test cases. A simple
test case on ARGS could spot the issue. Since that is an important
fix, I don't want to hold this for one more review cycle; therefore,
I am committing the fix myself.

Thank you all involved in the solution of this very own issue.

CHANGES

@@ -1,6 +1,9 @@
 v3.x.y - YYYY-MMM-DD (to be released)
 -------------------------------------
 
+  - Regex key selection should not be case-sensitive
+    [Issue #2296, #2107, #2297 - @michaelgranzow-avi, @victorhora,
+                                 @airween, @martinhsv, @zimmerle]
   - Fix: Only delete Multipart tmp files after rules have run
     [Issue #2427 - @martinhsv]
   - Fixed MatchedVar on chained rules

Makefile.am

@@ -160,6 +160,7 @@ TESTS+=test/test-cases/regression/issue-2111.json
 TESTS+=test/test-cases/regression/issue-2196.json
 TESTS+=test/test-cases/regression/issue-2423-msg-in-chain.json
 TESTS+=test/test-cases/regression/issue-2427.json
+TESTS+=test/test-cases/regression/issue-2296.json
 TESTS+=test/test-cases/regression/issue-394.json
 TESTS+=test/test-cases/regression/issue-849.json
 TESTS+=test/test-cases/regression/issue-960.json

src/collection/backend/in_memory-per_process.cc

@@ -134,7 +134,7 @@ void InMemoryPerProcess::resolveRegularExpression(const std::string& var,
     //std::string name = std::string(var, var.find(":") + 2,
     //    var.size() - var.find(":") - 3);
     //size_t keySize = col.size();
-    Utils::Regex r(var);
+    Utils::Regex r(var, true);
 
     for (const auto& x : *this) {
         //if (x.first.size() <= keySize + 1) {

src/collection/backend/lmdb.cc

@@ -537,7 +537,7 @@ void LMDB::resolveRegularExpression(const std::string& var,
     MDB_stat mst;
     MDB_cursor *cursor;
 
-    Utils::Regex r(var);
+    Utils::Regex r(var, true);
 
     rc = mdb_txn_begin(m_env, NULL, 0, &txn);
     lmdb_debug(rc, "txn", "resolveRegularExpression");

src/utils/regex.cc

@@ -52,12 +52,16 @@ bool crlfIsNewline() {
     return crlf_is_newline;
 }
 
-Regex::Regex(const std::string& pattern_)
+Regex::Regex(const std::string& pattern_, bool ignoreCase)
     : pattern(pattern_.empty() ? ".*" : pattern_) {
     const char *errptr = NULL;
     int erroffset;
+    int flags = (PCRE_DOTALL|PCRE_MULTILINE);
 
-    m_pc = pcre_compile(pattern.c_str(), PCRE_DOTALL|PCRE_MULTILINE,
+    if (ignoreCase == true) {
+        flags |= PCRE_CASELESS;
+    }
+    m_pc = pcre_compile(pattern.c_str(), flags,
         &errptr, &erroffset, NULL);
 
     m_pce = pcre_study(m_pc, pcre_study_opt, &errptr);

src/utils/regex.h

@@ -61,7 +61,7 @@ struct SMatchCapture {
 
 class Regex {
  public:
-    explicit Regex(const std::string& pattern_);
+    explicit Regex(const std::string& pattern_, bool ignoreCase = false);
     ~Regex();
 
     // m_pc and m_pce can't be easily copied

src/variables/variable.h

@@ -116,9 +116,9 @@ class KeyExclusion {
 class KeyExclusionRegex : public KeyExclusion {
  public:
     explicit KeyExclusionRegex(const Utils::Regex &re)
-        : m_re(re.pattern) { }
+        : m_re(re.pattern, true) { }
     explicit KeyExclusionRegex(const std::string &re)
-        : m_re(re) { }
+        : m_re(re, true) { }
 
     ~KeyExclusionRegex() override { }
 
@@ -595,7 +595,7 @@ class VariableDictElement : public Variable {
 class VariableRegex : public Variable {
  public:
     VariableRegex(const std::string &name, const std::string &regex)
-        :  m_r(regex),
+        :  m_r(regex, true),
         m_regex(regex),
         Variable(name + ":" + "regex(" + regex + ")") { }
 

test/test-cases/regression/issue-2296.json

@@ -0,0 +1,433 @@
+[
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression (1/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"Operator `Rx' with parameter `test' against variable `ARGS:THIS'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression (2/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Rule returned 0",
+      "error_log":""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIz$/ \"test\" \"id:1,deny,status:302\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - msg (3/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"msg \"Testing is a simple test\""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}'\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - matched_vars (4/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"msg \"Testing is a simple test\""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - rule (5/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":403,
+      "debug_log":"Target value: .1. .Variable: RULE:id.",
+      "error_log":"Operator `Rx' with parameter `1' against variable `RULE:id' .Value: `1' ."
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule RULE:/^Id$/ \"1\" \"id:1,msg:'Testing %{RULE.id}% -- ',deny\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - TX (6/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":437,
+      "error_log":"`Within' with parameter `/name1/' against variable `TX:header_name_name1'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"id:1,phase:1,setvar:'TX.restricted_headers=/name1/'\"",
+        "SecRule REQUEST_HEADERS_NAMES \"^.*$\" \"id:2,phase:2,setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',deny,status:437,chain,capture\"",
+        "SecRule TX:/^header_name_/ \"@within %{TX:/esTrictEd_headers/}\" \"setvar:'tx.matched=1'\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - TX (7/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":437,
+      "error_log":"`Within' with parameter `/name1/' against variable `TX:header_name_name1'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"id:1,phase:1,setvar:'TX.restricted_headers=/name1/'\"",
+        "SecRule REQUEST_HEADERS_NAMES \"^.*$\" \"id:2,phase:2,setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',deny,status:437,capture,chain\"",
+        "SecRule TX:/^HEADER_NAME_/ \"@within %{tx.restricted_headers}\" \"setvar:'tx.matched=1',log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion (8/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',deny,status:500,chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\"",
+        "SecRuleUpdateTargetById 1 !ARGS:/ThIs/"
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion/ARGS (9/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS|!ARGS:/tHiS/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',deny,status:500,chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion/TX (10/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"phase:1,setvar:'tx.a=10'\"",
+        "SecRule TX|!TX:/a/ \"10\" \"id:10,deny,status:500\""
+    ]
+  }
+]