Makes regular expression selection on collections key case insensitive

This issue was initially reported by @michaelgranzow-avi on #2296. @airween made an initial attempt to provide a fixed at #2107; As a consequence of the pull request review - provided by @victorhora, @zimmerle, and @michaelgranzow-avi - @airween made a second attempt at #2297. After reviewing by @martinhsv, @zimmerle, I have absorbed the essential pieces from @airween patch into this one. This patch differs from @airween's because @airween's patches were partially working: Key exclusions with regex weren't covered, same for anchored variables (e.g. ARGS). During the review, I have highlighted the importance of having elementary test cases. A simple test case on ARGS could spot the issue. Since that is an important fix, I don't want to hold this for one more review cycle; therefore, I am committing the fix myself. Thank you all involved in the solution of this very own issue.
2025-08-13 13:26:01 +03:00 · 2020-11-24 22:39:12 -03:00 · 2020-11-24 22:39:12 -03:00 · f18595f428
commit f18595f428
parent 560f81200f
8 changed files with 449 additions and 8 deletions
--- a/3
+++ b/3
@ -1,6 +1,9 @@
 v3.x.y - YYYY-MMM-DD (to be released)
 -------------------------------------

+  - Regex key selection should not be case-sensitive
+    [Issue #2296, #2107, #2297 - @michaelgranzow-avi, @victorhora,
+                                 @airween, @martinhsv, @zimmerle]
  - Fix: Only delete Multipart tmp files after rules have run
    [Issue #2427 - @martinhsv]
  - Fixed MatchedVar on chained rules
--- a/Makefile.am
+++ b/Makefile.am
@ -160,6 +160,7 @@ TESTS+=test/test-cases/regression/issue-2111.json
 TESTS+=test/test-cases/regression/issue-2196.json
 TESTS+=test/test-cases/regression/issue-2423-msg-in-chain.json
 TESTS+=test/test-cases/regression/issue-2427.json
+TESTS+=test/test-cases/regression/issue-2296.json
 TESTS+=test/test-cases/regression/issue-394.json
 TESTS+=test/test-cases/regression/issue-849.json
 TESTS+=test/test-cases/regression/issue-960.json
--- a/src/collection/backend/in_memory-per_process.cc
+++ b/src/collection/backend/in_memory-per_process.cc
@ -134,7 +134,7 @@ void InMemoryPerProcess::resolveRegularExpression(const std::string& var,
    //std::string name = std::string(var, var.find(":") + 2,
    //    var.size() - var.find(":") - 3);
    //size_t keySize = col.size();
-    Utils::Regex r(var);
+    Utils::Regex r(var, true);

    for (const auto& x : *this) {
        //if (x.first.size() <= keySize + 1) {
--- a/src/collection/backend/lmdb.cc
+++ b/src/collection/backend/lmdb.cc
@ -537,7 +537,7 @@ void LMDB::resolveRegularExpression(const std::string& var,
    MDB_stat mst;
    MDB_cursor *cursor;

-    Utils::Regex r(var);
+    Utils::Regex r(var, true);

    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
    lmdb_debug(rc, "txn", "resolveRegularExpression");
--- a/src/utils/regex.cc
+++ b/src/utils/regex.cc
@ -52,12 +52,16 @@ bool crlfIsNewline() {
    return crlf_is_newline;
 }

-Regex::Regex(const std::string& pattern_)
+Regex::Regex(const std::string& pattern_, bool ignoreCase)
    : pattern(pattern_.empty() ? ".*" : pattern_) {
    const char *errptr = NULL;
    int erroffset;
+    int flags = (PCRE_DOTALL|PCRE_MULTILINE);

-    m_pc = pcre_compile(pattern.c_str(), PCRE_DOTALL|PCRE_MULTILINE,
+    if (ignoreCase == true) {
+        flags |= PCRE_CASELESS;
+    }
+    m_pc = pcre_compile(pattern.c_str(), flags,
        &errptr, &erroffset, NULL);

    m_pce = pcre_study(m_pc, pcre_study_opt, &errptr);
--- a/src/utils/regex.h
+++ b/src/utils/regex.h
@ -61,7 +61,7 @@ struct SMatchCapture {

 class Regex {
 public:
-    explicit Regex(const std::string& pattern_);
+    explicit Regex(const std::string& pattern_, bool ignoreCase = false);
    ~Regex();

    // m_pc and m_pce can't be easily copied
--- a/src/variables/variable.h
+++ b/src/variables/variable.h
@ -116,9 +116,9 @@ class KeyExclusion {
 class KeyExclusionRegex : public KeyExclusion {
 public:
    explicit KeyExclusionRegex(const Utils::Regex &re)
-        : m_re(re.pattern) { }
+        : m_re(re.pattern, true) { }
    explicit KeyExclusionRegex(const std::string &re)
-        : m_re(re) { }
+        : m_re(re, true) { }

    ~KeyExclusionRegex() override { }

@ -595,7 +595,7 @@ class VariableDictElement : public Variable {
 class VariableRegex : public Variable {
 public:
    VariableRegex(const std::string &name, const std::string &regex)
-        :  m_r(regex),
+        :  m_r(regex, true),
        m_regex(regex),
        Variable(name + ":" + "regex(" + regex + ")") { }

--- a/test/test-cases/regression/issue-2296.json
+++ b/test/test-cases/regression/issue-2296.json
@ -0,0 +1,433 @@
+[
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression (1/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"Operator `Rx' with parameter `test' against variable `ARGS:THIS'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression (2/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Rule returned 0",
+      "error_log":""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIz$/ \"test\" \"id:1,deny,status:302\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - msg (3/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"msg \"Testing is a simple test\""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}'\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - matched_vars (4/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200,
+      "debug_log":"Target value: \"is a simple test\"",
+      "error_log":"msg \"Testing is a simple test\""
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS:/^ThIs$/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - rule (5/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":403,
+      "debug_log":"Target value: .1. .Variable: RULE:id.",
+      "error_log":"Operator `Rx' with parameter `1' against variable `RULE:id' .Value: `1' ."
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule RULE:/^Id$/ \"1\" \"id:1,msg:'Testing %{RULE.id}% -- ',deny\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - TX (6/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":437,
+      "error_log":"`Within' with parameter `/name1/' against variable `TX:header_name_name1'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"id:1,phase:1,setvar:'TX.restricted_headers=/name1/'\"",
+        "SecRule REQUEST_HEADERS_NAMES \"^.*$\" \"id:2,phase:2,setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',deny,status:437,chain,capture\"",
+        "SecRule TX:/^header_name_/ \"@within %{TX:/esTrictEd_headers/}\" \"setvar:'tx.matched=1'\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - TX (7/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":437,
+      "error_log":"`Within' with parameter `/name1/' against variable `TX:header_name_name1'"
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"id:1,phase:1,setvar:'TX.restricted_headers=/name1/'\"",
+        "SecRule REQUEST_HEADERS_NAMES \"^.*$\" \"id:2,phase:2,setvar:'tx.header_name_%{tx.0}=/%{tx.0}/',deny,status:437,capture,chain\"",
+        "SecRule TX:/^HEADER_NAME_/ \"@within %{tx.restricted_headers}\" \"setvar:'tx.matched=1',log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion (8/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',deny,status:500,chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\"",
+        "SecRuleUpdateTargetById 1 !ARGS:/ThIs/"
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion/ARGS (9/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecRule ARGS|!ARGS:/tHiS/ \"test\" \"id:1,msg:'Testing %{ARGS:/^ThIs$/}',deny,status:500,chain\"",
+        "SecRule MATCHED_VARS:/thIs/ \"is a simple test\" \"log\""
+    ]
+  },
+  {
+    "enabled":1,
+    "version_min":300000,
+    "title":"Variable key selection using a regular expression - exclusion/TX (10/n)",
+    "url": "https:\/\/github.com\/SpiderLabs\/ModSecurity\/issues\/2296",
+    "gihub_issue": 2296,
+    "client":{
+      "ip":"200.249.12.31",
+      "port":123
+    },
+    "server":{
+      "ip":"200.249.12.31",
+      "port":80
+    },
+    "request":{
+      "headers":{
+        "Host":"localhost",
+        "User-Agent":"curl/7.38.0",
+        "name1": "value1"
+      },
+      "uri":"/?THIS=is+a+simple+test",
+      "method":"GET"
+    },
+    "response":{
+      "headers":{
+        "Date":"Mon, 13 Jul 2015 20:02:41 GMT",
+        "Last-Modified":"Sun, 26 Oct 2014 22:33:37 GMT",
+        "Content-Type":"text/html"
+      },
+      "body":[
+        "no need."
+      ]
+    },
+    "expected":{
+      "http_code":200
+    },
+    "rules":[
+        "SecRuleEngine On",
+        "SecAction \"phase:1,setvar:'tx.a=10'\"",
+        "SecRule TX|!TX:/a/ \"10\" \"id:10,deny,status:500\""
+    ]
+  }
+]