Sync up branches/2.5.x and trunk.

CHANGES

@@ -1,33 +1,61 @@
+31 Jul 2008 - trunk
+=======
 
-16 Jul 2008 - trunk
+ * Implement cssDecode.
+
+ * Persistent counter updates are now atomic.
+
+
+31 Jul 2008 - 2.5.6
 -------------------
+  
+ * Transformation caching has been deprecated, and is now off by default. We
+   now advise against using transformation caching in production.
+   
+ * Fixed two separate transformation caching issues that could cause incorrect
+   content inspection in some circumstances.
+   
+ * Fixed an issue with the transformation cache using too much RAM, potentially
+   crashing Apache with a large number of cache entries. Two new configuration
+   options have been added to allow for a finer control of caching:
+   
+     maxitems: Max number of items to cache (default 1024)
+     incremental: Whether to cache incrementally (default off)
 
-* Implement cssDecode.
+ * Added an experimental regression testing suite. The regression suite may
+   be executed via "make test-regression", however it is strongly advised 
+   to only be executed on a non-production machine as it will startup the
+   Apache web server that ModSecurity is compiled against with various
+   configurations in which it will run tests.
 
-* Persistent counter updates are now atomic.
+ * Added a licensing exception so that ModSecurity can be used in a derivative
+   work when that derivative is also under an approved open source license.
+
+ * Updated mlogc to version 1.4.5 which adds a LockFile directive and fixes an
+   issue in which the configuration file may be deleted.
 
 
 05 Jun 2008 - 2.5.5
 -------------------
 
-* Fixed an issue where an alert was not logged in the error log
+ * Fixed an issue where an alert was not logged in the error log
-  unless "auditlog" was used.
+   unless "auditlog" was used.
 
-* Enable the "auditlog" action by default to help prevent a misconfiguration.
+ * Enable the "auditlog" action by default to help prevent a misconfiguration.
-  The new default is now: "phase:2,log,auditlog,pass"
+   The new default is now: "phase:2,log,auditlog,pass"
 
-* Improve request body processing error messages.
+ * Improve request body processing error messages.
 
-* Handle lack of a new line after the final boundary in a multipart request.
+ * Handle lack of a new line after the final boundary in a multipart request.
-  This fixes the reported WordPress Flash file uploader problem.
+   This fixes the reported WordPress Flash file uploader problem.
 
-* Fixed issue with multithreaded servers where concurrent XML processing
+ * Fixed issue with multithreaded servers where concurrent XML processing
-  could crash the web server (at least under Windows).
+   could crash the web server (at least under Windows).
 
-* Fixed blocking in phase 3.
+ * Fixed blocking in phase 3.
 
-* Force modules "mod_rpaf-2.0.c" and "mod_custom_header.c" to run before
+ * Force modules "mod_rpaf-2.0.c" and "mod_custom_header.c" to run before
-  ModSecurity so that the correct IP is used.
+   ModSecurity so that the correct IP is used.
 
 
 07 May 2008 - 2.5.4

README.TXT

@@ -5,9 +5,11 @@ ModSecurity for Apache is an open source product, released under terms of
 the General Public Licence, Version 2 (GPLv2). Please refer to the
 file LICENSE, which contains the complete text of the licence.
 
-Additionally, certain software may be granted further rights via
+There are special exceptions to the terms and conditions of the GPL
-a licensing exception.  Please refer to the file MODSECURITY_LICENSE_EXCEPTION,
+as it is applied to this software. View the full text of the exception in
-which contains the complete exception.
+file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+distribution.
+
 
 DOCUMENTATION
 

apache2/acmp.c

@@ -2,12 +2,23 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
+
+/* Aho-Corasick Matching  */
+
 #include "acmp.h"
 
 #ifdef ACMP_USE_UTF8

apache2/acmp.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef ACMP_H_

apache2/apache2.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _APACHE2_H_

apache2/apache2_config.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <limits.h>
@@ -102,8 +110,10 @@ void *create_directory_config(apr_pool_t *mp, char *path) {
 
     /* Cache */
     dcfg->cache_trans = NOT_SET;
+    dcfg->cache_trans_incremental = NOT_SET;
     dcfg->cache_trans_min = NOT_SET;
     dcfg->cache_trans_max = NOT_SET;
+    dcfg->cache_trans_maxitems = NOT_SET;
 
     dcfg->component_signatures = apr_array_make(mp, 16, sizeof(char *));
 
@@ -439,10 +449,14 @@ void *merge_directory_configs(apr_pool_t *mp, void *_parent, void *_child) {
     /* Cache */
     merged->cache_trans = (child->cache_trans == NOT_SET
         ? parent->cache_trans : child->cache_trans);
+    merged->cache_trans_incremental = (child->cache_trans_incremental == NOT_SET
+        ? parent->cache_trans_incremental : child->cache_trans_incremental);
     merged->cache_trans_min = (child->cache_trans_min == (apr_size_t)NOT_SET
         ? parent->cache_trans_min : child->cache_trans_min);
     merged->cache_trans_max = (child->cache_trans_max == (apr_size_t)NOT_SET
         ? parent->cache_trans_max : child->cache_trans_max);
+    merged->cache_trans_maxitems = (child->cache_trans_maxitems == (apr_size_t)NOT_SET
+        ? parent->cache_trans_maxitems : child->cache_trans_maxitems);
 
     /* Merge component signatures. */
     merged->component_signatures = apr_array_append(mp, parent->component_signatures,
@@ -528,9 +542,11 @@ void init_directory_config(directory_config *dcfg) {
     if (dcfg->geo == NOT_SET_P) dcfg->geo = NULL;
 
     /* Cache */
-    if (dcfg->cache_trans == NOT_SET) dcfg->cache_trans = MODSEC_CACHE_ENABLED;
+    if (dcfg->cache_trans == NOT_SET) dcfg->cache_trans = MODSEC_CACHE_DISABLED;
-    if (dcfg->cache_trans_min == (apr_size_t)NOT_SET) dcfg->cache_trans_min = 15;
+    if (dcfg->cache_trans_incremental == NOT_SET) dcfg->cache_trans_incremental = 0;
-    if (dcfg->cache_trans_max == (apr_size_t)NOT_SET) dcfg->cache_trans_max = 0;
+    if (dcfg->cache_trans_min == (apr_size_t)NOT_SET) dcfg->cache_trans_min = 32;
+    if (dcfg->cache_trans_max == (apr_size_t)NOT_SET) dcfg->cache_trans_max = 1024;
+    if (dcfg->cache_trans_maxitems == (apr_size_t)NOT_SET) dcfg->cache_trans_maxitems = 512;
 
     if (dcfg->request_encoding == NOT_SET_P) dcfg->request_encoding = NULL;
 }
@@ -1650,7 +1666,7 @@ static const char *cmd_cache_transformations(cmd_parms *cmd, void *_dcfg, const
 
     /* Process options */
     if (p2 != NULL) {
-        apr_table_t *vartable = apr_table_make(cmd->pool, 10);
+        apr_table_t *vartable = apr_table_make(cmd->pool, 4);
         apr_status_t rc;
         char *error_msg = NULL;
         const char *charval = NULL;
@@ -1664,7 +1680,18 @@ static const char *cmd_cache_transformations(cmd_parms *cmd, void *_dcfg, const
             return apr_psprintf(cmd->pool, "ModSecurity: Unable to parse options for SecCacheTransformations: %s", error_msg);
         }
 
-        /* minval */
+        /* incremental */
+        charval = apr_table_get(vartable, "incremental");
+        if (charval != NULL) {
+            if (strcasecmp(charval, "on") == 0)
+                dcfg->cache_trans_incremental = 1;
+            else if (strcasecmp(charval, "off") == 0)
+                dcfg->cache_trans_incremental = 0;
+            else
+                return apr_psprintf(cmd->pool, "ModSecurity: SecCacheTransformations invalid incremental value: %s", charval);
+        }
+
+        /* minlen */
         charval = apr_table_get(vartable, "minlen");
         if (charval != NULL) {
             intval = apr_atoi64(charval);
@@ -1684,7 +1711,7 @@ static const char *cmd_cache_transformations(cmd_parms *cmd, void *_dcfg, const
             dcfg->cache_trans_min = (apr_size_t)intval;
         }
 
-        /* maxval */
+        /* maxlen */
         charval = apr_table_get(vartable, "maxlen");
         if (charval != NULL) {
             intval = apr_atoi64(charval);
@@ -1707,6 +1734,19 @@ static const char *cmd_cache_transformations(cmd_parms *cmd, void *_dcfg, const
             dcfg->cache_trans_max = (apr_size_t)intval;
 
         }
+
+        /* maxitems */
+        charval = apr_table_get(vartable, "maxitems");
+        if (charval != NULL) {
+            intval = apr_atoi64(charval);
+            if (errno == ERANGE) {
+                return apr_psprintf(cmd->pool, "ModSecurity: SecCacheTransformations maxitems out of range: %s", charval);
+            }
+            if (intval < 0) {
+                return apr_psprintf(cmd->pool, "ModSecurity: SecCacheTransformations maxitems must be positive: %s", charval);
+            }
+            dcfg->cache_trans_maxitems = (apr_size_t)intval;
+        }
     }
 
     return NULL;

apache2/apache2_io.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "modsecurity.h"

apache2/apache2_util.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "modsecurity.h"
@@ -105,6 +113,10 @@ int apache2_exec(modsec_rec *msr, const char *command, const char **argv, char *
     apr_procattr_io_set(procattr, APR_NO_PIPE, APR_FULL_BLOCK, APR_NO_PIPE);
     apr_procattr_cmdtype_set(procattr, APR_SHELLCMD);
 
+    if (msr->txcfg->debuglog_level >= 9) {
+        msr_log(msr, 9, "Exec: %s", log_escape_nq(r->pool, command));
+    }
+
     rc = apr_proc_create(procnew, command, argv, env, procattr, r->pool);
     if (rc != APR_SUCCESS) {
         msr_log(msr, 1, "Exec: Execution failed: %s (%s)", log_escape_nq(r->pool, command),
@@ -139,8 +151,10 @@ int apache2_exec(modsec_rec *msr, const char *command, const char **argv, char *
                 p++;
             }
 
-            msr_log(msr, 4, "Exec: First line from script output: \"%s\"",
+            if (msr->txcfg->debuglog_level >= 4) {
-                log_escape(r->pool, buf));
+                msr_log(msr, 4, "Exec: First line from script output: \"%s\"",
+                    log_escape(r->pool, buf));
+            }
 
             if (output != NULL) *output = apr_pstrdup(r->pool, buf);
 
@@ -189,7 +203,9 @@ void record_time_checkpoint(modsec_rec *msr, int checkpoint_no) {
     apr_snprintf(note_name, 99, "mod_security-time%d", checkpoint_no);
     apr_table_set(msr->r->notes, note_name, note);
 
-    msr_log(msr, 4, "Time #%d: %s", checkpoint_no, note);
+    if (msr->txcfg->debuglog_level >= 4) {
+        msr_log(msr, 4, "Time #%d: %s", checkpoint_no, note);
+    }
 }
 
 /**

apache2/configure

@@ -1299,6 +1299,7 @@ Optional Features:
   --enable-debug-conf     Enable debug during configuration.
   --enable-debug-cache    Enable debug for transformation caching.
   --enable-debug-acmp     Enable debugging acmp code.
+  --enable-debug-mem      Enable debug during configuration.
   --enable-performance-measurement
                           Enable performance-measurement stats.
   --disable-modsec-api    Disable the API; compiling against some older Apache
@@ -5257,6 +5258,23 @@ else
 fi
 
 
+# DEBUG_MEM
+# Check whether --enable-debug-mem was given.
+if test "${enable_debug_mem+set}" = set; then
+  enableval=$enable_debug_mem;
+  if test "$enableval" != "no"; then
+    debug_mem="-DDEBUG_MEM"
+  else
+    debug_mem=
+  fi
+
+else
+
+  debug_mem=
+
+fi
+
+
 # PERFORMANCE_MEASUREMENT
 # Check whether --enable-performance-measurement was given.
 if test "${enable_performance_measurement+set}" = set; then
@@ -5293,8 +5311,12 @@ fi
 
 ### Build *EXTRA_CFLAGS vars
 
-EXTRA_CFLAGS="-O2 -g -Wall $strict_compile"
+if test -n "$debug_mem"; then
-MODSEC_EXTRA_CFLAGS="$debug_conf $debug_cache $debug_acmp $perf_meas $modsec_api"
+  EXTRA_CFLAGS="-O0 -g -Wall"
+else
+  EXTRA_CFLAGS="-O2 -g -Wall $strict_compile"
+fi
+MODSEC_EXTRA_CFLAGS="$debug_conf $debug_cache $debug_acmp $debug_mem $perf_meas $modsec_api"
 
 APXS_WRAPPER=build/apxs-wrapper
 APXS_EXTRA_CFLAGS=""

apache2/configure.in

@@ -225,6 +225,21 @@ AC_ARG_ENABLE(debug-acmp,
   debug_acmp=
 ])
 
+# DEBUG_MEM
+AC_ARG_ENABLE(debug-mem,
+              AS_HELP_STRING([--enable-debug-mem],
+                             [Enable debug during configuration.]),
+[
+  if test "$enableval" != "no"; then
+    debug_mem="-DDEBUG_MEM"
+  else
+    debug_mem=
+  fi
+],
+[
+  debug_mem=
+])
+
 # PERFORMANCE_MEASUREMENT
 AC_ARG_ENABLE(performance-measurement,
               AS_HELP_STRING([--enable-performance-measurement],
@@ -257,8 +272,12 @@ AC_ARG_ENABLE(modsec-api,
 
 ### Build *EXTRA_CFLAGS vars
 
-EXTRA_CFLAGS="-O2 -g -Wall $strict_compile"
+if test -n "$debug_mem"; then
-MODSEC_EXTRA_CFLAGS="$debug_conf $debug_cache $debug_acmp $perf_meas $modsec_api"
+  EXTRA_CFLAGS="-O0 -g -Wall"
+else
+  EXTRA_CFLAGS="-O2 -g -Wall $strict_compile"
+fi
+MODSEC_EXTRA_CFLAGS="$debug_conf $debug_cache $debug_acmp $debug_mem $perf_meas $modsec_api"
 
 APXS_WRAPPER=build/apxs-wrapper
 APXS_EXTRA_CFLAGS=""

apache2/mod_security2.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <limits.h>
@@ -962,6 +970,8 @@ static void hook_insert_filter(request_rec *r) {
     }
 }
 
+// TODO: Holding off on this for now (needs more testing)
+#if 0
 /**
  * Invoked whenever Apache starts processing an error. A chance
  * to insert ourselves into the output filter chain.
@@ -1003,6 +1013,7 @@ static void hook_insert_error_filter(request_rec *r) {
         }
     }
 }
+#endif
 
 #if (!defined(NO_MODSEC_API))
 /**
@@ -1106,7 +1117,9 @@ static void register_hooks(apr_pool_t *mp) {
 
     /* Filter hooks */
     ap_hook_insert_filter(hook_insert_filter, NULL, NULL, APR_HOOK_FIRST);
+#if 0
     ap_hook_insert_error_filter(hook_insert_error_filter, NULL, NULL, APR_HOOK_FIRST);
+#endif
 
     ap_register_input_filter("MODSECURITY_IN", input_filter,
         NULL, AP_FTYPE_CONTENT_SET);

apache2/modsecurity.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <stdlib.h>
@@ -314,8 +322,8 @@ apr_status_t modsecurity_tx_init(modsec_rec *msr) {
     if (msr->collections_dirty == NULL) return -1;
 
     /* Other */
-    msr->tcache = apr_hash_make(msr->mp);
+    msr->tcache = NULL;
-    if (msr->tcache == NULL) return -1;
+    msr->tcache_items = 0;
 
     msr->matched_rules = apr_array_make(msr->mp, 16, sizeof(void *));
     if (msr->matched_rules == NULL) return -1;
@@ -497,19 +505,76 @@ static apr_status_t modsecurity_process_phase_logging(modsec_rec *msr) {
 apr_status_t modsecurity_process_phase(modsec_rec *msr, unsigned int phase) {
     /* Check if we should run. */
     if ((msr->was_intercepted)&&(phase != PHASE_LOGGING)) {
-        msr_log(msr, 4, "Skipping phase %i as request was already intercepted.", phase);
+        msr_log(msr, 4, "Skipping phase %d as request was already intercepted.", phase);
         return 0;
     }
 
     /* Do not process the same phase twice. */
     if (msr->phase >= phase) {
-        msr_log(msr, 4, "Skipping phase %i because it was previously run (at %i now).",
+        msr_log(msr, 4, "Skipping phase %d because it was previously run (at %d now).",
             phase, msr->phase);
         return 0;
     }
 
     msr->phase = phase;
 
+    /* Clear out the transformation cache at the start of each phase */
+    if (msr->txcfg->cache_trans == MODSEC_CACHE_ENABLED) {
+        if (msr->tcache) {
+            apr_hash_index_t *hi;
+            void *dummy;
+            apr_table_t *tab;
+            const void *key;
+            apr_ssize_t klen;
+            #ifdef CACHE_DEBUG
+            apr_pool_t *mp = msr->msc_rule_mptmp;
+            const apr_array_header_t *ctarr;
+            const apr_table_entry_t *ctelts;
+            msre_cache_rec *rec;
+            int cn = 0;
+            int ri;
+            #else
+            apr_pool_t *mp = msr->mp;
+            #endif
+
+            for (hi = apr_hash_first(mp, msr->tcache); hi; hi = apr_hash_next(hi)) {
+                apr_hash_this(hi, &key, &klen, &dummy);
+                tab = (apr_table_t *)dummy;
+
+                if (tab == NULL) continue;
+
+                #ifdef CACHE_DEBUG
+                /* Dump the cache out as we clear */
+                ctarr = apr_table_elts(tab);
+                ctelts = (const apr_table_entry_t*)ctarr->elts;
+                for (ri = 0; ri < ctarr->nelts; ri++) {
+                    cn++;
+                    rec = (msre_cache_rec *)ctelts[ri].val;
+                    if (rec->changed) {
+                        if (msr->txcfg->debuglog_level >= 9) {
+                            msr_log(msr, 9, "CACHE: %5d) hits=%d key=%pp %x;%s=\"%s\" (%pp - %pp)", cn, rec->hits, key, rec->num, rec->path, log_escape_nq_ex(mp, rec->val, rec->val_len), rec->val, rec->val + rec->val_len);
+                        }
+                    }
+                    else {
+                        if (msr->txcfg->debuglog_level >= 9) {
+                            msr_log(msr, 9, "CACHE: %5d) hits=%d key=%pp %x;%s=<no change>", cn, rec->hits, key, rec->num, rec->path);
+                        }
+                    }
+                }
+                #endif
+
+                apr_table_clear(tab);
+                apr_hash_set(msr->tcache, key, klen, NULL);
+            }
+
+            msr_log(msr, 9, "Cleared transformation cache for phase %d", msr->phase);
+        }
+
+        msr->tcache_items = 0;
+        msr->tcache = apr_hash_make(msr->mp);
+        if (msr->tcache == NULL) return -1;
+    }
+
     switch(phase) {
         case 1 :
             return modsecurity_process_phase_request_headers(msr);

apache2/modsecurity.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MODSECURITY_H_
@@ -30,6 +38,10 @@ typedef struct msc_string msc_string;
 #define DSOLOCAL
 #endif
 
+#if defined(DEBUG_MEM)
+/* Nothing Yet */
+#endif
+
 /* For GNU C, tell the compiler to check printf like formatters */
 #if (defined(__GNUC__) && !defined(SOLARIS2))
 #define PRINTF_ATTRIBUTE(a,b) __attribute__((format (printf, a, b)))
@@ -366,6 +378,7 @@ struct modsec_rec {
 
     /* data cache */
     apr_hash_t          *tcache;
+    apr_size_t           tcache_items;
 
     /* removed rules */
     apr_array_header_t  *removed_rules;
@@ -475,8 +488,10 @@ struct directory_config {
 
     /* Cache */
     int                  cache_trans;
+    int                  cache_trans_incremental;
     apr_size_t           cache_trans_min;
     apr_size_t           cache_trans_max;
+    apr_size_t           cache_trans_maxitems;
 
     /* Array to hold signatures of components, which will
      * appear in the ModSecurity signature in the audit log.

apache2/msc_geo.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "msc_geo.h"

apache2/msc_geo.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_GEO_H_

apache2/msc_logging.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "re.h"

apache2/msc_logging.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_LOGGING_H_

apache2/msc_lua.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #if defined(WITH_LUA)

apache2/msc_lua.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #if defined(WITH_LUA)

apache2/msc_multipart.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <ctype.h>

apache2/msc_multipart.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_MULTIPART_H_

apache2/msc_parsers.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "msc_parsers.h"
@@ -49,10 +57,18 @@ int parse_cookies_v0(modsec_rec *msr, char *_cookie_header, apr_table_t *cookies
         /* we ignore cookies with empty names */
         if ((attr_name != NULL)&&(strlen(attr_name) != 0)) {
             if (attr_value != NULL) {
-                msr_log(msr, 5, "Adding request cookie: name \"%s\", value \"%s\"", log_escape(msr->mp, attr_name), log_escape(msr->mp, attr_value));
+                if (msr->txcfg->debuglog_level >= 5) {
+                    msr_log(msr, 5, "Adding request cookie: name \"%s\", value \"%s\"",
+                        log_escape(msr->mp, attr_name), log_escape(msr->mp, attr_value));
+                }
+
                 apr_table_add(cookies, attr_name, attr_value);
             } else {
-                msr_log(msr, 5, "Adding request cookie: name \"%s\", value empty", log_escape(msr->mp, attr_name));
+                if (msr->txcfg->debuglog_level >= 5) {
+                    msr_log(msr, 5, "Adding request cookie: name \"%s\", value empty",
+                        log_escape(msr->mp, attr_name));
+                }
+
                 apr_table_add(cookies, attr_name, "");
             }
 
@@ -169,12 +185,18 @@ int parse_cookies_v1(modsec_rec *msr, char *_cookie_header, apr_table_t *cookies
             }
 
             if (attr_value != NULL) {
-                msr_log(msr, 5, "Adding request cookie: name \"%s\", value \"%s\"",
+                if (msr->txcfg->debuglog_level >= 5) {
-                    log_escape(msr->mp, attr_name), log_escape(msr->mp, attr_value));
+                    msr_log(msr, 5, "Adding request cookie: name \"%s\", value \"%s\"",
+                        log_escape(msr->mp, attr_name), log_escape(msr->mp, attr_value));
+                }
+
                 apr_table_add(cookies, attr_name, attr_value);
             } else {
-                msr_log(msr, 5, "Adding request cookie: name \"%s\", value empty",
+                if (msr->txcfg->debuglog_level >= 5) {
-                    log_escape(msr->mp, attr_name));
+                    msr_log(msr, 5, "Adding request cookie: name \"%s\", value empty",
+                        log_escape(msr->mp, attr_name));
+                }
+
                 apr_table_add(cookies, attr_name, "");
             }
 
@@ -301,9 +323,11 @@ int parse_arguments(modsec_rec *msr, const char *s, apr_size_t inputlength,
  *
  */
 void add_argument(modsec_rec *msr, apr_table_t *arguments, msc_arg *arg) {
-    msr_log(msr, 5, "Adding request argument (%s): name \"%s\", value \"%s\"",
+    if (msr->txcfg->debuglog_level >= 5) {
-        arg->origin, log_escape_ex(msr->mp, arg->name, arg->name_len),
+        msr_log(msr, 5, "Adding request argument (%s): name \"%s\", value \"%s\"",
-        log_escape_ex(msr->mp, arg->value, arg->value_len));
+            arg->origin, log_escape_ex(msr->mp, arg->name, arg->name_len),
+            log_escape_ex(msr->mp, arg->value, arg->value_len));
+    }
 
     apr_table_addn(arguments, log_escape_nq_ex(msr->mp, arg->name, arg->name_len), (void *)arg);
 }

apache2/msc_parsers.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_PARSERS_H_

apache2/msc_pcre.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "msc_pcre.h"

apache2/msc_pcre.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_PCRE_H_

apache2/msc_reqbody.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "modsecurity.h"

apache2/msc_test.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <apr.h>

apache2/msc_util.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "msc_util.h"

apache2/msc_util.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _UTIL_H_

apache2/msc_xml.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "msc_xml.h"

apache2/msc_xml.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_XML_H_

apache2/pdf_protect.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "modsecurity.h"

apache2/pdf_protect.h

@@ -2,13 +2,20 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
-
 #ifndef _PDF_PROTECT_H_
 #define _PDF_PROTECT_H_
 

apache2/persist_dbm.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "persist_dbm.h"

apache2/persist_dbm.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _PERSIST_DBM_H_

apache2/re.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <ctype.h>
@@ -1774,7 +1782,7 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
     apr_pool_t *mptmp = msr->msc_rule_mptmp;
     apr_table_t *tartab = NULL;
     apr_table_t *vartab = NULL;
-    int i, rc, match_count = 0;
+    int i, rc = 0, match_count = 0;
     int invocations = 0;
     int multi_match = 0;
 
@@ -1789,6 +1797,7 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
         multi_match = 1;
     }
 
+    /* ENH: What is a good initial size? */
     tartab = apr_table_make(mptmp, 24);
     if (tartab == NULL) return -1;
     vartab = apr_table_make(mptmp, 24);
@@ -1857,28 +1866,48 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
     for (i = 0; i < arr->nelts; i++) {
         int changed;
         int usecache = 0;
-        apr_table_t **carr = NULL;
         apr_table_t *cachetab = NULL;
         apr_time_t time_before_trans = 0;
+        msre_var *var;
 
         /* Take one target. */
-        msre_var *var = (msre_var *)te[i].val;
+        var = (msre_var *)te[i].val;
 
         /* Is this var cacheable? */
         if (msr->txcfg->cache_trans != MODSEC_CACHE_DISABLED) {
             usecache = 1;
 
+            /* Counting vars are not cacheable due to them being created
+             * in a local per-rule pool.
+             */
+            if (var->is_counting) {
+                if (msr->txcfg->debuglog_level >= 9) {
+                    msr_log(msr, 9, "CACHE: Disabled - &%s is dynamic", var->name);
+                }
+
+                usecache = 0;
+            }
+            /* Only cache if if the variable is available in this phase */
+            else if (msr->phase < var->metadata->availability) {
+                if (msr->txcfg->debuglog_level >= 9) {
+                    msr_log(msr, 9, "CACHE: Disabled - %s is not yet available in phase %d (requires phase %d or later)", var->name, msr->phase, var->metadata->availability);
+                }
+
+                usecache = 0;
+            }
             /* check the cache options */
-            if (var->value_len < msr->txcfg->cache_trans_min) {
+            else if (var->value_len < msr->txcfg->cache_trans_min) {
                 if (msr->txcfg->debuglog_level >= 9) {
                     msr_log(msr, 9, "CACHE: Disabled - %s value length=%u, smaller than minlen=%" APR_SIZE_T_FMT, var->name, var->value_len, msr->txcfg->cache_trans_min);
                 }
+
                 usecache = 0;
             }
-            if ((msr->txcfg->cache_trans_max != 0) && (var->value_len > msr->txcfg->cache_trans_max)) {
+            else if ((msr->txcfg->cache_trans_max != 0) && (var->value_len > msr->txcfg->cache_trans_max)) {
                 if (msr->txcfg->debuglog_level >= 9) {
                     msr_log(msr, 9, "CACHE: Disabled - %s value length=%u, larger than maxlen=%" APR_SIZE_T_FMT, var->name, var->value_len, msr->txcfg->cache_trans_max);
                 }
+
                 usecache = 0;
             }
 
@@ -1889,26 +1918,44 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                         msr_log(msr, 9, "CACHE: Enabled");
                     }
 
+                    #ifdef CACHE_DEBUG
+                    msr_log(msr, 9, "CACHE: Fetching cache entry from hash=%pp: %pp=%s", msr->tcache, var, var->name);
+                    #endif
+
                     /* Fetch cache table for this target */
-                    carr = (apr_table_t **)apr_hash_get(msr->tcache, var->name, APR_HASH_KEY_STRING);
+                    cachetab = (apr_table_t *)apr_hash_get(msr->tcache, var->value, sizeof(var->value));
-                    if (carr != NULL) {
-                        cachetab = carr[msr->phase];
-                    }
-                    else {
-                        /* Create an array of cache tables (one table per phase) */
-                        carr = (apr_table_t **)apr_pcalloc(msr->mp, (sizeof(apr_table_t *) * (PHASE_LAST + 1)));
-                        if (carr == NULL) return -1;
-                        memset(carr, 0, (sizeof(apr_table_t *) * (PHASE_LAST + 1)));
-                        apr_hash_set(msr->tcache, var->name, APR_HASH_KEY_STRING, carr);
-                    }
 
                     /* Create an empty cache table if this is the first time */
-                    if (cachetab == NULL) {
+                    #ifdef CACHE_DEBUG
-                        cachetab = carr[msr->phase] = apr_table_make(msr->mp, 5);
+                    if (cachetab) {
+                        msr_log(msr, 9, "CACHE: Using cache table %pp", cachetab);
                     }
+                    else
+                    #else
+                    if (cachetab == NULL)
+                    #endif
+                    {
+                        /* NOTE: We use the pointer to the var value as a hash
+                         *       key as it is unique. This pointer *must*
+                         *       remain valid through the entire phase. If
+                         *       it does not, then we will not receive a cache
+                         *       hit and just wasted RAM. So, it is important
+                         *       that any such vars be marked as VAR_DONT_CACHE.
+                         *
+                         * ENH: Only use pointer for non-scalar vars
+                         */
+                        cachetab = apr_table_make(msr->mp, 3);
+                        apr_hash_set(msr->tcache, var->value, sizeof(var->value), cachetab);
+
+                        #ifdef CACHE_DEBUG
+                        msr_log(msr, 9, "CACHE: Created a new cache table %pp for %pp", cachetab, var->value);
+                        #endif
+                    }
+
                 }
                 else {
                     usecache = 0;
+
                     if (msr->txcfg->debuglog_level >= 9) {
                         msr_log(msr, 9, "CACHE: %s transformations are not cacheable", var->name);
                     }
@@ -1936,7 +1983,10 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
             msre_action *action;
             msre_tfn_metadata *metadata;
             apr_table_t *normtab;
+            const char *lastvarval = NULL;
+            apr_size_t lastvarlen = 0;
 
+            changed = 0;
             normtab = apr_table_make(mptmp, 10);
             if (normtab == NULL) return -1;
             tarr = apr_table_elts(rule->actionset->actions);
@@ -1945,6 +1995,7 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
             /* Build the final list of transformation functions. */
             for (k = 0; k < tarr->nelts; k++) {
                 action = (msre_action *)telts[k].val;
+
                 if (strcmp(telts[k].key, "t") == 0) {
                     if (strcmp(action->param, "none") == 0) {
                         apr_table_clear(normtab);
@@ -1968,9 +2019,14 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                          * starting point
                          */
                         if (usecache) {
-                            tfnspath = apr_psprintf(msr->mp, "%s%s%s", (tfnspath?tfnspath:""), (tfnspath?",":""), action->param);
+                            tfnspath = apr_psprintf(mptmp, "%s%s%s", (tfnspath?tfnspath:""), (tfnspath?",":""), action->param);
-                            tfnskey = apr_psprintf(msr->mp, "%x;%s", tfnscount, tfnspath);
+                            tfnskey = apr_psprintf(mptmp, "%x;%s", tfnscount, tfnspath);
                             crec = (msre_cache_rec *)apr_table_get(cachetab, tfnskey);
+
+                            #ifdef CACHE_DEBUG
+                            msr_log(msr, 9, "CACHE: %s %s cached=%d", var->name, tfnskey, (crec ? 1 : 0));
+                            #endif
+
                             if (crec != NULL) {
                                 last_crec = crec;
                                 last_cached_tfn = tfnscount;
@@ -1983,15 +2039,19 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
             /* If the last cached tfn is the last in the list
              * then we can stop here and just execute the action immediatly
              */
-            if (usecache && !multi_match && (crec != NULL) && (crec == last_crec)) {
+            if (usecache && !multi_match &&
+                (crec != NULL) && (crec == last_crec))
+            {
                 crec->hits++;
+
                 if (crec->changed) {
-                    var->value = apr_pmemdup(msr->mp, crec->val, crec->val_len);
+                    var->value = apr_pmemdup(mptmp, crec->val, crec->val_len);
                     var->value_len = crec->val_len;
                 }
 
                 if (msr->txcfg->debuglog_level >= 9) {
-                    msr_log(msr, 9, "T (%d) %s: \"%s\" [cached hits=%d]", crec->changed, crec->path, log_escape_nq_ex(mptmp, var->value, var->value_len), crec->hits);
+                    msr_log(msr, 9, "T (%d) %s: \"%s\" [fully cached hits=%d]", crec->changed, crec->path,
+                        log_escape_nq_ex(mptmp, var->value, var->value_len), crec->hits);
                 }
 
                 #if !defined(PERFORMANCE_MEASUREMENT)
@@ -1999,9 +2059,11 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                 #endif
                 {
                     apr_time_t t1 = apr_time_now();
+
                     #if defined(PERFORMANCE_MEASUREMENT)
                     rule->trans_time += (t1 - time_before_trans);
                     #endif
+
                     msr_log(msr, 4, "Transformation completed in %" APR_TIME_T_FMT " usec.",
                         (t1 - time_before_trans));
                 }
@@ -2030,14 +2092,6 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
 
             tarr = apr_table_elts(normtab);
 
-            /* Make a copy of the variable value so that
-             * we can change it in-place.
-             */
-            if (tarr->nelts) {
-                var->value = apr_pstrmemdup(mptmp, var->value, var->value_len);
-                /* var->value_len remains the same */
-            }
-
             /* Execute transformations in a loop. */
 
             /* Start after the last known cached transformation if we can */
@@ -2046,32 +2100,39 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                 tfnspath = last_crec->path;
                 last_crec->hits++;
 
-                if ((changed = last_crec->changed) == 1) {
+                if ((changed = last_crec->changed) > 0) {
-                    var->value = apr_pmemdup(msr->mp, last_crec->val, last_crec->val_len);
+                    var->value = last_crec->val;
                     var->value_len = last_crec->val_len;
                 }
 
                 if (msr->txcfg->debuglog_level >= 9) {
-                    msr_log(msr, 9, "T (%d) %s: \"%s\" [partially cached hits=%d]", last_crec->changed, tfnspath, log_escape_nq_ex(mptmp, var->value, var->value_len), last_crec->hits);
+                    msr_log(msr, 9, "T (%d) %s: \"%s\" [partially cached hits=%d]", last_crec->changed,
+                        tfnspath, log_escape_nq_ex(mptmp, var->value, var->value_len), last_crec->hits);
                 }
             }
             else {
-                changed = 1;
                 tfnspath = NULL;
                 k = 0;
             }
 
+            /* Make a copy of the value so that we can change it in-place. */
+            if (tarr->nelts) {
+                var->value = apr_pstrmemdup(mptmp, var->value, var->value_len);
+                /* var->value_len remains the same */
+            }
+
             telts = (const apr_table_entry_t*)tarr->elts;
             for (; k < tarr->nelts; k++) {
                 char *rval = NULL;
                 long int rval_length = -1;
+                int tfnchanged = 0;
 
                 /* In multi-match mode we execute the operator
                  * once at the beginning and then once every
                  * time the variable is changed by the transformation
                  * function.
                  */
-                if (multi_match && changed) {
+                if (multi_match && (k == 0 || tfnchanged)) {
                     invocations++;
 
                     #if !defined(PERFORMANCE_MEASUREMENT)
@@ -2079,9 +2140,11 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                     #endif
                     {
                         apr_time_t t1 = apr_time_now();
+
                         #if defined(PERFORMANCE_MEASUREMENT)
                         rule->trans_time += (t1 - time_before_trans);
                         #endif
+
                         msr_log(msr, 4, "Transformation completed in %" APR_TIME_T_FMT " usec.",
                             (t1 - time_before_trans));
                     }
@@ -2108,60 +2171,88 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
                 /* Perform one transformation. */
                 action = (msre_action *)telts[k].val;
                 metadata = (msre_tfn_metadata *)action->param_data;
-
+                tfnchanged = metadata->execute(mptmp,
-                /* Try to use the cache */
+                    (unsigned char *)var->value, var->value_len,
-                if (usecache) {
-                    /* Generate the cache key */
-                    tfnspath = apr_psprintf(msr->mp, "%s%s%s", (tfnspath?tfnspath:""), (tfnspath?",":""), action->param);
-                    tfnskey = apr_psprintf(msr->mp, "%x;%s", (k + 1), tfnspath);
-
-                    /* Try to fetch this transformation from cache */
-                    #ifdef CACHE_DEBUG
-                    msr_log(msr, 9, "CACHE: Fetching %s %s ", var->name, tfnskey);
-                    #endif
-                    crec = (msre_cache_rec *)apr_table_get(cachetab, tfnskey);
-                    if (crec != NULL) {
-                        crec->hits++;
-
-                        if ((changed = crec->changed) == 1) {
-                            var->value = apr_pmemdup(msr->mp, crec->val, crec->val_len);
-                            var->value_len = crec->val_len;
-                        }
-
-                        if (msr->txcfg->debuglog_level >= 9) {
-                            msr_log(msr, 9, "T (%d) %s: \"%s\" [cached hits=%d]", crec->changed, metadata->name, log_escape_nq_ex(mptmp, var->value, var->value_len), crec->hits);
-                        }
-                        continue;
-                    }
-                }
-
-                rc = metadata->execute(mptmp, (unsigned char *)var->value, var->value_len,
                     &rval, &rval_length);
-                if (rc < 0) {
+
+                if (tfnchanged < 0) {
                     return -1;
                 }
 
-                changed = rc;
+                if (tfnchanged) {
+                    changed++;
+                }
 
+                /* Use the new values */
                 var->value = rval;
                 var->value_len = rval_length;
 
                 /* Cache the transformation */
                 if (usecache) {
-                    /* ENH1: Add flag to vars to tell which ones can change across phases store the rest in a global cache */
+                    int tfnsnum = k + 1;
-                    crec = (msre_cache_rec *)apr_pcalloc(msr->mp, sizeof(msre_cache_rec));
-                    if (crec == NULL) return -1;
 
-                    crec->hits = 0;
+                    /* Generate the cache key */
-                    crec->changed = changed;
+                    tfnspath = apr_psprintf(msr->mp, "%s%s%s", (tfnspath ? tfnspath : ""),
-                    crec->num = k + 1;
+                        (tfnspath ? "," : ""), action->param);
-                    crec->path = tfnspath;
+                    tfnskey = apr_psprintf(msr->mp, "%x;%s", tfnsnum, tfnspath);
-                    crec->val = changed ? apr_pmemdup(msr->mp, var->value, var->value_len) : NULL;
+
-                    crec->val_len = changed ? var->value_len : 0;
+                    if ((msr->txcfg->cache_trans_maxitems != 0) &&
-                    #ifdef CACHE_DEBUG
+                        (msr->tcache_items >= msr->txcfg->cache_trans_maxitems))
-                    msr_log(msr, 9, "CACHE: Caching %s=\"%.*s\"", tfnskey, var->value_len, log_escape_nq_ex(mptmp, var->value, var->value_len));
+                    {
-                    #endif
+                        /* Warn only once if we attempt to go over the cache limit. */
-                    apr_table_setn(cachetab, tfnskey, (void *)crec);
+                        if (msr->tcache_items == msr->txcfg->cache_trans_maxitems) {
+                            msr->tcache_items++;
+                            msr_log(msr, 4, "CACHE: Disabled - phase=%d"
+                                            " maxitems=%" APR_SIZE_T_FMT
+                                            " limit reached.",
+                                            msr->phase,
+                                            msr->txcfg->cache_trans_maxitems);
+                        }
+                    }
+                    else if (msr->txcfg->cache_trans_incremental ||
+                        (tfnsnum == tarr->nelts))
+                    {
+                        /* ENH1: Add flag to vars to tell which ones can change across phases store the rest in a global cache */
+                        crec = (msre_cache_rec *)apr_pcalloc(msr->mp, sizeof(msre_cache_rec));
+                        if (crec == NULL) return -1;
+
+                        crec->hits = 0;
+                        crec->changed = changed;
+                        crec->num = k + 1;
+                        crec->path = tfnspath;
+
+                        /* We want to cache a copy if it changed otherwise
+                         * we just want to use a pointer to the last changed value.
+                         */
+                        crec->val = (!lastvarval || tfnchanged) ? apr_pmemdup(msr->mp, var->value, var->value_len) : lastvarval;
+                        crec->val_len = changed ? ((!lastvarval || tfnchanged) ? var->value_len : lastvarlen) : 0;
+
+                        /* Keep track of the last changed var value */
+                        if (tfnchanged) {
+                            lastvarval = crec->val;
+                            lastvarlen = crec->val_len;
+                        }
+
+                        #ifdef CACHE_DEBUG
+                        if (changed) {
+                            msr_log(msr, 9, "CACHE: Caching %s=\"%s\" (%pp)",
+                                            tfnskey,
+                                            log_escape_nq_ex(mptmp,
+                                                             crec->val,
+                                                             crec->val_len),
+                                            var);
+                        }
+                        else {
+                            msr_log(msr, 9, "CACHE: Caching %s=<no change> (%pp)",
+                                            tfnskey,
+                                            var);
+                        }
+                        #endif
+
+                        msr->tcache_items++;
+
+                        apr_table_setn(cachetab, tfnskey, (void *)crec);
+                    }
                 }
 
                 if (msr->txcfg->debuglog_level >= 9) {
@@ -2183,9 +2274,11 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
             #endif
             {
                 apr_time_t t1 = apr_time_now();
+
                 #if defined(PERFORMANCE_MEASUREMENT)
                 rule->trans_time += (t1 - time_before_trans);
                 #endif
+
                 msr_log(msr, 4, "Transformation completed in %" APR_TIME_T_FMT " usec.",
                     (t1 - time_before_trans));
             }
@@ -2210,44 +2303,6 @@ static apr_status_t msre_rule_process_normal(msre_rule *rule, modsec_rec *msr) {
         }
     }
 
-    #ifdef CACHE_DEBUG
-    if (msr->txcfg->debuglog_level >= 9) {
-        apr_hash_index_t *hi;
-        void *dummy;
-        apr_table_t **tab;
-        const apr_array_header_t *ctarr;
-        const apr_table_entry_t *ctelts;
-        msre_cache_rec *rec;
-        int cn = 0;
-        int ti, ri;
-
-        for (hi = apr_hash_first(msr->mp, msr->tcache); hi; hi = apr_hash_next(hi)) {
-            apr_hash_this(hi, NULL, NULL, &dummy);
-            tab = (apr_table_t **)dummy;
-            if (tab == NULL) continue;
-
-            for (ti = PHASE_FIRST; ti <= PHASE_LAST; ti++) {
-                if (tab[ti] == NULL) continue;
-                ctarr = apr_table_elts(tab[ti]);
-                ctelts = (const apr_table_entry_t*)ctarr->elts;
-                for (ri = 0; ri < ctarr->nelts; ri++) {
-                    cn++;
-                    rec = (msre_cache_rec *)ctelts[ri].val;
-                    if (rec->changed) {
-                        if (msr->txcfg->debuglog_level >= 9) {
-                            msr_log(msr, 9, "CACHE: %5d) phase=%d hits=%d %x;%s=\"%s\"", cn, msr->phase, rec->hits, rec->num, rec->path, log_escape_nq_ex(mptmp, rec->val, rec->val_len));
-                        }
-                    }
-                    else {
-                        if (msr->txcfg->debuglog_level >= 9) {
-                            msr_log(msr, 9, "CACHE: %5d) phase=%d hits=%d %x;%s=<no change>", cn, msr->phase, rec->hits, rec->num, rec->path);
-                        }
-                    }
-                }
-            }
-        }
-    }
-    #endif
 
     return (match_count ? RULE_MATCH : RULE_NO_MATCH);
 }

apache2/re.h

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef _MSC_RE_H_

apache2/re_actions.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "re.h"

apache2/re_operators.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "re.h"

apache2/re_tfns.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include <ctype.h>

apache2/re_variables.c

@@ -2,10 +2,18 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #include "http_core.h"
@@ -450,7 +458,7 @@ static int var_request_uri_raw_generate(modsec_rec *msr, msre_var *var, msre_rul
 /* REQUEST_URI */
 
 static int var_request_uri_generate(modsec_rec *msr, msre_var *var, msre_rule *rule,
-    apr_table_t *vartab, apr_pool_t *mptmp)
+    apr_table_t *vartab, apr_pool_t *mptmp) /* dynamic */
 {
     char *value = NULL;
 
@@ -2134,7 +2142,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_args_combined_size_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2233,7 +2241,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_files_combined_size_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2255,7 +2263,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 1,
         var_generic_list_validate,
         var_files_sizes_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2277,7 +2285,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_geo_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2288,7 +2296,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_global_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2299,7 +2307,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_highest_severity_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2310,7 +2318,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_ip_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2321,7 +2329,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_matched_var_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2332,7 +2340,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_matched_var_name_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2354,7 +2362,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_boundary_quoted_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2365,7 +2373,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_boundary_whitespace_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2376,7 +2384,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_data_after_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2387,7 +2395,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_data_before_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2398,7 +2406,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_header_folding_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2409,7 +2417,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_crlf_line_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2420,7 +2428,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_crlf_lf_lines_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2431,7 +2439,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_lf_line_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2442,7 +2450,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_missing_semicolon_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2453,7 +2461,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_strict_error_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2464,7 +2472,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_multipart_unmatched_boundary_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* flag */
         PHASE_REQUEST_BODY
     );
 
@@ -2519,7 +2527,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_remote_port_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2541,7 +2549,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_resource_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2552,7 +2560,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_reqbody_processor_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2563,7 +2571,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_reqbody_processor_error_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_BODY
     );
 
@@ -2574,7 +2582,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_reqbody_processor_error_msg_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_BODY
     );
 
@@ -2585,7 +2593,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_request_basename_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2695,7 +2703,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_request_uri_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2728,7 +2736,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_response_content_length,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_RESPONSE_HEADERS
     );
 
@@ -2783,7 +2791,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_response_status_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_RESPONSE_HEADERS
     );
 
@@ -2794,7 +2802,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         NULL,
         var_rule_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_RESPONSE_HEADERS
     );
 
@@ -2805,7 +2813,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_gid_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2816,7 +2824,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_basename_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2838,7 +2846,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_groupname_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2849,7 +2857,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_mode_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2860,7 +2868,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_uid_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2871,7 +2879,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_script_username_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_BODY
     );
 
@@ -2904,7 +2912,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_server_port_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* temp copy */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2915,7 +2923,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_session_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2926,7 +2934,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_sessionid_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_RESPONSE_HEADERS
     );
 
@@ -2948,7 +2956,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_user_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2959,7 +2967,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_userid_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_RESPONSE_HEADERS
     );
 
@@ -2970,7 +2978,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2981,7 +2989,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_day_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -2992,7 +3000,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_epoch_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3003,7 +3011,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_hour_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3014,7 +3022,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_min_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3025,7 +3033,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_mon_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3036,7 +3044,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_sec_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3047,7 +3055,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_wday_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3058,7 +3066,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_time_year_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3069,7 +3077,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         1, 1,
         var_generic_list_validate,
         var_tx_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3091,7 +3099,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 0,
         NULL,
         var_webserver_error_log_generate,
-        VAR_DONT_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_HEADERS
     );
 
@@ -3102,7 +3110,7 @@ void msre_engine_register_default_variables(msre_engine *engine) {
         0, 1,
         var_xml_validate,
         var_xml_generate,
-        VAR_CACHE,
+        VAR_DONT_CACHE, /* dynamic */
         PHASE_REQUEST_BODY
     );
 }

apache2/t/regression/action/10-detectiononly-actions.t

@@ -107,7 +107,7 @@
 	match_log => {
 		error => [ qr/ModSecurity: Warning. Unconditional match in SecAction.*ALLOWED/, 1 ],
 		-error => [ qr/Access allowed/, 1 ],
-# TODO: Allow should probably rule stop execution
+# TODO: Allow should probably stop rule execution
 #		-error => [ qr/DENIED/, 1 ],
 	},
 	match_response => {
@@ -131,7 +131,7 @@
 	match_log => {
 		error => [ qr/ModSecurity: Warning. Unconditional match in SecAction.*ALLOWED/, 1 ],
 		-error => [ qr/Access allowed/, 1 ],
-# TODO: Allow should probably rule stop execution
+# TODO: Allow should probably stop rule execution
 #		-error => [ qr/DENIED/, 1 ],
 	},
 	match_response => {
@@ -155,7 +155,7 @@
 	match_log => {
 		error => [ qr/ModSecurity: Warning. Unconditional match in SecAction.*ALLOWED/, 1 ],
 		-error => [ qr/Access allowed/, 1 ],
-# TODO: Allow should probably rule stop execution
+# TODO: Allow should probably stop rule execution
 #		-error => [ qr/DENIED/, 1 ],
 	},
 	match_response => {
@@ -179,7 +179,7 @@
 	match_log => {
 		error => [ qr/ModSecurity: Warning. Unconditional match in SecAction.*ALLOWED/, 1 ],
 		-error => [ qr/Access allowed/, 1 ],
-# TODO: Allow should probably rule stop execution
+# TODO: Allow should probably stop rule execution
 #		-error => [ qr/DENIED/, 1 ],
 	},
 	match_response => {

apache2/t/regression/misc/10-tfn-cache.t

@@ -0,0 +1,189 @@
+### Transformation Caching
+
+{
+	type => "misc",
+	comment => "tfncache (simple fully cached)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0"
+
+		# This should cache it
+		SecRule ARGS_GET "WillNotMatch" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+
+		# This should use the cached value
+		SecRule ARGS_GET:test "foobar" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,deny"
+	),
+	match_log => {
+		debug => [ qr/removeWhiteSpace,lowercase: "foobar" .*cached/, 1 ],
+		-debug => [ qr/partially cached/, 1 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		GET => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html?test=Foo+Bar",
+	),
+},
+{
+	type => "misc",
+	comment => "tfncache (simple partially cached)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0,incremental:off,maxitems:0"
+
+		# This should cache it
+		SecRule ARGS_GET "WillNotMatch" "phase:1,t:none,t:removeWhiteSpace,pass,nolog"
+
+		# This should use the partially cached value
+		SecRule ARGS_GET:test "foobar" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,deny"
+	),
+	match_log => {
+		debug => [ qr/removeWhiteSpace: "FooBar" .*partially cached/, 1 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		GET => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html?test=Foo+Bar",
+	),
+},
+{
+	type => "misc",
+	comment => "tfncache (separate phases)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0"
+
+		# This should cache it
+		SecRule ARGS_GET "WillNotMatch" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+
+		# This should use the cached value
+		SecRule ARGS_GET:test "foobar" "phase:2,t:none,t:removeWhiteSpace,t:lowercase,deny"
+	),
+	match_log => {
+		-debug => [ qr/removeWhiteSpace,lowercase: "foobar" .*cached/, 1 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		GET => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html?test=Foo+Bar",
+	),
+},
+{
+	type => "misc",
+	comment => "tfncache (non-modifying tfns cached)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0"
+
+		# This should cache it
+		SecRule ARGS_GET "WillNotMatch" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+
+		# This should use the cached value
+		SecRule ARGS_GET:test "foobar" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,deny"
+	),
+	match_log => {
+		debug => [ qr/removeWhiteSpace,lowercase: "foobar" .*cached/, 1 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		GET => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html?test=foo+bar",
+	),
+},
+{
+	type => "misc",
+	comment => "tfncache (unique keys)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+		SecRequestBodyAccess On
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0"
+
+		# This should cache it
+		SecRule ARGS "WillNotMatch" "phase:2,t:none,t:removeWhiteSpace,t:lowercase,pass"
+
+		# This should see cached versions of *both* ARGS_GET
+		SecRule ARGS:test "queryval" "phase:2,t:none,t:removeWhiteSpace,t:lowercase,deny,chain"
+		SecRule ARGS:test "firstval" "t:none,t:removeWhiteSpace,t:lowercase,chain"
+		SecRule ARGS:test "secondval" "t:none,t:removeWhiteSpace,t:lowercase"
+	),
+	match_log => {
+		debug => [ qr/removeWhiteSpace,lowercase: "queryval" .*removeWhiteSpace,lowercase: "firstval" .*cached.*removeWhiteSpace,lowercase: "secondval" .*cached/s, 1 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		POST => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html?test=Query+Val",
+		[
+			"Content-Type" => "application/x-www-form-urlencoded",
+		],
+		#  Args
+		"test=First+Val&test=Second+Val",
+	),
+},
+{
+	type => "misc",
+	comment => "tfncache (large cache)",
+	conf => qq(
+		SecRuleEngine On
+		SecDebugLog $ENV{DEBUG_LOG}
+		SecDebugLogLevel 9
+		SecRequestBodyAccess On
+
+		<IfDefine MODSEC_2.5>
+			SecRequestBodyNoFilesLimit 1048576
+		</IfDefine>
+		SecRequestBodyInMemoryLimit 131072
+		SecResponseBodyLimit 1048576
+
+		# We need to make this work no matter what the defaults may change to
+		SecCacheTransformations On "minlen:1,maxlen:0,maxitems:0"
+
+		# This should cache it in all phases
+		SecRule ARGS "WillNotMatch" "phase:1,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+		SecRule ARGS "WillNotMatch" "phase:2,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+		SecRule ARGS "WillNotMatch" "phase:3,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+		SecRule ARGS "WillNotMatch" "phase:4,t:none,t:removeWhiteSpace,t:lowercase,pass,nolog"
+
+		# This should use the cached value
+		SecRule ARGS "foobar" "phase:4,t:none,t:removeWhiteSpace,t:lowercase,deny"
+	),
+	match_log => {
+		debug => [ qr/Adding request argument \(BODY\): name "test", value "Foo Bar"/, 60, "Waiting for httpd to process request: "],
+		-error => [ qr/segmentation fault/i, 60 ],
+	},
+	match_response => {
+		status => qr/^403$/,
+	},
+	request => new HTTP::Request(
+		POST => "http://$ENV{SERVER_NAME}:$ENV{SERVER_PORT}/index.html",
+		[
+			"Content-Type" => "application/x-www-form-urlencoded",
+		],
+		# 1000 Args
+		join("&", map { sprintf "arg%08d=0123456789abcdef+0123456789ABCDEF+0123456789abcdef", $_ } (1 .. 1000))."&test=Foo+Bar",
+	),
+},

apache2/t/run-regression-tests.pl.in

@@ -35,6 +35,7 @@ my $PID_FILE = "$FILES_DIR/httpd.pid";
 my $HTTPD = q(@APXS_HTTPD@);
 my $PASSED = 0;
 my $TOTAL = 0;
+my $BUFSIZ = 32768;
 my %C = ();
 my %FILE = ();
 my $UA_NAME = "ModSecurity Regression Tests/1.2.3";
@@ -49,9 +50,9 @@ if ($HTTPD eq "\@APXS_HTTPD\@") {
 $SIG{TERM} = $SIG{INT} = \&handle_interrupt;
 
 my %opt;
-getopts('A:E:D:C:T:H:a:p:dh', \%opt);
+getopts('A:E:D:C:T:H:a:p:dvh', \%opt);
 
-if ($opt{D}) {
+if ($opt{d}) {
 	$Data::Dumper::Indent = 1;
 	$Data::Dumper::Terse = 1;
 	$Data::Dumper::Pad = "";
@@ -72,7 +73,8 @@ Usage: $SCRIPT [options] [file [N]]
   -S path   Specify Apache httpd server root path.
   -a file   Specify Apache httpd binary (default: httpd)
   -p port   Specify Apache httpd port (default: 8088)
-  -d        Enable debugging.
+  -v        Enable verbose output (details on failure).
+  -d        Enable debugging output.
   -h        This help.
 
 EOT
@@ -98,6 +100,7 @@ $opt{E} = "$FILES_DIR/error.log" unless (defined $opt{E});
 $opt{C} = "$CONF_DIR/httpd.conf" unless (defined $opt{C});
 $opt{H} = "$SROOT_DIR/htdocs" unless (defined $opt{H});
 $opt{p} = 8088 unless (defined $opt{p});
+$opt{v} = 1 if ($opt{d});
 
 unless (defined $opt{S}) {
 	my $httpd_root = `$HTTPD -V`;
@@ -194,7 +197,7 @@ sub runfile {
 		if (exists $t{conf} and defined $t{conf}) {
 			$conf_fn = sprintf "%s/%s_%s_%06d.conf",
 			                   $CONF_DIR, $t{type}, $cfg, $n;
-#			dbg("Writing test config to: $conf_fn");
+			#dbg("Writing test config to: $conf_fn");
 			open(CONF, ">$conf_fn") or die "Failed to open conf \"$conf_fn\": $!\n";
 			print CONF (ref $t{conf} eq "CODE" ? eval { &{$t{conf}} } : $t{conf});
 			msg("$@") if ($@);
@@ -207,9 +210,9 @@ sub runfile {
 
 		# Run any prerun setup
 		if ($rc == 0 and exists $t{prerun} and defined $t{prerun}) {
-			dbg("Executing perl prerun...");
+			vrb("Executing perl prerun...");
 			$rc = &{$t{prerun}};
-			dbg("Perl prerun returned: $rc");
+			vrb("Perl prerun returned: $rc");
 		}
 
 		if ($httpd_up) {
@@ -218,7 +221,7 @@ sub runfile {
 				my $resp = do_request($t{request});
 				if (!$resp) {
 					msg("invalid response");
-					dbg("RESPONSE: ", $resp);
+					vrb("RESPONSE: ", $resp);
 					$rc = 1;
 				}
 				else {
@@ -229,14 +232,13 @@ sub runfile {
 						if ($neg and defined $match) {
 							$rc = 1;
 							msg("response $mtype matched: $m");
-							dbg($resp);
+							vrb($resp);
-
 							last;
 						}
 						elsif (!$neg and !defined $match) {
 							$rc = 1;
 							msg("response $mtype failed to match: $m");
-							dbg($resp);
+							vrb($resp);
 							last;
 						}
 					}
@@ -245,13 +247,13 @@ sub runfile {
 
 			# Run any arbitrary perl tests
 			if ($rc == 0 and exists $t{test} and defined $t{test}) {
-				#dbg("Executing perl test(s)...");
+				dbg("Executing perl test(s)...");
 				$rc = eval { &{$t{test}} };
 				if (! defined $rc) {
 					msg("Error running test: $@");
 					$rc = -1;
 				}
-				#dbg("Perl tests returned: $rc");
+				dbg("Perl tests returned: $rc");
 			}
 
 			# Search for all log matches
@@ -263,15 +265,11 @@ sub runfile {
 					if ($neg and defined $match) {
 						$rc = 1;
 						msg("$mtype log matched: $m->[0]");
-						msg("Log: $FILE{$mtype}{fn}");
-						dbg(escape("$FILE{$mtype}{buf}"));
 						last;
 					}
 					elsif (!$neg and !defined $match) {
 						$rc = 1;
 						msg("$mtype log failed to match: $m->[0]");
-						msg("Log: $FILE{$mtype}{fn}");
-						dbg(escape("$FILE{$mtype}{buf}"));
 						last;
 					}
 				}
@@ -287,13 +285,11 @@ sub runfile {
 					if ($neg and defined $match) {
 						$rc = 1;
 						msg("$fn file matched: $m");
-						dbg(escape("$FILE{$fn}{buf}"));
 						last;
 					}
 					elsif (!$neg and !defined $match) {
 						$rc = 1;
 						msg("$fn file failed match: $m");
-						dbg(escape("$FILE{$fn}{buf}"));
 						last;
 					}
 				}
@@ -308,7 +304,11 @@ sub runfile {
 			$pass++;
 		}
 		else {
-			dbg("Test config: $conf_fn");
+			vrb("Test Config: $conf_fn");
+			vrb("Debug Log: $FILE{debug}{fn}");
+			dbg(escape("$FILE{debug}{buf}"));
+			vrb("Error Log: $FILE{error}{fn}");
+			dbg(escape("$FILE{error}{buf}"));
 		}
 
 		msg(sprintf("%s) %s%s: %s%s", $id, $t{type}, (exists($t{comment}) ? " - $t{comment}" : ""), ($rc ? "failed" : "passed"), ((defined($out) && $out ne "")? " ($out)" : "")));
@@ -375,15 +375,11 @@ sub do_request {
 	}
 
 	if (ref $r eq "HTTP::Request") {
-#		dbg("REQUEST: ", $r);
 		my $resp = $UA->request($r);
-		if ($opt{d}) {
+		dbg($resp->request()->as_string()) if ($opt{d});
-			dbg($resp->request()->as_string());
-		}
 		return $resp
 	}
 	else {
-#		dbg("REQUEST:\n", $r);
 		return do_raw_request($r);
 	}
 
@@ -409,13 +405,17 @@ sub match_response {
 	return;
 }
 
+sub read_log {
+	my($name, $timeout, $graph) = @_;
+	return match_log($name, undef, $timeout, $graph);
+}
+
 sub match_log {
-	my($name, $re, $timeout) = @_;
+	my($name, $re, $timeout, $graph) = @_;
 	my $t0 = gettimeofday;
 	my($fh,$rbuf) = ($FILE{$name}{fd}, \$FILE{$name}{buf});
 	my $n = length($$rbuf);
-
+	my $rc = undef;
-	msg("Warning: Empty regular expression.") if (!defined $re or $re eq "");
 
   unless (defined $fh) {
     msg("Error: File \"$name\" is not opened for matching.");
@@ -424,15 +424,45 @@ sub match_log {
 
 	$timeout = 0 unless (defined $timeout);
 
-	do {
+	my $i = 0;
-		$n += $fh->sysread($$rbuf, 1024, $n);
+	my $graphed = 0;
-#		dbg("Match \"$re\" in $name \"$$rbuf\" ($n)");
+	READ: {
-		return $& if ($$rbuf =~ m/$re/m);
+		do {
-		# TODO: Use select()/poll()
+			my $nbytes = $fh->sysread($$rbuf, $BUFSIZ, $n);
-		sleep 0.1;
+			if (!defined($nbytes)) {
-	} while (gettimeofday - $t0 < $timeout);
+				msg("Error: Could not read \"$name\" log: $!");
+				last;
+			}
+			elsif (!defined($re) and $nbytes == 0) {
+				last;
+			}
 
-	return;
+			# Remove APR pool debugging
+			$$rbuf =~ s/POOL DEBUG:[^\n]+PALLOC[^\n]+\n//sg;
+
+			$n = length($$rbuf);
+
+			#dbg("Match \"$re\" in $name \"$$rbuf\" ($n)");
+			if ($$rbuf =~ m/$re/m) {
+				$rc = $&;
+				last;
+			}
+			# TODO: Use select()/poll()
+			sleep 0.1 unless ($nbytes == $BUFSIZ);
+			if ($graph and $opt{d}) {
+				$i++;
+				if ($i == 10) {
+					$graphed++;
+					$i=0;
+					print STDERR $graph if ($graphed == 1);
+					print STDERR "."
+				}
+			}
+		} while (gettimeofday - $t0 < $timeout);
+	}
+	print STDERR "\n" if ($graphed);
+
+	return $rc;
 }
 
 sub match_file {
@@ -477,6 +507,11 @@ sub dbg {
 	print STDOUT "$out\n";
 }
 
+sub vrb {
+	return unless(@_ and $opt{v});
+	msg(@_);
+}
+
 sub msg {
 	return unless(@_);
 	my $out = join "", map {
@@ -524,14 +559,14 @@ sub httpd_start {
 
 	my $httpd_out;
 	my $httpd_pid = open3(undef, $httpd_out, undef, @p) or quit(1);
-	my $out = join("\\n", split(/\n/, <$httpd_out>));
+	my $out = join("\\n", grep(!/POOL DEBUG/, (<$httpd_out>)));
 	close $httpd_out;
 	waitpid($httpd_pid, 0);
 
 	my $rc = $?;
 	if ( WIFEXITED($rc) ) {
 		$rc = WEXITSTATUS($rc);
-		dbg("Httpd start returned with $rc.") if ($rc);
+		vrb("Httpd start returned with $rc.") if ($rc);
 	}
 	elsif( WIFSIGNALED($rc) ) {
 		msg("Httpd start failed with signal " . WTERMSIG($rc) . ".");
@@ -543,15 +578,15 @@ sub httpd_start {
 	}
 
 	if (defined $out and $out ne "") {
-		dbg(join(" ", map { quote_shell($_) } @p));
+		vrb(join(" ", map { quote_shell($_) } @p));
 		msg("Httpd start failed with error messages:\n$out");
 		return -1
 	}
 
 	# Look for startup msg
-	unless (defined match_log("error", qr/resuming normal operations/, 10)) {
+	unless (defined match_log("error", qr/resuming normal operations/, 60, "Waiting on httpd to start: ")) {
-		dbg(join(" ", map { quote_shell($_) } @p));
+		vrb(join(" ", map { quote_shell($_) } @p));
-		dbg(match_log("error", qr/(^.*ModSecurity: .*)/sm, 10));
+		vrb(match_log("error", qr/(^.*ModSecurity: .*)/sm, 10));
 		msg("Httpd server failed to start.");
 		return -1;
 	}
@@ -571,7 +606,7 @@ sub httpd_stop {
 
 	my $httpd_out;
 	my $httpd_pid = open3(undef, $httpd_out, undef, @p) or quit(1);
-	my $out = join("\\n", split(/\n/, <$httpd_out>));
+	my $out = join("\\n", grep(!/POOL DEBUG/, (<$httpd_out>)));
 	close $httpd_out;
 	waitpid($httpd_pid, 0);
 
@@ -583,7 +618,7 @@ sub httpd_stop {
 	my $rc = $?;
 	if ( WIFEXITED($rc) ) {
 		$rc = WEXITSTATUS($rc);
-		dbg("Httpd stop returned with $rc.") if ($rc);
+		vrb("Httpd stop returned with $rc.") if ($rc);
 	}
 	elsif( WIFSIGNALED($rc) ) {
 		msg("Httpd stop failed with signal " . WTERMSIG($rc) . ".");
@@ -595,8 +630,8 @@ sub httpd_stop {
 	}
 
 	# Look for startup msg
-	unless (defined match_log("error", qr/caught SIG[A-Z]+, shutting down/, 10)) {
+	unless (defined match_log("error", qr/caught SIG[A-Z]+, shutting down/, 60, "Waiting on httpd to stop: ")) {
-		dbg(join(" ", map { quote_shell($_) } @p));
+		vrb(join(" ", map { quote_shell($_) } @p));
 		msg("Httpd server failed to shutdown.");
 		return -1;
 	}
@@ -617,7 +652,7 @@ sub httpd_reload {
 
 	my $httpd_out;
 	my $httpd_pid = open3(undef, $httpd_out, undef, @p) or quit(1);
-	my $out = join("\\n", split(/\n/, <$httpd_out>));
+	my $out = join("\\n", grep(!/POOL DEBUG/, (<$httpd_out>)));
 	close $httpd_out;
 	waitpid($httpd_pid, 0);
 
@@ -629,7 +664,7 @@ sub httpd_reload {
 	my $rc = $?;
 	if ( WIFEXITED($rc) ) {
 		$rc = WEXITSTATUS($rc);
-		dbg("Httpd reload returned with $rc.") if ($rc);
+		vrb("Httpd reload returned with $rc.") if ($rc);
 	}
 	elsif( WIFSIGNALED($rc) ) {
 		msg("Httpd reload failed with signal " . WTERMSIG($rc) . ".");
@@ -641,8 +676,8 @@ sub httpd_reload {
 	}
 
 	# Look for startup msg
-	unless (defined match_log("error", qr/resuming normal operations/, 10)) {
+	unless (defined match_log("error", qr/resuming normal operations/, 60, "Waiting on httpd to restart: ")) {
-		dbg(join(" ", map { quote_shell($_) } @p));
+		vrb(join(" ", map { quote_shell($_) } @p));
 		msg("Httpd server failed to reload.");
 		return -1;
 	}

apache2/utf8tables.h

@@ -2,17 +2,25 @@
  * ModSecurity for Apache 2.x, http://www.modsecurity.org/
  * Copyright (c) 2004-2008 Breach Security, Inc. (http://www.breach.com/)
  *
- * You should have received a copy of the licence along with this
+ * This product is released under the terms of the General Public Licence,
- * program (stored in the file "LICENSE"). If the file is missing,
+ * version 2 (GPLv2). Please refer to the file LICENSE (included with this
- * or if you have any other questions related to the licence, please
+ * distribution) which contains the complete text of the licence.
- * write to Breach Security, Inc. at support@breach.com.
+ *
+ * There are special exceptions to the terms and conditions of the GPL
+ * as it is applied to this software. View the full text of the exception in
+ * file MODSECURITY_LICENSING_EXCEPTION in the directory of this software
+ * distribution.
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Breach Security, Inc.
+ * directly using the email address support@breach.com.
  *
  */
 #ifndef UTF8TABLES_H_
 #define UTF8TABLES_H_
 
 /**
- * This include file is used by acmp.c only, it's not included anywhere else
+ * This include file is used by acmp.c only; it's not included anywhere else.
  */
 
 typedef long acmp_utf8_char_t;

doc/modsecurity2-apache-reference.xml

@@ -4,7 +4,7 @@
   Manual</title>
 
   <articleinfo>
-    <releaseinfo>Version 2.6.0-trunk (July 16, 2008)</releaseinfo>
+    <releaseinfo>Version 2.6.0-trunk (July 31, 2008)</releaseinfo>
 
     <copyright>
       <year>2004-2008</year>
@@ -182,15 +182,15 @@
       Security.</para>
 
       <note>
-        <para>ModSecurity, mod_security, and ModSecurity Pro are trademarks or
+        <para>ModSecurity, mod_security, ModSecurity Pro, and ModSecurity Core
-        registered trademarks of Breach Security, Inc.</para>
+        Rules are trademarks or registered trademarks of Breach Security,
+        Inc.</para>
       </note>
     </section>
   </section>
 
   <section>
-    <title><trademark class="registered">ModSecurity</trademark> Core
+    <title><trademark>ModSecurity Core Rules</trademark></title>
-    Rules</title>
 
     <section>
       <title>Overview</title>
@@ -858,10 +858,12 @@ SecAuditLogStorageDir logs/audit
     </section>
 
     <section>
-      <title><literal>SecCacheTransformations</literal></title>
+      <title><literal>SecCacheTransformations</literal>
+      (Deprecated/Experimental)</title>
 
       <para><emphasis>Description:</emphasis> Controls caching of
-      transformations.</para>
+      transformations. Caching is off by default starting with 2.5.6, when it
+      was deprecated and downgraded back to experimental.</para>
 
       <para><emphasis>Syntax:</emphasis> <literal
       moreinfo="none">SecCacheTransformations On|Off
@@ -898,16 +900,30 @@ SecAuditLogStorageDir logs/audit
       <para>The following options are allowed (comma separated):</para>
 
       <itemizedlist>
+        <listitem>
+          <para><literal moreinfo="none">incremental:on|off</literal> -
+          enabling this option will cache every transformation instead of just
+          the final transformation. (default: off)</para>
+        </listitem>
+
+        <listitem>
+          <para><literal moreinfo="none">maxitems:N</literal> - do not allow
+          more than N transformations to be cached. The cache will then be
+          disabled. A zero value is interpreted as "unlimited". This option
+          may be useful to limit caching for a form with a large number of
+          ARGS. (default: 512)</para>
+        </listitem>
+
         <listitem>
           <para><literal moreinfo="none">minlen:N</literal> - do not cache the
           transformation if the value's length is less than N bytes. (default:
-          15)</para>
+          32)</para>
         </listitem>
 
         <listitem>
           <para><literal moreinfo="none">maxlen:N</literal> - do not cache the
           transformation if the value's length is more than N bytes. A zero
-          value is interpreted as "unlimited". (default: 0)</para>
+          value is interpreted as "unlimited". (default: 1024)</para>
         </listitem>
       </itemizedlist>
     </section>
@@ -1284,7 +1300,8 @@ SecAuditLogStorageDir logs/audit
 
       <para><emphasis>Dependencies/Notes:</emphasis> None</para>
 
-      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" "chain,<emphasis>skipAfter:99</emphasis>"
+      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" \
+    "chain,t:none,t:urlDecode,t:lowercase,t:normalisePath,<emphasis>skipAfter:99</emphasis>"
 SecRule REMOTE_ADDR "^127\.0\.0\.1$" "chain"
 SecRule REQUEST_HEADERS:User-Agent \
     "^Apache \(internal dummy connection\)$" "t:none"  
@@ -1452,10 +1469,11 @@ SecRule &amp;REQUEST_HEADERS:Accept "@eq 0" \
       <para><emphasis>Version:</emphasis> 2.0.0</para>
 
       <para><emphasis>Dependencies/Notes:</emphasis> This directive is
-      required if you plan to inspect POST_PAYLOADS of requests. This
+      required if you plan to inspect <literal>POST_PAYLOAD</literal>. This
       directive must be used along with the "phase:2" processing phase action
-      and REQUEST_BODY variable/location. If any of these 3 parts are not
+      and <literal>REQUEST_BODY</literal> variable/location. If any of these 3
-      configured, you will not be able to inspect the request bodies.</para>
+      parts are not configured, you will not be able to inspect the request
+      bodies.</para>
 
       <para>Possible values are:</para>
 
@@ -1724,7 +1742,10 @@ SecResponseBodyLimit 524288</programlisting>
       VARIABLES OPERATOR [ACTIONS]</literal></para>
 
       <para><emphasis>Example Usage:</emphasis> <literal
-      moreinfo="none">SecRule REQUEST_URI "attack"</literal></para>
+      moreinfo="none">SecRule REQUEST_URI "attack" \</literal></para>
+
+      <para><literal>
+      "phase:1,t:none,t:urlDecode,t:lowercase,t:normalisePath"</literal></para>
 
       <para><emphasis>Processing Phase:</emphasis> Any</para>
 
@@ -1752,11 +1773,11 @@ SecResponseBodyLimit 524288</programlisting>
         following rule will reject a transaction that has the word<emphasis>
         dirty</emphasis> in the URI:</para>
 
-        <programlisting format="linespecific">SecRule REQUEST_URI dirty</programlisting>
+        <programlisting format="linespecific">SecRule ARGS dirty</programlisting>
 
         <para>Each rule can specify one or more variables:</para>
 
-        <programlisting format="linespecific">SecRule REQUEST_URI|QUERY_STRING dirty</programlisting>
+        <programlisting format="linespecific">SecRule ARGS|REQUEST_HEADERS:User-Agent dirty</programlisting>
 
         <para>There is a third format supported by the selection operator -
         XPath expression. XPath expressions can only used against the special
@@ -1783,7 +1804,7 @@ SecResponseBodyLimit 524288</programlisting>
         moreinfo="none">@</literal> as the first character in the second rule
         parameter:</para>
 
-        <programlisting format="linespecific">SecRule REQUEST_URI "@rx dirty"</programlisting>
+        <programlisting format="linespecific">SecRule ARGS "@rx dirty"</programlisting>
 
         <para>Note how we had to use double quotes to delimit the second rule
         parameter. This is because the second parameter now has a whitespace
@@ -2019,7 +2040,7 @@ function main()
 
     -- Retrieve one variable, applying one transformation function.
     -- The second parameter is a string.
-    local var2 = m.getvar("REQUEST_URI", "normalisePath");
+    local var2 = m.getvar("ARGS", "lowercase");
 
     -- Retrieve one variable, applying several transformation functions.
     -- The second parameter is now a list. You should note that m.getvar()
@@ -2537,7 +2558,8 @@ SecRule REQUEST_HEADERS:Host "!^$" "deny,<emphasis>phase:1</emphasis>"</programl
       prevent buffer overflow issues). Example: Block request if the size of
       the arguments is above 25 characters.</para>
 
-      <programlisting format="linespecific">SecRule REQUEST_FILENAME "^/cgi-bin/login\.php$" "chain,log,deny,phase:2"
+      <programlisting format="linespecific">SecRule REQUEST_FILENAME "^/cgi-bin/login\.php" \
+    "chain,log,deny,phase:2,t:none,t:lowercase,t:normalisePath"
 SecRule <emphasis>ARGS_COMBINED_SIZE</emphasis> "@gt 25"</programlisting>
     </section>
 
@@ -2551,8 +2573,9 @@ SecRule <emphasis>ARGS_COMBINED_SIZE</emphasis> "@gt 25"</programlisting>
       allow 2 argument names - p and a. If any other argument names are
       injected, it will be blocked.</para>
 
-      <programlisting format="linespecific">SecRule REQUEST_FILENAME "/index.php" "chain,log,deny,status:403,phase:2"
+      <programlisting format="linespecific">SecRule REQUEST_FILENAME "/index.php" \
-SecRule<emphasis> ARGS_NAMES</emphasis> "!^(p|a)$"</programlisting>
+    "chain,log,deny,status:403,phase:2,t:none,t:lowercase,t:normalisePath"
+SecRule<emphasis> ARGS_NAMES</emphasis> "!^(p|a)$" "t:none,t:lowercase"</programlisting>
     </section>
 
     <section>
@@ -2605,9 +2628,9 @@ SecRule<emphasis> ARGS_NAMES</emphasis> "!^(p|a)$"</programlisting>
     <section>
       <title><literal moreinfo="none">ENV</literal></title>
 
-      <para>Collection, requires a single parameter (after a colon character).
+      <para>Collection, requires a single parameter (after colon). The
-      The ENV variable is set with setenv and does not give access to the CGI
+      <literal>ENV</literal> variable is set with setenv and does not give
-      environment variables. Example:</para>
+      access to the CGI environment variables. Example:</para>
 
       <programlisting format="linespecific">SecRule REQUEST_FILENAME "printenv" pass,<emphasis>setenv:tag=suspicious</emphasis>
 SecRule <emphasis>ENV:tag</emphasis> "suspicious"</programlisting>
@@ -2988,10 +3011,17 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
       <title><literal moreinfo="none">REQUEST_BASENAME</literal></title>
 
       <para>This variable holds just the filename part of
-      <literal>REQUEST_FILENAME</literal> (e.g. index.php). Warning: not URL
+      <literal>REQUEST_FILENAME</literal> (e.g. index.php).</para>
-      decoded. Example:</para>
 
-      <programlisting format="linespecific">SecRule <emphasis>REQUEST_BASENAME</emphasis> "^login\.php$"</programlisting>
+      <para>Example:</para>
+
+      <programlisting format="linespecific">SecRule <emphasis>REQUEST_BASENAME</emphasis> "^login\.php$" phase:2,t:none,t:lowercase</programlisting>
+
+      <note>
+        <para>Please note that anti-evasion transformations are not applied to
+        this variable by default. <literal>REQUEST_BASENAME</literal> will
+        recognise both / and \ as path separators.</para>
+      </note>
     </section>
 
     <section>
@@ -3007,7 +3037,7 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
       <para><emphasis>Note</emphasis></para>
 
       <para>This variable is only available if the content type is
-      application/x-www-form-urlencoded.</para>
+      <literal>application/x-www-form-urlencoded</literal>.</para>
     </section>
 
     <section>
@@ -3034,25 +3064,32 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
     <section>
       <title><literal moreinfo="none">REQUEST_FILENAME</literal></title>
 
-      <para>This variable holds the relative REQUEST_URI minus the
+      <para>This variable holds the relative <literal>REQUEST_URI</literal>
-      QUERY_STRING part (e.g. /index.php). Example:</para>
+      minus the <literal>QUERY_STRING</literal> part (e.g. /index.php).
+      Example:</para>
 
-      <programlisting format="linespecific">SecRule <emphasis>REQUEST_FILENAME</emphasis> "^/cgi-bin/login\.php$"</programlisting>
+      <programlisting format="linespecific">SecRule <emphasis>REQUEST_FILENAME</emphasis> "^/cgi-bin/login\.php$" phase:2,t:none,t:normalisePath</programlisting>
+
+      <note>
+        <para>Please note that anti-evasion transformations are not used on
+        <literal>REQUEST_FILENAME</literal> by default.</para>
+      </note>
     </section>
 
     <section>
       <title><literal moreinfo="none">REQUEST_HEADERS</literal></title>
 
       <para>This variable can be used as either a collection of all of the
-      Request Headers or can be used to specify individual headers (by using
+      request headers or can be used to specify individual headers (by using
       REQUEST_HEADERS<emphasis>:Header-Name</emphasis>). Example: the first
-      example uses REQUEST_HEADERS as a collection and is applying the
+      example uses <literal>REQUEST_HEADERS</literal> as a collection and is
-      validateUrlEncoding operator against all headers.</para>
+      applying the <literal>validateUrlEncoding</literal> operator against all
+      headers.</para>
 
       <programlisting format="linespecific">SecRule <emphasis>REQUEST_HEADERS</emphasis> "@validateUrlEncoding"</programlisting>
 
-      <para>Example: the second example is targeting only the Host
+      <para>Example: the second example is targeting only the
-      header.</para>
+      <literal>Host</literal> header.</para>
 
       <programlisting format="linespecific">SecRule <emphasis>REQUEST_HEADERS:Host</emphasis> "^[\d\.]+$" \
     "deny,log,status:400,msg:'Host header is a numeric IP address'"</programlisting>
@@ -3061,8 +3098,8 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
     <section>
       <title><literal moreinfo="none">REQUEST_HEADERS_NAMES</literal></title>
 
-      <para>This variable is a collection of the names of all of the Request
+      <para>This variable is a collection of the names of all of the request
-      Headers. Example:</para>
+      headers. Example:</para>
 
       <programlisting format="linespecific">SecRule <emphasis>REQUEST_HEADERS_NAMES</emphasis> "^x-forwarded-for" \
     "log,deny,status:403,t:lowercase,msg:'Proxy Server Used'"</programlisting>
@@ -3083,9 +3120,10 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
     <section>
       <title><literal moreinfo="none">REQUEST_METHOD</literal></title>
 
-      <para>This variable holds the Request Method used by the client.
+      <para>This variable holds the request method used by the client.</para>
-      Example: the following example will trigger if the Request Method is
+
-      either CONNECT or TRACE.</para>
+      <para>The following example will trigger if the request method is either
+      <literal>CONNECT</literal> or TRACE.</para>
 
       <programlisting format="linespecific">SecRule <emphasis>REQUEST_METHOD</emphasis> "^((?:connect|trace))$" t:none,t:lowercase</programlisting>
     </section>
@@ -3093,7 +3131,7 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
     <section>
       <title><literal moreinfo="none">REQUEST_PROTOCOL</literal></title>
 
-      <para>This variable holds the Request Protocol Version information.
+      <para>This variable holds the request protocol version information.
       Example:</para>
 
       <programlisting format="linespecific">SecRule <emphasis>REQUEST_PROTOCOL</emphasis> "!^http/(0\.9|1\.0|1\.1)$" t:none,t:lowercase</programlisting>
@@ -3102,31 +3140,45 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
     <section>
       <title><literal moreinfo="none">REQUEST_URI</literal></title>
 
-      <para>This variable holds the full URL including the QUERY_STRING data
+      <para>This variable holds the full URL including the
-      (e.g. /index.php?p=X), however it will never contain a domain name, even
+      <literal>QUERY_STRING</literal> data (e.g. /index.php?p=X), however it
-      if it was provided on the request line. Warning: not URL decoded. It
+      will never contain a domain name, even if it was provided on the request
-      also does not include either the REQUEST_METHOD or the HTTP version
+      line. It also does not include either the
-      info. Example:</para>
+      <literal>REQUEST_METHOD</literal> or the HTTP version info.</para>
 
-      <programlisting format="linespecific">SecRule <emphasis>REQUEST_URI</emphasis> "attack"</programlisting>
+      <para>Example:</para>
+
+      <programlisting format="linespecific">SecRule <emphasis>REQUEST_URI</emphasis> "attack" phase:1,t:none,t:urlDecode,t:lowercase,t:normalisePath</programlisting>
+
+      <note>
+        <para>Please note that anti-evasion transformations are not used on
+        <literal>REQUEST_URI</literal> by default.</para>
+      </note>
     </section>
 
     <section>
       <title><literal moreinfo="none">REQUEST_URI_RAW</literal></title>
 
-      <para>Same as REQUEST_URI but will contain the domain name if it was
+      <para>Same as <literal>REQUEST_URI</literal> but will contain the domain
-      provided on the request line (e.g.
+      name if it was provided on the request line (e.g.
-      http://www.example.com/index.php?p=X). Warning: not URL decoded.
+      http://www.example.com/index.php?p=X).</para>
-      Example:</para>
 
-      <programlisting format="linespecific">SecRule<emphasis> REQUEST_URI_RAW</emphasis> "http:/"</programlisting>
+      <para>Example:</para>
+
+      <programlisting format="linespecific">SecRule<emphasis> REQUEST_URI_RAW</emphasis> "http:/" phase:1,t:none,t:urlDecode,t:lowercase,t:normalisePath</programlisting>
+
+      <note>
+        <para>Please note that anti-evasion transformations are not used on
+        <literal>REQUEST_URI_RAW</literal> by default.</para>
+      </note>
     </section>
 
     <section>
       <title><literal moreinfo="none">RESPONSE_BODY</literal></title>
 
-      <para>This variable holds the data for the response payload.
+      <para>This variable holds the data for the response payload.</para>
-      Example:</para>
+
+      <para>Example:</para>
 
       <programlisting format="linespecific">SecRule<emphasis> RESPONSE_BODY</emphasis> "ODBC Error Code"</programlisting>
     </section>
@@ -3361,7 +3413,8 @@ SecRule XML "@validateDTD /opt/apache-frontend/conf/xml.dtd"</programlisting>
 
       <programlisting format="linespecific">SecRule REQUEST_COOKIES:PHPSESSID !^$ chain,nolog,pass
 SecAction setsid:%{REQUEST_COOKIES.PHPSESSID}
-SecRule REQUEST_URI "^/cgi-bin/finger$" "pass,log,setvar:<emphasis>session.score</emphasis>=+10"
+SecRule REQUEST_URI "^/cgi-bin/finger$" \
+    "phase:2,t:none,t:lowercase,t:normalisePath,pass,log,setvar:<emphasis>session.score</emphasis>=+10"
 SecRule<emphasis> SESSION:SCORE</emphasis> "@gt 50" "pass,log,setvar:<emphasis>session.blocked</emphasis>=1"
 SecRule<emphasis> SESSION:BLOCKED</emphasis> "@eq 1" "log,deny,status:403"</programlisting>
     </section>
@@ -4113,9 +4166,11 @@ SecRule TX:1 "(?:(?:a(dmin|nonymous)))"</programlisting>
       <para>Example:</para>
 
       <programlisting format="linespecific"># Refuse to accept POST requests that do
-# not specify request body length 
+# not specify request body length. Do note that
-SecRule REQUEST_METHOD ^POST$<emphasis> chain</emphasis>
+# this rule should be preceeded by a rule that verifies
-SecRule REQUEST_HEADER:Content-Length ^$</programlisting>
+# only valid request methods (e.g. GET, HEAD and POST) are used.
+SecRule REQUEST_METHOD ^POST$<emphasis> chain</emphasis>,t:none
+SecRule REQUEST_HEADER:Content-Length ^$ t:none</programlisting>
 
       <para><emphasis>Note</emphasis></para>
 
@@ -4294,7 +4349,7 @@ SecRule IP:AUTH_ATTEMPT "@gt 25" \
       <programlisting format="linespecific"># The following is going to execute /usr/local/apache/bin/test.sh
 # as a shell script on rule match.
 SecRule REQUEST_URI "^/cgi-bin/script\.pl" \
-    "log,<emphasis>exec:/usr/local/apache/bin/test.sh</emphasis>"
+    "phase:2,t:none,t:lowercase,t:normalisePath,log,<emphasis>exec:/usr/local/apache/bin/test.sh</emphasis>"
 
 # The following is going to process /usr/local/apache/conf/exec.lua
 # internally as a Lua script on rule match.
@@ -4328,7 +4383,8 @@ SecRule ARGS:p attack log,<emphasis>exec:/usr/local/apache/conf/exec.lua</emphas
       <programlisting format="linespecific">SecRule REQUEST_COOKIES:JSESSIONID "!^$" nolog,phase:1,pass,chain
 SecAction setsid:%{REQUEST_COOKIES:JSESSIONID}
 SecRule REQUEST_URI "^/cgi-bin/script\.pl" \
-    "log,allow,setvar:session.suspicious=1,<emphasis>expirevar:session.suspicious=3600</emphasis>,phase:1"</programlisting>
+    "phase:2,t:none,t:lowercase,t:normalisePath,log,allow,\
+setvar:session.suspicious=1,<emphasis>expirevar:session.suspicious=3600</emphasis>,phase:1"</programlisting>
 
       <para><emphasis>Note</emphasis></para>
 
@@ -4946,7 +5002,8 @@ SecAction <emphasis>setsid:%{REQUEST_COOKIES.PHPSESSID}</emphasis></programlisti
 
       <para>Example:</para>
 
-      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" "chain,<emphasis>skip:2</emphasis>"
+      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" \
+"phase:2,chain,t:none<emphasis>,skip:2</emphasis>"
 SecRule REMOTE_ADDR "^127\.0\.0\.1$" "chain"
 SecRule REQUEST_HEADERS:User-Agent "^Apache \(internal dummy connection\)$" "t:none"  
 SecRule &amp;REQUEST_HEADERS:Host "@eq 0" \
@@ -4975,7 +5032,7 @@ SecRule &amp;REQUEST_HEADERS:Accept "@eq 0" \
 
       <para>Example:</para>
 
-      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" "chain,<emphasis>skipAfter:960015</emphasis>"
+      <para><programlisting format="linespecific">SecRule REQUEST_URI "^/$" "chain,t:none,<emphasis>skipAfter:960015</emphasis>"
 SecRule REMOTE_ADDR "^127\.0\.0\.1$" "chain"
 SecRule REQUEST_HEADERS:User-Agent "^Apache \(internal dummy connection\)$" "t:none"  
 SecRule &amp;REQUEST_HEADERS:Host "@eq 0" \
@@ -5050,7 +5107,8 @@ SecRule REQUEST_COOKIES:SESSIONID "47414e81cbbef3cf8366e84eeacba091" \
       <para>Example:</para>
 
       <programlisting format="linespecific">SecRule REQUEST_FILENAME "\b(?:n(?:map|et|c)|w(?:guest|sh)|cmd(?:32)?|telnet|rcmd|ftp)\.exe\b" \
-    "deny,msg:'System Command Access',id:'950002',<emphasis>tag:'WEB_ATTACK/FILE_INJECTION',tag:'OWASP/A2'</emphasis>,severity:'2'"</programlisting>
+    "t:none,t:lowercase,deny,msg:'System Command Access',id:'950002',<emphasis>\
+tag:'WEB_ATTACK/FILE_INJECTION',tag:'OWASP/A2'</emphasis>,severity:'2'"</programlisting>
 
       <para><emphasis>Note</emphasis></para>
 
@@ -6123,4 +6181,4 @@ Server: Apache/2.x.x
       </section>
     </section>
   </section>
-</article>
+</article>