Merge branch 'v2/master' of https://github.com/marcstern/ModSecurity into v2/master

CHANGES

@@ -1,6 +1,8 @@
 DD mmm YYYY - 2.9.x (to be released)
 -------------------
 
+ * Set the minimum security protocol version for SecRemoteRules
+   [Issue security/code-scanning/2 - @airween]
  * Allow lua version 5.4
    [Issue #2996 - @3eka, @martinhsv]
  * Configure: do not check for pcre1 if pcre2 requested

README.md

@@ -7,12 +7,12 @@ Copyright (c) 2004-2013 Trustwave Holdings, Inc. (http://www.trustwave.com/)
 
 You may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
 
-If any of the files related to licensing are missing or if you have any other questions related to licensing please contact Trustwave Holdings, Inc. directly using the email address: security@modsecurity.org.
+If any of the files related to licensing are missing or if you have any other questions related to licensing please contact Trustwave Holdings, Inc. directly using the email address: modsecurity@owasp.org.
 
 
 ## Documentation
 
-Please refer to: [the documentation folder](https://github.com/SpiderLabs/ModSecurity/tree/v2/master/doc) for the reference manual.
+Please refer to: [the documentation folder](https://github.com/owasp-modsecurity/ModSecurity/tree/v2/master/doc) for the reference manual.
 
 ## Sponsor Note
 

apache2/msc_json.c

@@ -354,15 +354,17 @@ int json_init(modsec_rec *msr, char **error_msg) {
 int json_process_chunk(modsec_rec *msr, const char *buf, unsigned int size, char **error_msg) {
     if (error_msg == NULL) return -1;
     *error_msg = NULL;
-    base_offset=buf;
+    // Take a copy in case libyajl decodes the buffer inline
+    base_offset = apr_pstrmemdup(msr->mp, buf, size);
+    if (!base_offset) return -1;
 
     /* Feed our parser and catch any errors */
-    msr->json->status = yajl_parse(msr->json->handle, buf, size);
+    msr->json->status = yajl_parse(msr->json->handle, (unsigned char*)base_offset, size);
     if (msr->json->status != yajl_status_ok) {
 	if (msr->json->depth_limit_exceeded) {
            *error_msg = "JSON depth limit exceeded";
 	} else {
-           char *yajl_err = yajl_get_error(msr->json->handle, 0, buf, size);
+           char *yajl_err = yajl_get_error(msr->json->handle, 0, base_offset, size);
            *error_msg = apr_pstrdup(msr->mp, yajl_err);
            yajl_free_error(msr->json->handle, yajl_err);
 	}

apache2/msc_logging.c

@@ -234,15 +234,7 @@ static char *construct_auditlog_filename(apr_pool_t *mp, const char *uniqueid) {
      * This is required for mpm-itk & mod_ruid2, though should be harmless for other implementations 
      * It also changes the return statement.
      */
-    char *userinfo;
+    char *userinfo = get_username(mp);
-    apr_status_t rc;
-    apr_uid_t uid;
-    apr_gid_t gid;
-    apr_uid_current(&uid, &gid, mp);
-    rc = apr_uid_name_get(&userinfo, uid, mp);
-    if (rc != APR_SUCCESS) {
-      userinfo = apr_psprintf(mp, "%u", uid);
-    }
 
     apr_time_exp_lt(&t, apr_time_now());
 

apache2/msc_pcre.c

@@ -31,11 +31,7 @@ static apr_status_t msc_pcre_cleanup(msc_regex_t *regex) {
         }
 #else
         if (regex->pe != NULL) {
-#if defined(VERSION_NGINX)
             pcre_free(regex->pe);
-#else
-            free(regex->pe);
-#endif
             regex->pe = NULL;
         }
         if (regex->re != NULL) {
@@ -152,19 +148,15 @@ void *msc_pregcomp_ex(apr_pool_t *pool, const char *pattern, int options,
 
     #ifdef WITH_PCRE_STUDY
         #ifdef WITH_PCRE_JIT
-                pe = pcre_study(regex->re, PCRE_STUDY_JIT_COMPILE, &errptr);
+                pe = pcre_study(regex->re, PCRE_STUDY_EXTRA_NEEDED|PCRE_STUDY_JIT_COMPILE, &errptr);
         #else
-                pe = pcre_study(regex->re, 0, &errptr);
+                pe = pcre_study(regex->re, PCRE_STUDY_EXTRA_NEEDED, &errptr);
         #endif
     #endif
 
     /* Setup the pcre_extra record if pcre_study did not already do it */
     if (pe == NULL) {
-#if defined(VERSION_NGINX)
+        pe = (pcre_extra*)pcre_malloc(sizeof(pcre_extra));
-        pe = pcre_malloc(sizeof(pcre_extra));
-#else
-        pe = malloc(sizeof(pcre_extra));
-#endif
         if (pe == NULL) {
             return NULL;
         }

apache2/msc_remote_rules.c

@@ -331,8 +331,8 @@ int msc_remote_download_content(apr_pool_t *mp, const char *uri, const char *key
             headers_chunk = curl_slist_append(headers_chunk, header_key);
         }
 
-        /* Make it TLS 1.x only. */
+        /* Make it TLS 1.2 at least. */
-        curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
+        curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
 
 #ifdef WIN32
         res_len = SearchPathA(NULL, "curl-ca-bundle.crt", NULL, (2048 + 1), buf, &ptr);

apache2/msc_util.c

@@ -2843,3 +2843,14 @@ char* strtok_r(
 }
 #endif
 
+// Function compatible with Linux & Windows, also with mpm-itk & mod_ruid2
+char* get_username(apr_pool_t* mp) {
+  	char* username;
+  	apr_uid_t uid;
+  	apr_gid_t gid;
+  	int rc = apr_uid_current(&uid, &gid, mp);
+  	if (rc != APR_SUCCESS) return "apache";
+   rc = apr_uid_name_get(&username, uid, mp);
+   if (rc != APR_SUCCESS) return "apache";
+   return username;
+}

apache2/msc_util.h

@@ -159,6 +159,8 @@ int DSOLOCAL tree_contains_ip(apr_pool_t *mp, TreeRoot *rtree,
 int DSOLOCAL ip_tree_from_param(apr_pool_t *pool,
     char *param, TreeRoot **rtree, char **error_msg);
 
+char DSOLOCAL *get_username(apr_pool_t* mp);
+
 #ifdef WITH_CURL
 int ip_tree_from_uri(TreeRoot **rtree, char *uri,
     apr_pool_t *mp, char **error_msg);

apache2/persist_dbm.c

@@ -100,18 +100,7 @@ static apr_table_t *collection_retrieve_ex(apr_sdbm_t *existing_dbm, modsec_rec
     apr_table_entry_t *te;
     int expired = 0;
     int i;
-
+    char *userinfo = get_username(msr->mp);
-    /**
-     * This is required for mpm-itk & mod_ruid2, though should be harmless for other implementations 
-     */
-    char *userinfo;
-    apr_uid_t uid;
-    apr_gid_t gid;
-    apr_uid_current(&uid, &gid, msr->mp);
-    rc = apr_uid_name_get(&userinfo, uid, msr->mp);
-    if (rc != APR_SUCCESS) {
-      userinfo = apr_psprintf(msr->mp, "%u", uid);
-    }
 
     if (msr->txcfg->data_dir == NULL) {
         msr_log(msr, 1, "collection_retrieve_ex: Unable to retrieve collection (name \"%s\", key \"%s\"). Use "
@@ -384,18 +373,7 @@ int collection_store(modsec_rec *msr, apr_table_t *col) {
     int i;
     const apr_table_t *stored_col = NULL;
     const apr_table_t *orig_col = NULL;
-
+    char *userinfo = get_username(msr->mp);
-    /**
-     * This is required for mpm-itk & mod_ruid2, though should be harmless for other implementations 
-     */
-    char *userinfo;
-    apr_uid_t uid;
-    apr_gid_t gid;
-    apr_uid_current(&uid, &gid, msr->mp);
-    rc = apr_uid_name_get(&userinfo, uid, msr->mp);
-    if (rc != APR_SUCCESS) {
-      userinfo = apr_psprintf(msr->mp, "%u", uid);
-    }
 
     var_name = (msc_string *)apr_table_get(col, "__name");
     if (var_name == NULL) {
@@ -677,18 +655,7 @@ int collections_remove_stale(modsec_rec *msr, const char *col_name) {
     char **keys;
     apr_time_t now = apr_time_sec(msr->request_time);
     int i;
-
+    char *userinfo = get_username(msr->mp);
-    /**
-     * This is required for mpm-itk & mod_ruid2, though should be harmless for other implementations 
-     */
-    char *userinfo;
-    apr_uid_t uid;
-    apr_gid_t gid;
-    apr_uid_current(&uid, &gid, msr->mp);
-    rc = apr_uid_name_get(&userinfo, uid, msr->mp);
-    if (rc != APR_SUCCESS) {
-      userinfo = apr_psprintf(msr->mp, "%u", uid);
-    }
 
     if (msr->txcfg->data_dir == NULL) {
         /* The user has been warned about this problem enough times already by now.

apache2/re.c

@@ -76,7 +76,7 @@ static int fetch_target_exception(msre_rule *rule, modsec_rec *msr, msre_var *va
     if(rule->actionset == NULL)
         return 0;
 
-    if(rule->actionset->id !=NULL)    {
+    {
 
         myvar = apr_pstrdup(msr->mp, var->name);
 
@@ -353,11 +353,11 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
                     rc = msre_parse_targets(ruleset, p, rule->targets, &my_error_msg);
                     if (rc < 0) {
                         if(msr) {
-                            msr_log(msr, 9, "Error parsing rule targets to replace variable");
+                            msr_log(msr, 9, "Error parsing rule targets to replace variable: %s", my_error_msg);
                         }
 #if !defined(MSC_TEST)
                         else {
-                            ap_log_error(APLOG_MARK, APLOG_ERR, 0, NULL, " ModSecurity: Error parsing rule targets to replace variable");
+                            ap_log_error(APLOG_MARK, APLOG_ERR, 0, NULL, " ModSecurity: Error parsing rule targets to replace variable: %s", my_error_msg);
                         }
 #endif
                         goto end;
@@ -378,7 +378,7 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
                     }
 #if !defined(MSC_TEST)
                     else {
-                        ap_log_error(APLOG_MARK, APLOG_ERR, 0, NULL, " ModSecurity: Cannot find varibale to replace");
+                        ap_log_error(APLOG_MARK, APLOG_ERR, 0, NULL, " ModSecurity: Cannot find variable to replace");
                     }
 #endif
                     goto end;
@@ -386,8 +386,13 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
             } else {
 
                 target = strdup(p);
-                if(target == NULL)
+                if(target == NULL) {
+                   if(target_list != NULL)
+                       free(target_list);
+                   if(replace != NULL)
+                       free(replace);
                    return NULL;
+                  }
 
                 is_negated = is_counting = 0;
                 param = name = value = NULL;
@@ -421,6 +426,8 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
                         free(target_list);
                     if(replace != NULL)
                         free(replace);
+                    if(target != NULL)
+                        free(target);
                     if(msr) {
                         msr_log(msr, 9, "Error to update target - [%s] is not valid target", name);
                     }
@@ -499,7 +506,7 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
         if(var_appended == 1)  {
             current_targets = msre_generate_target_string(ruleset->mp, rule);
             rule->unparsed = msre_rule_generate_unparsed(ruleset->mp, rule, current_targets, NULL, NULL);
-            rule->p1 = apr_pstrdup(ruleset->mp, current_targets);
+            rule->p1 = current_targets;
             if(msr) {
                 msr_log(msr, 9, "Successfully appended variable");
             }
@@ -512,18 +519,12 @@ char *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *r
     }
 
 end:
-    if(target_list != NULL) {
+    if(target_list != NULL)
         free(target_list);
-        target_list = NULL;
+    if(replace != NULL)
-    }
-    if(replace != NULL) {
         free(replace);
-        replace = NULL;
+    if(target != NULL)
-    }
-    if(target != NULL)  {
         free(target);
-        target = NULL;
-    }
     return NULL;
 }
 
@@ -637,7 +638,10 @@ static char *msre_generate_target_string(apr_pool_t *pool, msre_rule *rule)  {
 /**
  * Generate an action string from an actionset.
  */
-static char *msre_actionset_generate_action_string(apr_pool_t *pool, const msre_actionset *actionset)  {
+#ifndef DEBUG_CONF
+ static 
+#endif
+char *msre_actionset_generate_action_string(apr_pool_t *pool, const msre_actionset *actionset)  {
     const apr_array_header_t *tarr = NULL;
     const apr_table_entry_t *telts = NULL;
     char *actions = NULL;
@@ -1055,6 +1059,12 @@ int msre_parse_generic(apr_pool_t *mp, const char *text, apr_table_t *vartable,
         while(isspace(*p)) p++;
         if (*p == '\0') return count;
  
+        /* ignore empty action */
+        if (*p == ',') {
+          	p++;
+           continue;
+        }
+
         /* we are at the beginning of the name */
         name = p;
         while((*p != '\0')&&(*p != '|')&&(*p != ':')&&(*p != ',')&&(!isspace(*p))) p++; /* ENH replace with isvarnamechar() */

apache2/re.h

@@ -75,6 +75,10 @@ int DSOLOCAL rule_id_in_range(int ruleid, const char *range);
 msre_var DSOLOCAL *generate_single_var(modsec_rec *msr, msre_var *var, apr_array_header_t *tfn_arr,
     msre_rule *rule, apr_pool_t *mptmp);
 
+#ifdef DEBUG_CONF
+ char DSOLOCAL* msre_actionset_generate_action_string(apr_pool_t* pool, const msre_actionset* actionset);
+#endif
+
 #if defined(WITH_LUA)
 apr_table_t DSOLOCAL *generate_multi_var(modsec_rec *msr, msre_var *var, apr_array_header_t *tfn_arr,
     msre_rule *rule, apr_pool_t *mptmp);

apache2/re_actions.c

@@ -183,9 +183,9 @@ int expand_macros(modsec_rec *msr, msc_string *var, msre_rule *rule, apr_pool_t
      *      no macros in the input data.
      */
 
-    data = apr_pstrdup(mptmp, var->value); /* IMP1 Are we modifying data anywhere? */
+    data = var->value;
     arr = apr_array_make(mptmp, 16, sizeof(msc_string *));
-    if ((data == NULL)||(arr == NULL)) return -1;
+    if (arr == NULL) return -1;
 
     text_start = next_text_start = data;
     do {

apache2/re_operators.c

@@ -630,18 +630,13 @@ nextround:
     }
 
     if(msr->stream_input_data != NULL && input_body == 1) {
-        memset(msr->stream_input_data, 0x0, msr->stream_input_length);
         free(msr->stream_input_data);
         msr->stream_input_data = NULL;
         msr->stream_input_length = 0;
 #ifdef MSC_LARGE_STREAM_INPUT
         msr->stream_input_allocated_length  = 0;
-
-        msr->stream_input_data = (char *)malloc(size);
-#else
-        msr->stream_input_data = (char *)malloc(size+1);
 #endif
-
+        msr->stream_input_data = (char *)malloc(size+1);
         if(msr->stream_input_data == NULL)  {
             return -1;
         }
@@ -649,16 +644,11 @@ nextround:
         msr->stream_input_length = size;
 #ifdef MSC_LARGE_STREAM_INPUT
         msr->stream_input_allocated_length = size;
-        memset(msr->stream_input_data, 0x0, size);
-#else
-        memset(msr->stream_input_data, 0x0, size+1);
 #endif
         msr->if_stream_changed = 1;
 
         memcpy(msr->stream_input_data, data, size);
-#ifndef MSC_LARGE_STREAM_INPUT
         msr->stream_input_data[size] = '\0';
-#endif
 
         var->value_len = size;
         var->value = msr->stream_input_data;
@@ -688,7 +678,7 @@ static int msre_op_validateHash_param_init(msre_rule *rule, char **error_msg) {
     const char *pattern = rule->op_param;
     #ifdef WITH_PCRE_STUDY
         #ifdef WITH_PCRE_JIT
-    int rc, jit;
+    int rc, jit = 0;
         #endif
     #endif
 
@@ -761,10 +751,9 @@ static int msre_op_validateHash_execute(modsec_rec *msr, msre_rule *rule, msre_v
     char *my_error_msg = NULL;
     int ovector[33];
     int rc;
-    const char *pattern = NULL;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int jit;
+    int jit = 0;
        #endif
     #endif
 
@@ -791,8 +780,8 @@ static int msre_op_validateHash_execute(modsec_rec *msr, msre_rule *rule, msre_v
 
             expand_macros(msr, re_pattern, rule, msr->mp);
 
-            pattern = log_escape_re(msr->mp, re_pattern->value);
             if (msr->txcfg->debuglog_level >= 6) {
+                const char *pattern = log_escape_re(msr->mp, re_pattern->value);
                 msr_log(msr, 6, "Escaping pattern [%s]",pattern);
             }
 
@@ -958,7 +947,7 @@ static int msre_op_rx_param_init(msre_rule *rule, char **error_msg) {
     const char *pattern = rule->op_param;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int rc, jit;
+    int rc, jit = 0;
        #endif
     #endif
 
@@ -1028,7 +1017,7 @@ static int msre_op_rx_execute(modsec_rec *msr, msre_rule *rule, msre_var *var, c
     msc_parm *mparm = NULL;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int jit;
+    int jit = 0;
        #endif
     #endif
 
@@ -1545,10 +1534,10 @@ static const char *gsb_replace_tpath(apr_pool_t *pool, const char *domain, int l
     url = apr_palloc(pool, len + 1);
     data = apr_palloc(pool, len + 1);
 
-    memset(data, 0, len+1);
+    data[0] = '\0';
-    memset(url, 0, len+1);
     
     memcpy(url, domain, len);
+    url[len] = 0;
 
     while(( pos = strstr(url , "/./" )) != NULL) {
         match = 1;
@@ -2841,7 +2830,7 @@ static int msre_op_verifyCC_execute(modsec_rec *msr, msre_rule *rule, msre_var *
     msc_parm *mparm = NULL;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int jit;
+    int jit = 0;
        #endif
     #endif
 
@@ -3171,7 +3160,7 @@ static int msre_op_verifyCPF_execute(modsec_rec *msr, msre_rule *rule, msre_var
     msc_parm *mparm = NULL;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int jit;
+    int jit = 0;
        #endif
     #endif
 
@@ -3485,7 +3474,7 @@ static int msre_op_verifySSN_execute(modsec_rec *msr, msre_rule *rule, msre_var
     msc_parm *mparm = NULL;
     #ifdef WITH_PCRE_STUDY
        #ifdef WITH_PCRE_JIT
-    int jit;
+    int jit = 0;
        #endif
     #endif
 

doc/README.txt

@@ -8,4 +8,4 @@ Please access the ModSecurity Github space to access the below documentation.
     * Reference Manual
     * RoadMap
 
-https://github.com/SpiderLabs/ModSecurity/wiki/
+https://github.com/owasp-modsecurity/ModSecurity/wiki/

iis/dependencies/howto.txt

@@ -38,7 +38,7 @@ ssdeep-2.13
 
 --------------------------------------
 
-1. Create working directory (e.g. c:\work) and drop the latest clone from ModSecurity's 2.x Github (https://github.com/SpiderLabs/ModSecurity/archive/v2/master.zip)
+1. Create working directory (e.g. c:\work) and drop the latest clone from ModSecurity's 2.x Github (https://github.com/owasp-modsecurity/ModSecurity/archive/v2/master.zip)
 2. Make sure the prerequisites mentioned above are all set
 3. If you haven't download the dependency files before, uncomment the "@call download_files.bat" line on build_dependencies.bat to have them downloaded prior
 4. Open a command prompt (cmd.exe) and head to the "iis" folder inside ModSecurity tree working directory (e.g. cd c:\work\ModSecurity\iis)

iis/installer.wxs

@@ -617,7 +617,7 @@
                 <Control Id="Description" Type="Text" X="25" Y="23" Width="280" Height="15" Transparent="yes" NoPrefix="yes" Text="Choose to configure ModSecurity on IIS or not." />
                 <Control Id="Text" Type="Text" X="25" Y="55" Width="320" Height="50" Text="ModSecurityIIS needs to be configured under IIS as a module. It is recommended to perform this configuration during the installation. However, if you are facing problems in the installation, the recomendation is to disable this step. This will facilitate the debugging process since the files will be installed in place. Note that some scripts will be installed along with ModSecurity common files that can be later used to help this configuration/debugging process." />
                 <Control Type="CheckBox" Id="ConfigureIIS" Width="200" Height="14" X="25" Y="124" CheckBoxValue="1" Property="IIS_SETUP" Text="Perform ModSecurityIIS configuration." />
-                <Control Type="Text" Id="troubleshooting" Width="314" Height="67" X="26" Y="161" Text="For further information about problems during the installation, have a look at ModSecurityIIS Troubleshooting guide. Available at: https://github.com/SpiderLabs/ModSecurity/wiki/IIS-Troubleshooting. ATTENTION: This installation process no longer install OWASP CRS. Please refer to the OWASP CRS Project to understand how to install it. " />
+                <Control Type="Text" Id="troubleshooting" Width="314" Height="67" X="26" Y="161" Text="For further information about problems during the installation, have a look at ModSecurityIIS Troubleshooting guide. Available at: https://github.com/owasp-modsecurity/ModSecurity/wiki/IIS-Troubleshooting. ATTENTION: This installation process no longer install OWASP CRS. Please refer to the OWASP CRS Project to understand how to install it. " />
             </Dialog>
             <Binary Id="bannrbmp" SourceFile="wix\banner.jpg" />
             <TextStyle Id="WixUI_Font_Normal" FaceName="Tahoma" Size="8" />