diff --git a/apache2/acmp.c b/apache2/acmp.c index 105e7bc1..5ca6d5c1 100644 --- a/apache2/acmp.c +++ b/apache2/acmp.c @@ -16,6 +16,9 @@ * directly using the email address support@breach.com. * */ + +/* Aho-Corasick Matching */ + #include "acmp.h" #ifdef ACMP_USE_UTF8 diff --git a/apache2/modsecurity.c b/apache2/modsecurity.c index 7721c608..5be51acf 100644 --- a/apache2/modsecurity.c +++ b/apache2/modsecurity.c @@ -311,6 +311,8 @@ apr_status_t modsecurity_tx_init(modsec_rec *msr) { msr->geo_vars = apr_table_make(msr->mp, 8); if (msr->geo_vars == NULL) return -1; + msr->collections_original = apr_table_make(msr->mp, 8); + if (msr->collections_original == NULL) return -1; msr->collections = apr_table_make(msr->mp, 8); if (msr->collections == NULL) return -1; msr->collections_dirty = apr_table_make(msr->mp, 8); @@ -567,10 +569,7 @@ apr_status_t modsecurity_process_phase(modsec_rec *msr, unsigned int phase) { msr->tcache_items = 0; msr->tcache = apr_hash_make(msr->mp); - if (msr->tcache == NULL) { - msr_log(msr, 1, "Internal error: Failed to allocate transformation cache for phase %d", msr->phase); - return -1; - } + if (msr->tcache == NULL) return -1; } switch(phase) { diff --git a/apache2/modsecurity.h b/apache2/modsecurity.h index 2c56ffb9..7cfdd5ea 100644 --- a/apache2/modsecurity.h +++ b/apache2/modsecurity.h @@ -326,6 +326,7 @@ struct modsec_rec { int upload_remove_files; /* other */ + apr_table_t *collections_original; apr_table_t *collections; apr_table_t *collections_dirty; diff --git a/apache2/msc_test.c b/apache2/msc_test.c index fc787c3f..947fc6d4 100644 --- a/apache2/msc_test.c +++ b/apache2/msc_test.c @@ -33,6 +33,8 @@ #define RESULT_WRONGSIZE -3 #define RESULT_WRONGRET -4 +#define DEFAULT_ACTION "phase:2,log,auditlog,pass" + #define CMDLINE_OPTS "t:n:p:P:r:I:D:Nh" /* Types */ @@ -322,7 +324,7 @@ static int init_op(op_data_t *data, const char *name, const char *param, unsigne *errmsg = apr_psprintf(g_mp, "Failed to create ruleset for op \"%s\".", name); return -1; } - data->rule = msre_rule_create(data->ruleset, RULE_TYPE_NORMAL, conf_fn, 1, "UNIT_TEST", args, "t:none,pass,nolog", errmsg); + data->rule = msre_rule_create(data->ruleset, RULE_TYPE_NORMAL, conf_fn, 1, "UNIT_TEST", args, DEFAULT_ACTION, errmsg); if (data->rule == NULL) { *errmsg = apr_psprintf(g_mp, "Failed to create rule for op \"%s\": %s", name, *errmsg); return -1; @@ -523,6 +525,9 @@ static void init_msr() { g_msr->hostname = "localhost"; g_msr->msc_rule_mptmp = g_mp; g_msr->tx_vars = apr_table_make(g_mp, 1); + g_msr->collections_original = apr_table_make(g_mp, 1); + g_msr->collections = apr_table_make(g_mp, 1); + g_msr->collections_dirty = apr_table_make(g_mp, 1); } /** @@ -814,6 +819,9 @@ int main(int argc, const char * const argv[]) } else if (strcmp("action", type) == 0) { /* Actions */ + int n; + const apr_array_header_t *arr; + apr_table_entry_t *te; rc = test_action(&action_data, &errmsg); if (rc < 0) { @@ -835,6 +843,47 @@ int main(int argc, const char * const argv[]) ec = 1; } + /* Store any collections that were initialized and changed */ + arr = apr_table_elts(g_msr->collections); + te = (apr_table_entry_t *)arr->elts; + for (n = 0; n < arr->nelts; n++) { + apr_table_t *col = (apr_table_t *)te[n].val; +// apr_table_t *orig_col = NULL; + + if (g_msr->txcfg->debuglog_level >= 9) { + msr_log(g_msr, 9, "Found loaded collection: %s", te[n].key); + } + /* Only store those collections that changed. */ + if (apr_table_get(g_msr->collections_dirty, te[n].key)) { + int x = collection_store(g_msr, col); + + if (g_msr->txcfg->debuglog_level >= 9) { + msr_log(g_msr, 9, "Stored collection: %s (%d)", te[n].key, x); + } + } +#if 0 + /* Re-populate the original values with the new ones. */ + if ((orig_col = (apr_table_t *)apr_table_get(g_msr->collections_original, te[n].key)) != NULL) { + const apr_array_header_t *orig_arr = apr_table_elts(orig_col); + apr_table_entry_t *orig_te = (apr_table_entry_t *)orig_arr->elts; + int m; + + for (m = 0; m < orig_arr->nelts; m++) { + msc_string *mstr = (msc_string *)apr_table_get(col, orig_te[m].key); + + if (g_msr->txcfg->debuglog_level >= 9) { + msr_log(g_msr, 9, "Updating original collection: %s.%s=%s", te[n].key, mstr->name, mstr->value); + } + //apr_table_setn(orig_col, orig_te[m].key, (void *)mstr ); + collection_original_setvar(g_msr, te[n].key, mstr); + + + } + } +#endif + } + apr_table_clear(g_msr->collections_dirty); + apr_table_clear(g_msr->collections_original); } else { fprintf(stderr, "Unknown type: \"%s\"\n", type); diff --git a/apache2/persist_dbm.c b/apache2/persist_dbm.c index 59e4794b..c3e205cb 100644 --- a/apache2/persist_dbm.c +++ b/apache2/persist_dbm.c @@ -91,7 +91,7 @@ static apr_table_t *collection_unpack(modsec_rec *msr, const unsigned char *blob /** * */ -apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, +static apr_table_t *collection_retrieve_ex(apr_sdbm_t *existing_dbm, modsec_rec *msr, const char *col_name, const char *col_key, int col_key_len) { char *dbm_filename = NULL; @@ -117,17 +117,19 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, key.dptr = (char *)col_key; key.dsize = col_key_len + 1; - rc = apr_sdbm_open(&dbm, dbm_filename, APR_READ | APR_SHARELOCK, - CREATEMODE, msr->mp); - if (rc != APR_SUCCESS) { - return NULL; + if (existing_dbm == NULL) { + rc = apr_sdbm_open(&dbm, dbm_filename, APR_READ | APR_SHARELOCK, + CREATEMODE, msr->mp); + if (rc != APR_SUCCESS) { + return NULL; + } + } + else { + dbm = existing_dbm; } value = (apr_sdbm_datum_t *)apr_pcalloc(msr->mp, sizeof(apr_sdbm_datum_t)); rc = apr_sdbm_fetch(dbm, value, key); - - apr_sdbm_close(dbm); - if (rc != APR_SUCCESS) { msr_log(msr, 1, "Failed to read from DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename), get_apr_error(msr->mp, rc)); @@ -139,7 +141,7 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, } /* ENH Need expiration (and perhaps other metadata) accessible in blob - * form so we can determine if we need to convert to a table. This will + * form to determine if converting to a table is needed. This will * save some cycles. */ @@ -147,6 +149,11 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, col = collection_unpack(msr, (const unsigned char *)value->dptr, value->dsize, 1); if (col == NULL) return NULL; + /* Close after "value" used from fetch or memory may be overwritten. */ + if (existing_dbm == NULL) { + apr_sdbm_close(dbm); + } + /* Remove expired variables. */ do { arr = apr_table_elts(col); @@ -181,22 +188,24 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, /* Delete the collection if the variable "KEY" does not exist. * * ENH It would probably be more efficient to hold the DBM - * open until we determine if it needs deleted than to open a second + * open until determined if it needs deleted than to open a second * time. */ if (apr_table_get(col, "KEY") == NULL) { - rc = apr_sdbm_open(&dbm, dbm_filename, APR_CREATE | APR_WRITE | APR_SHARELOCK, - CREATEMODE, msr->mp); - if (rc != APR_SUCCESS) { - msr_log(msr, 1, "Failed to access DBM file \"%s\": %s", - log_escape(msr->mp, dbm_filename), get_apr_error(msr->mp, rc)); - return NULL; + if (existing_dbm == NULL) { + rc = apr_sdbm_open(&dbm, dbm_filename, APR_CREATE | APR_WRITE | APR_SHARELOCK, + CREATEMODE, msr->mp); + if (rc != APR_SUCCESS) { + msr_log(msr, 1, "Failed to access DBM file \"%s\": %s", + log_escape(msr->mp, dbm_filename), get_apr_error(msr->mp, rc)); + return NULL; + } + } + else { + dbm = existing_dbm; } rc = apr_sdbm_delete(dbm, key); - - apr_sdbm_close(dbm); - if (rc != APR_SUCCESS) { msr_log(msr, 1, "Failed deleting collection (name \"%s\", " "key \"%s\"): %s", log_escape(msr->mp, col_name), @@ -204,6 +213,11 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, return NULL; } + + if (existing_dbm == NULL) { + apr_sdbm_close(dbm); + } + if (expired && (msr->txcfg->debuglog_level >= 9)) { msr_log(msr, 9, "Collection expired (name \"%s\", key \"%s\").", col_name, log_escape_ex(msr->mp, col_key, col_key_len)); } @@ -231,7 +245,7 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, apr_time_t td; counter = atoi(var->value); - /* UPDATE_RATE is removed on store, so we add it back here */ + /* UPDATE_RATE is removed on store, so add it back here */ var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string)); var->name = "UPDATE_RATE"; var->name_len = strlen(var->name); @@ -259,6 +273,15 @@ apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, return col; } +/** + * + */ +apr_table_t *collection_retrieve(modsec_rec *msr, const char *col_name, + const char *col_key, int col_key_len) { + return collection_retrieve_ex(NULL, msr, col_name, col_key, col_key_len); +} + + /** * */ @@ -274,6 +297,8 @@ int collection_store(modsec_rec *msr, apr_table_t *col) { const apr_array_header_t *arr; apr_table_entry_t *te; int i; + const apr_table_t *stored_col = NULL; + const apr_table_t *orig_col = NULL; var_name = (msc_string *)apr_table_get(col, "__name"); if (var_name == NULL) { @@ -347,7 +372,35 @@ int collection_store(modsec_rec *msr, apr_table_t *col) { * convert back to table form */ - /* Calculate the size first. */ + rc = apr_sdbm_open(&dbm, dbm_filename, APR_CREATE | APR_WRITE | APR_SHARELOCK, + CREATEMODE, msr->mp); + if (rc != APR_SUCCESS) { + msr_log(msr, 1, "Failed to access DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename), + get_apr_error(msr->mp, rc)); + return -1; + } + + /* Only need to lock to pull in the stored data again. */ + rc = apr_sdbm_lock(dbm, APR_FLOCK_EXCLUSIVE); + if (rc != APR_SUCCESS) { + msr_log(msr, 1, "Failed to exclusivly lock DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename), + get_apr_error(msr->mp, rc)); + apr_sdbm_close(dbm); + return -1; + } + + /* If there is an original value, then create a delta and + * apply the delta to the current value */ + orig_col = (const apr_table_t *)apr_table_get(msr->collections_original, var_name->value); + if (orig_col != NULL) { + if (msr->txcfg->debuglog_level >= 9) { + msr_log(msr, 9, "Re-retrieving collection prior to store: %s", apr_psprintf(msr->mp, "%.*s", var_name->value_len, var_name->value)); + } + + stored_col = (const apr_table_t *)collection_retrieve_ex(dbm, msr, var_name->value, var_key->value, var_key->value_len); + } + + /* Merge deltas and calculate the size first. */ blob_size = 3 + 2; arr = apr_table_elts(col); te = (apr_table_entry_t *)arr->elts; @@ -355,6 +408,33 @@ int collection_store(modsec_rec *msr, apr_table_t *col) { msc_string *var = (msc_string *)te[i].val; int len; + /* If there is an original value, then apply the delta + * to the latest stored value */ + if (stored_col != NULL) { + const msc_string *orig_var = (const msc_string *)apr_table_get(orig_col, var->name); + if (orig_var != NULL) { + const msc_string *stored_var = (const msc_string *)apr_table_get(stored_col, var->name); + if (stored_var != NULL) { + int origval = atoi(orig_var->value); + int ourval = atoi(var->value); + int storedval = atoi(stored_var->value); + int delta = ourval - origval; + int value = storedval + delta; + + if (value < 0) value = 0; /* Counters never go below zero. */ + + var->value = apr_psprintf(msr->mp, "%d", value); + var->value_len = strlen(var->value); + if (msr->txcfg->debuglog_level >= 9) { + msr_log(msr, 9, "Delta applied for %s.%s %d->%d (%d): %d + (%d) = %d [%s,%d]", + log_escape_ex(msr->mp, var_name->value, var_name->value_len), + log_escape_ex(msr->mp, var->name, var->name_len), + origval, ourval, delta, storedval, delta, value, var->value, var->value_len); + } + } + } + } + len = var->name_len + 1; if (len >= 65536) len = 65536; blob_size += len + 2; @@ -418,24 +498,17 @@ int collection_store(modsec_rec *msr, apr_table_t *col) { value.dptr = (char *)blob; value.dsize = blob_size; - rc = apr_sdbm_open(&dbm, dbm_filename, APR_CREATE | APR_WRITE | APR_SHARELOCK, - CREATEMODE, msr->mp); - if (rc != APR_SUCCESS) { - msr_log(msr, 1, "Failed to access DBM file \"%s\": %s", log_escape(msr->mp, dbm_filename), - get_apr_error(msr->mp, rc)); - return -1; - } - rc = apr_sdbm_store(dbm, key, value, APR_SDBM_REPLACE); - - apr_sdbm_close(dbm); - if (rc != APR_SUCCESS) { msr_log(msr, 1, "Failed to write to DBM file \"%s\": %s", dbm_filename, get_apr_error(msr->mp, rc)); return -1; } + /* ENH: Do we need to unlock()? Or will just close() suffice? */ + apr_sdbm_unlock(dbm); + apr_sdbm_close(dbm); + if (msr->txcfg->debuglog_level >= 4) { msr_log(msr, 4, "Persisted collection (name \"%s\", key \"%s\").", log_escape_ex(msr->mp, var_name->value, var_name->value_len), log_escape_ex(msr->mp, var_key->value, var_key->value_len)); @@ -485,8 +558,8 @@ int collections_remove_stale(modsec_rec *msr, const char *col_name) { return -1; } - /* No one can write to the file while we're - * doing this so let's do it as fast as we can. + /* No one can write to the file while doing this so + * do it as fast as possible. */ rc = apr_sdbm_firstkey(dbm, &key); while(rc == APR_SUCCESS) { diff --git a/apache2/re.h b/apache2/re.h index eb9d153c..bcf256ad 100644 --- a/apache2/re.h +++ b/apache2/re.h @@ -49,6 +49,8 @@ typedef struct msre_cache_rec msre_cache_rec; /* Actions, variables, functions and operator functions */ +apr_status_t DSOLOCAL collection_original_setvar(modsec_rec *msr, const char *col_name, const msc_string *orig_var); + int DSOLOCAL expand_macros(modsec_rec *msr, msc_string *var, msre_rule *rule, apr_pool_t *mptmp); apr_status_t DSOLOCAL msre_parse_targets(msre_ruleset *ruleset, const char *text, diff --git a/apache2/re_actions.c b/apache2/re_actions.c index 96c0993f..d18df4ae 100644 --- a/apache2/re_actions.c +++ b/apache2/re_actions.c @@ -307,6 +307,64 @@ int expand_macros(modsec_rec *msr, msc_string *var, msre_rule *rule, apr_pool_t return 1; } +/** + * Record the original collection values to use to calculate deltas. + * This can be called multiple times and will not overwrite the first + * value that is set. + */ +apr_status_t collection_original_setvar(modsec_rec *msr, const char *col_name, const msc_string *orig_var) { + apr_table_t *table = NULL; + msc_string *var = NULL; + const char *var_name = NULL; + + if (orig_var == NULL) { + msr_log(msr, 1, "Internal Error: Attempt to record NULL original variable."); + return -1; + } + + var_name = orig_var->name; + table = (apr_table_t *)apr_table_get(msr->collections_original, col_name); + + /* Does the collection exist already? */ + if (table == NULL) { + table = apr_table_make(msr->mp, 24); + if (table == NULL) { + msr_log(msr, 1, "Failed to allocate space for original collection."); + return -1; + } + apr_table_setn(msr->collections_original, apr_pstrdup(msr->mp, col_name), (void *)table); + } + else { + /* Does the variable exist already? */ + var = (msc_string *)apr_table_get(table, var_name); + if (var != NULL) { + if (msr->txcfg->debuglog_level >= 9) { + msr_log(msr, 9, "Original collection variable: %s.%s = \"%s\"", col_name, var_name, log_escape_ex(msr->mp, orig_var->value, orig_var->value_len)); + } + return 1; + } + } + + var = (msc_string *)apr_palloc(msr->mp, sizeof(msc_string)); + if (var == NULL) { + msr_log(msr, 1, "Failed to allocate space for original collection variable."); + return -1; + } + + /* Copy the original var and add to collection. */ + var->name = orig_var->name ? apr_pstrmemdup(msr->mp, orig_var->name, orig_var->name_len) : NULL; + var->name_len = orig_var->name_len; + var->value = orig_var->value ? apr_pstrmemdup(msr->mp, orig_var->value, orig_var->value_len) : NULL; + var->value_len = orig_var->value_len; + apr_table_setn(table, apr_pstrmemdup(msr->mp, var->name, var->name_len), (void *)var); + + if (msr->txcfg->debuglog_level >= 9) { + msr_log(msr, 9, "Recorded original collection variable: %s.%s = \"%s\"", col_name, var_name, log_escape_ex(msr->mp, var->value, var->value_len)); + } + + return 0; +} + /* id */ static apr_status_t msre_action_id_init(msre_engine *engine, msre_actionset *actionset, @@ -1211,11 +1269,16 @@ static apr_status_t msre_action_setvar_execute(modsec_rec *msr, apr_pool_t *mptm rec->name = apr_pstrdup(msr->mp, var_name); rec->name_len = strlen(rec->name); value = 0; + rec->value = apr_psprintf(msr->mp, "%d", value); + rec->value_len = strlen(rec->value); } else { value = atoi(rec->value); } + /* Record the original value before we change it */ + collection_original_setvar(msr, col_name, rec); + /* Expand values in value */ val->value = var_value; val->value_len = strlen(val->value); @@ -1488,6 +1551,7 @@ static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name, const char *col_name, const char *col_key, unsigned int col_key_len) { apr_table_t *table = NULL; + msc_string *var = NULL; /* IMP1 Cannot initialise the built-in collections this way. */ @@ -1501,13 +1565,12 @@ static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name, table = collection_retrieve(msr, real_col_name, col_key, col_key_len); if (table == NULL) { - msc_string *var = NULL; - /* Does not exist yet - create new. */ msr_log(msr, 4, "Creating collection (name \"%s\", key \"%s\").", real_col_name, col_key); table = apr_table_make(msr->mp, 24); + if (table == NULL) return -1; /* IMP1 Is the timeout hard-coded to 3600? */ @@ -1584,6 +1647,12 @@ static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name, apr_table_setn(table, var->name, (void *)var); } + /* Record the original counter value before we change it */ + var = (msc_string *)apr_table_get(table, "UPDATE_COUNTER"); + if (var != NULL) { + collection_original_setvar(msr, col_name, var); + } + /* Add the collection to the list. */ apr_table_setn(msr->collections, apr_pstrdup(msr->mp, col_name), (void *)table); diff --git a/apache2/t/regression/action/10-detectiononly-actions.t b/apache2/t/regression/action/10-detectiononly-actions.t index 14fc4072..238972c7 100644 --- a/apache2/t/regression/action/10-detectiononly-actions.t +++ b/apache2/t/regression/action/10-detectiononly-actions.t @@ -107,7 +107,7 @@ match_log => { error => [ qr/ModSecurity: Warning. Unconditional match in SecAction.*ALLOWED/, 1 ], -error => [ qr/Access allowed/, 1 ], -# TODO: Allow should probably stop stop execution +# TODO: Allow should probably stop rule execution # -error => [ qr/DENIED/, 1 ], }, match_response => { diff --git a/apache2/t/tfn/cssDecode.t b/apache2/t/tfn/cssDecode.t new file mode 100644 index 00000000..180186bd --- /dev/null +++ b/apache2/t/tfn/cssDecode.t @@ -0,0 +1,58 @@ +### Empty +{ + type => "tfn", + name => "cssDecode", + input => "", + output => "", + ret => 0, +}, + +### Nothing +{ + type => "tfn", + name => "cssDecode", + input => "TestCase", + output => "TestCase", + ret => 0, +}, +{ + type => "tfn", + name => "cssDecode", + input => "Test\0Case", + output => "Test\0Case", + ret => 0, +}, + +### Valid Sequences +{ + type => "tfn", + name => "cssDecode", + input => "test\\a\\b\\f\\n\\r\\t\\v\\?\\'\\\"\\0\\12\\123\\1234\\12345\\123456\\ff01\\ff5e\\\n\\0 string", + output => qq(test\x0a\x0b\x0fnrtv?'"\x00\x12\x23\x34\x45\x56\x21\x7e\x00 string), + ret => 1, +}, + +### Invalid Sequences +# Trailing escape == line continuation with no line following (ie nothing) +{ + type => "tfn", + name => "cssDecode", + input => "test\\", + output => "test", + ret => 1, +}, + +# Edge cases +# "\1A" == "\x1A" +# "\1 A" == "\x01A" +# "\1234567" == "\x567" +# "\123456 7" == "\x567" +# "\1x" == "\x01x" +# "\1 x" == "\x01 x" +{ + type => "tfn", + name => "cssDecode", + input => "\\1A\\1 A\\1234567\\123456 7\\1x\\1 x", + output => "\x1A\x01A\x567\x567\x01x\x01x", + ret => 1, +},