Adds support to collection using memcache.

This is the initial support to collections using memcache.

apache2/Makefile.am

@@ -20,6 +20,7 @@ mod_security2_la_SOURCES = acmp.c \
     msc_multipart.c \
     msc_parsers.c \
     msc_pcre.c \
+    msc_persistent_memcache.c \
     msc_release.c \
     msc_reqbody.c \
     msc_tree.c \

apache2/Makefile.win

@@ -57,6 +57,7 @@ OBJS = mod_security2.obj apache2_config.obj apache2_io.obj apache2_util.obj \
        msc_release.obj \
        msc_status_engine.obj \
        msc_json.obj \
+       msc_persistent_memcache.obj \
        libinjection/libinjection_html5.obj \
        libinjection/libinjection_sqli.obj \
        libinjection/libinjection_xss.obj

apache2/apache2_config.c

@@ -26,6 +26,9 @@
 #include "msc_lua.h"
 #endif
 
+#ifdef WITH_LIBMEMCACHED
+#include <memcached.h>
+#endif
 
 /* -- Directory context creation and initialisation -- */
 
@@ -65,6 +68,7 @@ void *create_directory_config(apr_pool_t *mp, char *path)
     dcfg->cookie_format = NOT_SET;
     dcfg->argument_separator = NOT_SET;
     dcfg->cookiev0_separator = NOT_SET_P;
+    dcfg->persistent_storage = NOT_SET;
 
     dcfg->rule_inheritance = NOT_SET;
     dcfg->rule_exceptions = apr_array_make(mp, 16, sizeof(rule_exception *));
@@ -1125,6 +1129,34 @@ static const char *cmd_marker(cmd_parms *cmd, void *_dcfg, const char *p1)
     return add_marker(cmd, (directory_config *)_dcfg, SECMARKER_TARGETS, SECMARKER_ARGS, action);
 }
 
+static const char *cmd_persistent_storage(cmd_parms *cmd, void *_dcfg,
+        const char *p1, const char *p2)
+{
+    directory_config *dcfg = (directory_config *)_dcfg;
+
+    if (strcmp(p1, "memcache") == 0) {
+#if WITH_LIBMEMCACHED
+        dcfg->persistent_storage = STORAGE_TYPE_MEMCACHE;
+        memcache = memcached(p2, strlen(p2));
+        if (memcache == NULL) {
+            return apr_psprintf(cmd->pool, "ModSecurity: Failed to connect " \
+                    "to memcache server(s)");
+        }
+#else
+
+        return apr_psprintf(cmd->pool, "ModSecurity: Memcached is not " \
+                "supported on this version. See compilation options for " \
+                "further information.");
+#endif
+    }
+    else {
+        dcfg->persistent_storage = STORAGE_TYPE_LOCAL;
+    }
+
+    return NULL;
+}
+
+
 static const char *cmd_cookiev0_separator(cmd_parms *cmd, void *_dcfg,
         const char *p1)
 {
@@ -3306,6 +3338,14 @@ const command_rec module_directives[] = {
         "marker for a skipAfter target"
     ),
 
+    AP_INIT_TAKE12 (
+        "SecPersistentStorage",
+        cmd_persistent_storage,
+        NULL,
+        CMD_SCOPE_ANY,
+        "set storage type"
+    ),
+
     AP_INIT_TAKE1 (
         "SecPcreMatchLimit",
         cmd_pcre_match_limit,

apache2/mod_security2.c

@@ -44,6 +44,10 @@
 #include <yajl/yajl_version.h>
 #endif /* WITH_YAJL */
 
+#ifdef WITH_LIBMEMCACHED
+#include <memcached.h>
+#endif
+
 /* ModSecurity structure */
 
 msc_engine DSOLOCAL *modsecurity = NULL;
@@ -62,6 +66,10 @@ apr_file_t DSOLOCAL *guardianlog_fd = NULL;
 
 char DSOLOCAL *guardianlog_condition = NULL;
 
+#if WITH_LIBMEMCACHED
+memcached_st *memcache = NULL;
+#endif
+
 unsigned long int DSOLOCAL msc_pcre_match_limit = 0;
 
 unsigned long int DSOLOCAL msc_pcre_match_limit_recursion = 0;

apache2/modsecurity.c

@@ -18,11 +18,13 @@
 
 #include "modsecurity.h"
 #include "msc_parsers.h"
+#include "msc_persistent_memcache.h"
 #include "msc_util.h"
 #include "msc_json.h"
 #include "msc_xml.h"
 #include "apr_version.h"
 
+
 unsigned long int DSOLOCAL unicode_codepage = 0;
 
 int DSOLOCAL *unicode_map_table = NULL;
@@ -209,7 +211,13 @@ static void modsecurity_persist_data(modsec_rec *msr) {
 
         /* Only store those collections that changed. */
         if (apr_table_get(msr->collections_dirty, te[i].key)) {
+
+            if (msr->dcfg1->persistent_storage != STORAGE_TYPE_MEMCACHE) {
                 collection_store(msr, col);
+            } else
+            {
+                msc_memcache_collection_store(msr, col);
+            }
         }
     }
 
@@ -229,8 +237,10 @@ static void modsecurity_persist_data(modsec_rec *msr) {
         arr = apr_table_elts(msr->collections);
         te = (apr_table_entry_t *)arr->elts;
         for (i = 0; i < arr->nelts; i++) {
+            if (msr->dcfg1->persistent_storage != STORAGE_TYPE_MEMCACHE) {
                 collections_remove_stale(msr, te[i].key);
             }
+        }
 
         msr->time_gc = apr_time_now() - time_after;
 

apache2/modsecurity.h

@@ -60,6 +60,13 @@ typedef struct msc_parm msc_parm;
 #include "msc_lua.h"
 #endif
 
+#ifdef WITH_LIBMEMCACHED
+#include <memcached.h>
+#endif
+
+#define STORAGE_TYPE_LOCAL 1
+#define STORAGE_TYPE_MEMCACHE 2
+
 #define PHASE_REQUEST_HEADERS       1
 #define PHASE_REQUEST_BODY          2
 #define PHASE_RESPONSE_HEADERS      3
@@ -139,6 +146,10 @@ extern module AP_MODULE_DECLARE_DATA security2_module;
 
 extern DSOLOCAL const command_rec module_directives[];
 
+#if WITH_LIBMEMCACHED
+extern DSOLOCAL memcached_st *memcache;
+#endif
+
 extern DSOLOCAL unsigned long int msc_pcre_match_limit;
 
 extern DSOLOCAL unsigned long int msc_pcre_match_limit_recursion;
@@ -491,6 +502,7 @@ struct directory_config {
     int                  cookie_format;
     int                  argument_separator;
     const char           *cookiev0_separator;
+    int                  persistent_storage;
 
     int                  rule_inheritance;
     apr_array_header_t  *rule_exceptions;

apache2/msc_persistent_memcache.c

@@ -0,0 +1,495 @@
+/*
+* ModSecurity for Apache 2.x, http://www.modsecurity.org/
+* Copyright (c) 2004-2013 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+*
+* You may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* If any of the files related to licensing are missing or if you have any
+* other questions related to licensing please contact Trustwave Holdings, Inc.
+* directly using the email address security@modsecurity.org.
+*/
+
+#include "msc_persistent_memcache.h"
+
+#ifdef WITH_LIBMEMCACHED
+#include <memcached.h>
+#endif
+
+#define MEMCACHE_INDEX_ELEMENT "__index"
+
+
+static char *msc_collection_default_keys[]= {
+    "__expire_KEY",
+    "__key",
+    "__name",
+    MEMCACHE_INDEX_ELEMENT,
+    "CREATE_TIME",
+    "LAST_UPDATE_TIME",
+    "TIMEOUT",
+    "UPDATE_COUNTER",
+    "UPDATE_RATE",
+    "KEY",
+    NULL
+};
+
+
+int memcache_is_collection_default_key(const char *key)
+{
+#ifdef WITH_LIBMEMCACHED
+    char **keys = msc_collection_default_keys;
+    char *k = NULL;
+
+    while (k = *keys++) {
+        if (strcmp(key, k) == 0) {
+            return 1;
+        }
+    }
+#endif
+    return 0;
+}
+
+
+static int memcache_store_key(modsec_rec *msr, const msc_string *var_name,
+        const msc_string *var_key, const char *key, msc_string* var,
+        const time_t timeout)
+{
+#ifdef WITH_LIBMEMCACHED
+    char memc_key[MEMCACHED_MAX_KEY];
+    memcached_return rc;
+
+    snprintf(memc_key, MEMCACHED_MAX_KEY, "%s,%s,%s", var_name->value,
+            var_key->value, key);
+
+    rc = memcached_set(memcache, memc_key, strlen(memc_key), var->value,
+             var->value_len, timeout, (uint32_t)0);
+
+    if (rc != MEMCACHED_SUCCESS) {
+        msr_log(msr, 9, "memcache_store_key: Problems storing the key: %s " \
+               "(%s)  with value: %s", key, memc_key, var->value);
+        return -1;
+    }
+
+    msr_log(msr, 9, "memcache_store_key: key: %s, value: %s stored.", memc_key,
+            var->value);
+#endif
+    return 0;
+}
+
+
+static msc_string *memcache_fetch_key(modsec_rec *msr, const char *col_name,
+        const char *col_key, char *key)
+{
+    msc_string *var = NULL;
+#ifdef WITH_LIBMEMCACHED
+    char namespace_key[MEMCACHED_MAX_KEY];
+    char *value = NULL;
+    size_t value_length = 0;
+
+    snprintf(namespace_key, MEMCACHED_MAX_KEY-1, "%s,%s,%s", col_name, col_key,
+            key);
+    value = memcached_get(memcache, namespace_key, strlen(namespace_key),
+            &value_length, 0, 0);
+
+    if (value) {
+        var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
+        var->name = key;
+        var->name_len = strlen(key);
+        var->value = apr_pstrdup(msr->mp, value);
+        var->value_len = value_length;
+
+        free(value);
+
+        msr_log(msr, 9, "memcache_fetch_key: key: %s, value: %s retrieved",
+                 key, var->value);
+    }
+    else {
+        msr_log(msr, 9, "memcache_fetch_key: Failed to fetch key: %s (%s)",
+                key, namespace_key);
+    }
+#endif
+    return var;
+}
+
+
+static int memcache_apr_table_to_memcache (modsec_rec *msr, const apr_table_t *col,
+        const msc_string *var_name, const msc_string *var_key)
+{
+    int ret = 0;
+#ifdef WITH_LIBMEMCACHED
+    const apr_array_header_t *arr = NULL;
+    apr_table_entry_t *te = NULL;
+    msc_string *var = NULL;
+    msc_string *index = NULL;
+    msc_string *timeout_string = NULL;
+    time_t timeout = NULL;
+    int i = 0;
+
+    timeout_string = (msc_string *)apr_table_get(col, "TIMEOUT");
+    if (timeout_string == NULL) {
+        msr_log(msr, 9, "memcache_apr_table_to_memcache: Failed to convert an "\
+                "apr table to memcache, it is missing the attribute TIMEOUT. " \
+                "Aborting.");
+        ret = -1;
+        goto missing_timeout;
+    }
+
+    //FIXME: OMG. Too many cast.
+    timeout = (int) atoi((char *)timeout_string->value);
+
+    index = apr_pcalloc(msr->mp, sizeof(msc_string));
+
+    arr = apr_table_elts(col);
+    te = (apr_table_entry_t *)arr->elts;
+    for (i = 0; i < arr->nelts; i++) {
+        msc_string *var = (msc_string *)te[i].val;
+        
+        /* Index will be stored after all other alements, so that we can
+           compute its value.
+         */
+        if (strcmp(te[i].key, MEMCACHE_INDEX_ELEMENT) == 0) 
+            continue;
+
+        if (!memcache_is_collection_default_key(te[i].key)) {
+            msr_log(msr, 9, "memcache_apr_table_to_memcache: key: \"%s\" is " \
+                    "a custom attribute, index will be necessary.", te[i].key);
+
+            if (index->value_len > 0) {
+                index->value = apr_psprintf(msr->mp, "%s %s", index->value,
+                        te[i].key);
+                index->value_len = strlen(index->value);
+            }
+            else {
+                index->value = apr_psprintf(msr->mp, "%s", te[i].key);
+                index->value_len = strlen(index->value);
+            }
+        }
+
+        ret = memcache_store_key(msr, var_name, var_key, te[i].key, var,
+                timeout);
+        if (ret) {
+           msr_log(msr, 9, "memcache_apr_table_to_memcache: Failed to store " \
+                   "element into memcache server. Aborting.");
+           goto failed;
+        }
+
+    }
+
+    ret = memcache_store_key(msr, var_name, var_key, MEMCACHE_INDEX_ELEMENT,
+            index, timeout);
+
+    if (ret) {
+        msr_log(msr, 9, "memcache_apr_table_to_memcache: Failed to store " \
+                "memcache into memcache server. Aborting.");
+        goto failed;
+    }
+
+    msr_log(msr, 4, "memcache_apr_table_to_memcache: Sucessfully saved " \
+            "object, with the following custom elements: %s", index->value);
+
+missing_timeout:
+failed:
+#endif
+    return ret;
+}
+
+
+static apr_table_t *memcache_to_apr_table (modsec_rec *msr,
+        const char *col_name, const char *col_key, const int col_key_len)
+{
+#ifdef WITH_LIBMEMCACHED
+    char **keys = msc_collection_default_keys;
+    char **custom_keys = NULL;
+    char *k = NULL;
+    apr_table_t *ret = NULL;
+    msc_string *value = NULL;
+    msc_string *index;
+    int fetched_any = 0;
+
+    msr_log(msr, 9, "memcache_to_apr_table: Retrieving %s form memcache " \
+            "server", col_name);
+
+    ret = apr_table_make(msr->mp, 24);
+    if (ret == NULL) {
+        msr_log(msr, 9, "memcache_to_apr_table: failed to create the apr " \
+                "table");
+        goto failed_make_table;
+    }
+
+   
+    while (k = *keys++) {
+        value = memcache_fetch_key(msr, col_name, col_key, k);
+        if (value) {
+            /* msr_log(msr, 9, "memcache_to_apr_table: Successfully retrieved: " \
+                    "%s with value: %s", value->name, value->value); */
+            apr_table_setn(ret, value->name, (void *)value);
+            fetched_any = 1;
+        }
+    }
+
+    /* If the table does not exist at all. */
+    if (fetched_any == 0) {
+        goto not_exist;
+    }
+
+    index = (msc_string *)apr_table_get(ret, MEMCACHE_INDEX_ELEMENT);
+    if (index) {
+            msr_log(msr, 9, "memcache_to_apr_table: This collection contains "\
+                    "custom elements: %s", index->value);
+
+        apr_tokenize_to_argv(index->value, &custom_keys, msr->mp);
+        while (k = *custom_keys++) {
+            value = memcache_fetch_key(msr, col_name, col_key, k);
+            if (value) {
+                /*msr_log(msr, 9, "memcache_to_apr_table: Successfully " \
+                        "retrieved custom attribute: %s with value: %s",
+                        value->name, value->value);*/
+
+                apr_table_setn(ret, value->name, (void *)value);
+            }
+        }
+    }
+
+    /* Sanity check by looking at TIMEOUT key */
+    if (apr_table_get(ret, MEMCACHE_INDEX_ELEMENT) == NULL) {
+        msr_log(msr, 9, "memcache_to_apr_table: Something odd happens, this " \
+                        "collection is missing basic elements such as: %s. " \
+                        "Acting like it does not exist.",
+                        MEMCACHE_INDEX_ELEMENT);
+        goto not_exist;
+    }
+
+failed_make_table:
+    return ret;
+not_exist:
+#endif
+    return NULL;
+}
+
+
+static void memcache_remove_expired_variables(modsec_rec *msr,
+        apr_table_t *rtable)
+{
+#ifdef WITH_LIBMEMCACHED
+    const apr_array_header_t *arr = NULL;
+    apr_table_entry_t *te = NULL;
+    int expired = 0;
+    int i = 0;
+
+    /* Remove expired variables. */
+    do {
+        arr = apr_table_elts(rtable);
+        te = (apr_table_entry_t *)arr->elts;
+
+        for (i = 0; i < arr->nelts; i++) {
+            if (strncmp(te[i].key, "__expire_", 9) == 0) {
+                msc_string *var = (msc_string *)te[i].val;
+                int expiry_time = atoi(var->value);
+
+                if (expiry_time <= apr_time_sec(msr->request_time)) {
+                    char *key_to_expire = te[i].key;
+
+                    /* Done early if the col expired */
+                    if (strcmp(key_to_expire, "__expire_KEY") == 0) {
+                        expired = 1;
+                    }
+                    
+                    if (msr->txcfg->debuglog_level >= 9) {
+                        msr_log(msr, 9, "memcache_remove_expired_variables: " \
+                                "Removing key \"%s\" from collection.",
+                                key_to_expire + 9);
+                        msr_log(msr, 9, "memcache_remove_expired_variables: " \
+                                "Removing key \"%s\" from collection.",
+                                key_to_expire);
+                    }
+                    
+                    apr_table_unset(rtable, key_to_expire + 9);
+                    apr_table_unset(rtable, key_to_expire);
+                    
+                    if (msr->txcfg->debuglog_level >= 4) {
+                        msr_log(msr, 4, "memcache_remove_expired_variables: " \
+                                "Removed expired variable \"%s\".",
+                                key_to_expire + 9);
+                    }
+                    
+                    break;
+                }
+            }
+        }
+    } while((!expired && (i != arr->nelts)));
+#endif
+}
+
+
+static void memcache_collection_update_rate(modsec_rec *msr,
+        apr_table_t *rtable)
+{
+#ifdef WITH_LIBMEMCACHED
+    msc_string *var;
+    int create_time, counter;
+
+    var = (msc_string *)apr_table_get(rtable, "CREATE_TIME");
+    if (var == NULL) {
+        /* Error. */
+    } else {
+        create_time = atoi(var->value);
+        var = (msc_string *)apr_table_get(rtable, "UPDATE_COUNTER");
+        if (var == NULL) {
+            /* Error. */
+        } else {
+            apr_time_t td;
+            counter = atoi(var->value);
+
+            /* UPDATE_RATE is removed on store, so add it back here */
+            var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
+            var->name = "UPDATE_RATE";
+            var->name_len = strlen(var->name);
+            apr_table_setn(rtable, var->name, (void *)var);
+
+            /* NOTE: No rate if there has been no time elapsed */
+            td = (apr_time_sec(apr_time_now()) - create_time);
+            if (td == 0) {
+                var->value = apr_psprintf(msr->mp, "%d", 0);
+            }
+            else {
+                var->value = apr_psprintf(msr->mp, "%" APR_TIME_T_FMT,
+                    (apr_time_t)((60 * counter)/td));
+            }
+            var->value_len = strlen(var->value);
+        }
+    }
+#endif
+}
+
+
+apr_table_t *msc_memcache_collection_retrieve(modsec_rec *msr,
+        const char *col_name, const char *col_key, const int col_key_len)
+{
+    apr_table_t *rtable = NULL;
+#ifdef WITH_LIBMEMCACHED
+    int i = 0;
+    apr_time_t time_before = apr_time_now();
+
+    rtable = memcache_to_apr_table(msr, col_name, col_key, col_key_len);
+
+    if (rtable == NULL) {
+        msr_log(msr, 4, "msc_memcache_collection_retrieve: Collection: %s " \
+                "was not found on memcache server or there was a problem " \
+                "fetching it.", col_name);
+        goto not_know;
+    }
+
+    memcache_remove_expired_variables(msr, rtable);
+    memcache_collection_update_rate(msr, rtable);
+
+    if (msr->txcfg->debuglog_level >= 4) {
+        msr_log(msr, 4, "msc_memcache_collection_retrieve: Retrieved " \
+                "collection (name \"%s\", key \"%s\").",
+                log_escape(msr->mp, col_name),
+                log_escape_ex(msr->mp, col_key, col_key_len));
+    }
+
+not_know:
+    msr->time_storage_read += apr_time_now() - time_before;
+#endif
+    return rtable;
+}
+
+
+int msc_memcache_collection_store(modsec_rec *msr, apr_table_t *col) {
+#ifdef WITH_LIBMEMCACHED
+    msc_string *var_name = NULL, *var_key = NULL;
+    apr_status_t rc;
+    const apr_array_header_t *arr;
+    apr_table_entry_t *te;
+    int i;
+    const apr_table_t *stored_col = NULL;
+    const apr_table_t *orig_col = NULL;
+    unsigned int blob_size, blob_offset;
+
+    var_name = (msc_string *)apr_table_get(col, "__name");
+    var_key = (msc_string *)apr_table_get(col, "__key");
+
+    if (var_key == NULL || var_name == NULL) {
+        msr_log(msr, 4, "msc_memcache_collection_store: Not able to store " \
+                "collection, missing basic elements.");
+
+        goto failed;
+    }
+
+    /* Delete IS_NEW on store. */
+    apr_table_unset(col, "IS_NEW");
+
+    /* Delete UPDATE_RATE on store to save space as it is calculated */
+    apr_table_unset(col, "UPDATE_RATE");
+
+    /* Update the timeout value. */
+    {
+        msc_string *var = (msc_string *)apr_table_get(col, "TIMEOUT");
+        if (var != NULL) {
+            int timeout = atoi(var->value);
+            var = (msc_string *)apr_table_get(col, "__expire_KEY");
+            if (var != NULL) {
+                var->value = apr_psprintf(msr->mp, "%" APR_TIME_T_FMT, \
+                        (apr_time_t)(apr_time_sec(apr_time_now()) + timeout));
+                var->value_len = strlen(var->value);
+            }
+        }
+    }
+
+    /* LAST_UPDATE_TIME */
+    {
+        msc_string *var = (msc_string *)apr_table_get(col, "LAST_UPDATE_TIME");
+        if (var == NULL) {
+            var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
+            var->name = "LAST_UPDATE_TIME";
+            var->name_len = strlen(var->name);
+            apr_table_setn(col, var->name, (void *)var);
+        }
+        var->value = apr_psprintf(msr->mp, "%" APR_TIME_T_FMT, \
+                (apr_time_t)(apr_time_sec(apr_time_now())));
+        var->value_len = strlen(var->value);
+    }
+
+    /* UPDATE_COUNTER */
+    {
+        msc_string *var = (msc_string *)apr_table_get(col, "UPDATE_COUNTER");
+        int counter = 0;
+        if (var == NULL) {
+            var = (msc_string *)apr_pcalloc(msr->mp, sizeof(msc_string));
+            var->name = "UPDATE_COUNTER";
+            var->name_len = strlen(var->name);
+            apr_table_setn(col, var->name, (void *)var);
+        } else {
+            counter = atoi(var->value);
+        }
+        var->value = apr_psprintf(msr->mp, "%d", counter + 1);
+        var->value_len = strlen(var->value);
+    }
+
+    orig_col = (const apr_table_t *)apr_table_get(msr->collections_original, \
+            var_name->value);
+
+    if (orig_col != NULL) {
+        if (msr->txcfg->debuglog_level >= 9) {
+            msr_log(msr, 9, "msc_memcache_collection_store: Re-retrieving " \
+                    "collection prior to store: %s", apr_psprintf(msr->mp,
+                "%.*s", var_name->value_len, var_name->value));
+        }
+
+        stored_col = msc_memcache_collection_retrieve(msr, var_name->value,
+                var_key->value, var_key->value_len);
+    }
+
+    memcache_apr_table_to_memcache(msr, col, var_name, var_key);
+
+    msr_log(msr, 4, "msc_memcache_collection_store: Successfully stored " \
+                    "collection.");
+    return 0;
+failed:
+#endif
+    return -1;
+}
+

apache2/msc_persistent_memcache.h

@@ -0,0 +1,30 @@
+/*
+* ModSecurity for Apache 2.x, http://www.modsecurity.org/
+* Copyright (c) 2004-2013 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+*
+* You may not use this file except in compliance with
+* the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* If any of the files related to licensing are missing or if you have any
+* other questions related to licensing please contact Trustwave Holdings, Inc.
+* directly using the email address security@modsecurity.org.
+*/
+
+#ifndef _PERSIST_MEMCACHE_H_
+#define _PERSIST_MEMCACHE_H_
+
+#include "apr_general.h"
+#include "modsecurity.h"
+
+apr_table_t DSOLOCAL *msc_memcache_collection_retrieve(modsec_rec *msr,
+	const char *col_name, const char *col_value, int col_value_length);
+
+int DSOLOCAL msc_memcache_collection_store(modsec_rec *msr,
+	apr_table_t *collection);
+
+int DSOLOCAL msc_memcache_collections_remove_stale(modsec_rec *msr,
+	const char *col_name);
+
+#endif

apache2/re.h

@@ -46,6 +46,10 @@ typedef struct msre_cache_rec msre_cache_rec;
 #include "msc_lua.h"
 #endif
 
+#ifdef WITH_LIBMEMCACHED
+#include <memcached.h>
+#endif
+
 /* Actions, variables, functions and operator functions */
 char DSOLOCAL *update_rule_target_ex(modsec_rec *msr, msre_ruleset *ruleset, msre_rule *rule, const char *p2,
         const char *p3);

apache2/re_actions.c

@@ -17,6 +17,7 @@
 #include "apr_lib.h"
 #include "apr_strmatch.h"
 
+#include "msc_persistent_memcache.h"
 
 /**
  * Register action with the engine.
@@ -1933,33 +1934,22 @@ static apr_status_t msre_action_deprecatevar_execute(modsec_rec *msr, apr_pool_t
     return 1;
 }
 
-static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name,
+static apr_table_t *create_collection_table(modsec_rec *msr, const char *real_col_name,
     const char *col_name, const char *col_key, unsigned int col_key_len)
 {
     apr_table_t *table = NULL;
     msc_string *var = NULL;
 
-    /* IMP1 Cannot initialise the built-in collections this way. */
-
-    /* Does the collection exist already? */
-    if (apr_table_get(msr->collections, col_name) != NULL) {
-        /* ENH Warn about this. */
-        return 0;
-    }
-
-    /* Init collection from storage. */
-    table = collection_retrieve(msr, real_col_name, col_key, col_key_len);
-
-    if (table == NULL) {
-        /* Does not exist yet - create new. */
-
     if (msr->txcfg->debuglog_level >= 4) {
         msr_log(msr, 4, "Creating collection (name \"%s\", key \"%s\").",
            real_col_name, col_key);
     }
 
     table = apr_table_make(msr->mp, 24);
-        if (table == NULL) return -1;
+    if (table == NULL) {
+        msr_log(msr, 4, "Problems creating collection table. Aborting.");
+        return NULL;
+    }
 
     /* IMP1 Is the timeout hard-coded to 3600? */
 
@@ -2038,6 +2028,32 @@ static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name,
     var->value = "1";
     var->value_len = strlen(var->value);
     apr_table_setn(table, var->name, (void *)var);
+
+    return table;
+}
+
+static apr_status_t init_collection(modsec_rec *msr, const char *real_col_name,
+    const char *col_name, const char *col_key, unsigned int col_key_len)
+{
+    apr_table_t *table = NULL;
+    msc_string *var = NULL;
+
+    if (apr_table_get(msr->collections, col_name) != NULL) {
+        /* ENH Warn about this. */
+        return 0;
+    }
+
+    msr_log(msr, 4, "Initializing collection: %s.", real_col_name);
+
+    if (msr->dcfg1->persistent_storage == STORAGE_TYPE_MEMCACHE) {
+        table = msc_memcache_collection_retrieve(msr, real_col_name, col_key, col_key_len);
+    } else {
+        table = collection_retrieve(msr, real_col_name, col_key, col_key_len);
+    }
+
+    if (table == NULL) {
+        msr_log(msr, 4, "Collection was not found, creating it...");
+        table = create_collection_table(msr, real_col_name, col_name, col_key, col_key_len);
     }
 
     /* Record the original counter value before we change it */

iis/Makefile.win

@@ -70,6 +70,7 @@ OBJS1 = mod_security2.obj apache2_config.obj apache2_io.obj apache2_util.obj \
        msc_parsers.obj msc_util.obj msc_pcre.obj persist_dbm.obj \
        msc_reqbody.obj msc_geo.obj msc_gsb.obj msc_unicode.obj acmp.obj msc_lua.obj \
        msc_release.obj msc_crypt.obj msc_tree.obj \
+       msc_persistent_memcache.obj \
        msc_status_engine.obj \
        msc_json.obj
        

iis/ModSecurityIIS.vcxproj

@@ -265,6 +265,7 @@ net start W3SVC /y</Command>
     <ClCompile Include="..\apache2\msc_lua.c" />
     <ClCompile Include="..\apache2\msc_multipart.c" />
     <ClCompile Include="..\apache2\msc_parsers.c" />
+    <ClCompile Include="..\apache2\msc_persistent_memcache.c" />
     <ClCompile Include="..\apache2\msc_pcre.c" />
     <ClCompile Include="..\apache2\msc_release.c" />
     <ClCompile Include="..\apache2\msc_reqbody.c" />

iis/ModSecurityIIS.vcxproj.filters

@@ -39,6 +39,9 @@
     <ClCompile Include="..\apache2\msc_parsers.c">
       <Filter>ModSecurity</Filter>
     </ClCompile>
+    <ClCompile Include="..\apache2\msc_persistent_memcache.c">
+      <Filter>ModSecurity</Filter>
+    </ClCompile>
     <ClCompile Include="..\apache2\msc_pcre.c">
       <Filter>ModSecurity</Filter>
     </ClCompile>

nginx/modsecurity/config.in

@@ -4,6 +4,7 @@ CFLAGS="$CFLAGS \
     @APR_CFLAGS@ \
     @APU_CFLAGS@ \
     @APXS_CFLAGS@ \
+    @LIBMEMCACHED_CFLAGS@ \
     @LIBXML2_CFLAGS@ \
     @LUA_CFLAGS@ \
     @MODSEC_EXTRA_CFLAGS@ \
@@ -17,6 +18,7 @@ CORE_LIBS="$CORE_LIBS \
     @APU_LINKLD@ \
     @APXS_CFLAGS@ \
     @CURL_LDADD@ \
+    @LIBMEMCACHED_LDADD@ \
     @LIBXML2_LDADD@ \
     @LUA_LDADD@ \
     @PCRE_LDADD@ \

standalone/Makefile.am

@@ -21,6 +21,7 @@ standalone_la_SOURCES = ../apache2/acmp.c \
     ../apache2/msc_multipart.c \
     ../apache2/msc_parsers.c \
     ../apache2/msc_pcre.c \
+    ../apache2/msc_persistent_memcache.c \
     ../apache2/msc_release.c \
     ../apache2/msc_reqbody.c \
     ../apache2/msc_tree.c \

standalone/standalone.vcxproj

@@ -136,6 +136,7 @@
     <ClCompile Include="..\apache2\msc_lua.c" />
     <ClCompile Include="..\apache2\msc_multipart.c" />
     <ClCompile Include="..\apache2\msc_parsers.c" />
+    <ClCompile Include="..\apache2\msc_persistent_memcache.c" />
     <ClCompile Include="..\apache2\msc_pcre.c" />
     <ClCompile Include="..\apache2\msc_release.c" />
     <ClCompile Include="..\apache2\msc_reqbody.c" />

standalone/standalone.vcxproj.filters

@@ -54,6 +54,9 @@
     <ClCompile Include="..\apache2\msc_parsers.c">
       <Filter>ModSecurity Sources</Filter>
     </ClCompile>
+    <ClCompile Include="..\apache2\msc_persistent_memcache.c">
+      <Filter>ModSecurity Sources</Filter>
+    </ClCompile>
     <ClCompile Include="..\apache2\msc_pcre.c">
       <Filter>ModSecurity Sources</Filter>
     </ClCompile>

tests/Makefile.am

@@ -12,6 +12,7 @@ msc_test_SOURCES = msc_test.c \
     ../apache2/msc_logging.c \
     ../apache2/msc_lua.c \
     ../apache2/msc_multipart.c \
+    ../apache2/msc_persistent_memcache.c \
     ../apache2/msc_parsers.c \
     ../apache2/msc_pcre.c \
     ../apache2/msc_release.c \

tests/msc_test.c

@@ -77,6 +77,9 @@ static unsigned char buf[BUFLEN];
 msc_engine *modsecurity = NULL;
 unsigned long int DSOLOCAL msc_pcre_match_limit = 0;
 unsigned long int DSOLOCAL msc_pcre_match_limit_recursion = 0;
+#ifdef WITH_LIBMEMCACHED
+memcached_st *memcache = NULL;
+#endif
 
 /* Stubs */
 char *format_error_log_message(apr_pool_t *mp, error_message_t *em) {