Adds support to global collections shared among different process

There is a memory leak in the variable resolution that should be
contained by an internal change in the way that the variables
are resolved.

src/collection/backend/lmdb.cc

@@ -16,54 +16,499 @@
 
 #include "src/collection/backend/lmdb.h"
 
-#include "lmdb.h"
+#include <sys/types.h>
+#include <unistd.h>
 
 #include <string>
 
 #include "modsecurity/collection/variable.h"
 #include "src/utils.h"
 
+#undef LMDB_STDOUT_COUT
+
+
 namespace modsecurity {
 namespace collection {
 namespace backend {
 
 
-LMDB::LMDB() {
+LMDB::LMDB() : m_env(NULL) {
+    mdb_env_create(&m_env);
+    mdb_env_open(m_env, "./testdb", MDB_WRITEMAP | MDB_NOSUBDIR, 0664);
 }
 
+
 LMDB::~LMDB() {
+    mdb_env_close(m_env);
 }
 
-void LMDB::store(std::string key, std::string value) {
 
+void LMDB::string2val(const std::string& str, MDB_val *val) {
+    val->mv_size = sizeof(char)*(str.size());
+    val->mv_data = (char *)str.c_str();
+}
+
+void LMDB::lmdb_debug(int rc, std::string op, std::string scope) {
+#ifndef LMDB_STDOUT_COUT
+    return;
+#else
+    if (rc == 0) {
+        return;
+    }
+
+    if (op == "txn") {
+        std::cout << scope << ", LMDB failure while starting txn: ";
+        switch (rc) {
+            case MDB_PANIC:
+                std::cout << "panic: a fatal error occurred earlier ";
+                std::cout << "and the environment must be shut down.";
+                break;
+            case MDB_MAP_RESIZED:
+                std::cout << "map resized: another process wrote data ";
+                std::cout << "beyond this MDB_env's mapsize and this ";
+                std::cout << "environment's map must be resized as well. ";
+                std::cout << "See mdb_env_set_mapsize().";
+                break;
+            case MDB_READERS_FULL:
+                std::cout << "max readers: a read-only transaction was ";
+                std::cout << "requested and the reader lock table is full. ";
+                std::cout << "See mdb_env_set_maxreaders().";
+                break;
+            case ENOMEM:
+                std::cout << "out of memory.";
+                break;
+            default:
+                std::cout << "not sure what is wrong, code: " +
+                    std::to_string(rc);
+                break;
+        }
+        std::cout << std::endl;
+    } else if (op == "dbi") {
+        std::cout << scope << ", LMDB failure while opening dbi: ";
+        switch (rc) {
+            case MDB_NOTFOUND:
+                std::cout << "not found: the specified database doesn't ";
+                std::cout << "exist in the environment and MDB_CREATE was ";
+                std::cout << "not specified.";
+                break;
+            case MDB_DBS_FULL:
+                std::cout << "full: too many databases have been opened. See ";
+                std::cout << "mdb_env_set_maxdbs().";
+                break;
+            default:
+                std::cout << "not sure what is wrong.";
+                break;
+        }
+        std::cout << std::endl;
+    } else if (op == "get") {
+        std::cout << scope << ", LMDB failure while getting the key: ";
+        switch (rc) {
+            case MDB_NOTFOUND:
+                std::cout << "not found: the key was not in the database.";
+                break;
+            case EINVAL:
+                std::cout << "an invalid parameter was specified.";
+                break;
+            default:
+                std::cout << "not sure what is wrong.";
+                break;
+        }
+        std::cout << std::endl;
+    } else if (op == "del") {
+        td::cout << scope << ", delete procedure failed: ";
+        switch (rc) {
+            case EACCES:
+                std::cout << "an attempt was made to write in a ";
+                std::cout << "read-only transaction.";
+                break;
+            case EINVAL:
+                std::cout << "an invalid parameter was specified.";
+                break;
+            default:
+                std::cout << "not sure what is wrong. Code: " +
+                std::to_string(rc);
+                break;
+        }
+        std::cout << std::endl;
+    } else if (op == "commit") {
+        std::cout << scope << ", commit procedure failed: ";
+        switch (rc) {
+            case EINVAL:
+                std::cout << "an invalid parameter was specified.";
+                break;
+            case ENOSPC:
+                std::cout << "no more disk space.";
+                break;
+            case EIO:
+                std::cout << "a low-level I/O error occurred while writing.";
+                break;
+            case ENOMEM:
+                std::cout << "out of memory.";
+                break;
+            default:
+                std::cout << "not sure what is wrong. Code: " +
+                    std::to_string(rc);
+                break;
+        }
+        std::cout << std::endl;
+    }
+#endif
+}
+
+std::string* LMDB::resolveFirst(const std::string& var) {
+    int rc;
+    MDB_val mdb_key;
+    MDB_val mdb_value;
+    MDB_val mdb_value_ret;
+    std::string *ret = NULL;
+    MDB_txn *txn = NULL;
+    MDB_dbi dbi;
+
+    string2val(var, &mdb_key);
+
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "resolveFirst");
+    if (rc != 0) {
+        goto end_txn;
+    }
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "resolveFirst");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+    rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "get", "resolveFirst");
+    if (rc != 0) {
+        goto end_get;
+    }
+
+    ret = new std::string(
+        reinterpret_cast<char *>(mdb_value_ret.mv_data),
+        mdb_value_ret.mv_size);
+
+end_get:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    mdb_txn_abort(txn);
+end_txn:
+    return ret;
 }
 
 
 bool LMDB::storeOrUpdateFirst(const std::string &key,
     const std::string &value) {
+    int rc;
+    MDB_txn *txn;
+    MDB_dbi dbi;
+    MDB_val mdb_key;
+    MDB_val mdb_value;
+    MDB_val mdb_value_ret;
+
+    string2val(key, &mdb_key);
+    string2val(value, &mdb_value);
+
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "storeOrUpdateFirst");
+    if (rc != 0) {
+        goto end_txn;
+    }
+
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "storeOrUpdateFirst");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "get", "storeOrUpdateFirst");
+    if (rc == 0) {
+        rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret);
+        lmdb_debug(rc, "del", "storeOrUpdateFirst");
+        if (rc != 0) {
+            goto end_del;
+        }
+    }
+
+    rc = mdb_put(txn, dbi, &mdb_key, &mdb_value, 0);
+    lmdb_debug(rc, "put", "storeOrUpdateFirst");
+    if (rc != 0) {
+        goto end_put;
+    }
+
+    rc = mdb_txn_commit(txn);
+    lmdb_debug(rc, "commit", "storeOrUpdateFirst");
+    if (rc != 0) {
+        goto end_commit;
+    }
+
+end_commit:
+end_put:
+end_del:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    if (rc != 0) {
+        mdb_txn_abort(txn);
+    }
+end_txn:
     return true;
 }
 
 
-bool LMDB::updateFirst(const std::string &key,
-    const std::string &value) {
-
-}
-
-
-void LMDB::del(const std::string& key) {
-}
-
-
 void LMDB::resolveSingleMatch(const std::string& var,
     std::vector<const Variable *> *l) {
+    int rc;
+    MDB_txn *txn;
+    MDB_dbi dbi;
+    MDB_val mdb_key;
+    MDB_val mdb_value;
+    MDB_val mdb_value_ret;
+    MDB_cursor *cursor;
 
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "resolveSingleMatch");
+    if (rc != 0) {
+        goto end_txn;
+    }
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "resolveSingleMatch");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    string2val(var, &mdb_key);
+
+    rc = mdb_cursor_open(txn, dbi, &cursor);
+    while ((rc = mdb_cursor_get(cursor, &mdb_key,
+            &mdb_value_ret, MDB_NEXT_DUP)) == 0) {
+        std::string *a = new std::string(
+            reinterpret_cast<char *>(mdb_value_ret.mv_data),
+            mdb_value_ret.mv_size);
+        l->push_back(new Variable(var, *a));
+    }
+
+    mdb_cursor_close(cursor);
+
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    mdb_txn_abort(txn);
+end_txn:
+    return;
+}
+
+
+void LMDB::store(std::string key, std::string value) {
+    MDB_val mdb_key, mdb_data;
+    MDB_txn *txn = NULL;
+    MDB_dbi dbi;
+    int rc;
+    MDB_stat mst;
+
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "store");
+    if (rc != 0) {
+        goto end_txn;
+    }
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "store");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    string2val(key, &mdb_key);
+    string2val(value, &mdb_data);
+    rc = mdb_put(txn, dbi, &mdb_key, &mdb_data, 0);
+    lmdb_debug(rc, "put", "store");
+    if (rc != 0) {
+        goto end_put;
+    }
+
+    rc = mdb_txn_commit(txn);
+    lmdb_debug(rc, "commit", "store");
+    if (rc != 0) {
+        goto end_commit;
+    }
+
+end_commit:
+end_put:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    if (rc != 0) {
+        mdb_txn_abort(txn);
+    }
+end_txn:
+    return;
+}
+
+
+bool LMDB::updateFirst(const std::string &key,
+    const std::string &value) {
+    int rc;
+    MDB_txn *txn;
+    MDB_dbi dbi;
+    MDB_val mdb_key;
+    MDB_val mdb_value;
+    MDB_val mdb_value_ret;
+
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "updateFirst");
+    if (rc != 0) {
+        goto end_txn;
+    }
+
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "updateFirst");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    string2val(key, &mdb_key);
+    string2val(value, &mdb_value);
+
+    rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "get", "updateFirst");
+    if (rc != 0) {
+        goto end_get;
+    }
+
+    rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "del", "updateFirst");
+    if (rc != 0) {
+        goto end_del;
+    }
+
+    rc = mdb_put(txn, dbi, &mdb_key, &mdb_value, 0);
+    lmdb_debug(rc, "put", "updateFirst");
+    if (rc != 0) {
+        goto end_put;
+    }
+
+    rc = mdb_txn_commit(txn);
+    lmdb_debug(rc, "commit", "updateFirst");
+    if (rc != 0) {
+        goto end_commit;
+    }
+
+end_commit:
+end_put:
+end_del:
+end_get:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    if (rc != 0) {
+        mdb_txn_abort(txn);
+    }
+end_txn:
+
+    return rc == 0;
+}
+
+
+void LMDB::del(const std::string& key) {
+    int rc;
+    MDB_txn *txn;
+    MDB_dbi dbi;
+    MDB_val mdb_key;
+    MDB_val mdb_value;
+    MDB_val mdb_value_ret;
+    MDB_stat mst;
+
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "del");
+    if (rc != 0) {
+        goto end_txn;
+    }
+
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "del");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    string2val(key, &mdb_key);
+
+    rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "get", "del");
+    if (rc != 0) {
+        goto end_get;
+    }
+
+    rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret);
+    lmdb_debug(rc, "del", "del");
+    if (rc != 0) {
+        goto end_del;
+    }
+
+    rc = mdb_txn_commit(txn);
+    lmdb_debug(rc, "commit", "del");
+    if (rc != 0) {
+        goto end_commit;
+    }
+
+end_commit:
+end_del:
+end_get:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    if (rc != 0) {
+        mdb_txn_abort(txn);
+    }
+end_txn:
+    return;
 }
 
 
 void LMDB::resolveMultiMatches(const std::string& var,
     std::vector<const Variable *> *l) {
+    MDB_val key, data;
+    MDB_txn *txn = NULL;
+    MDB_dbi dbi;
+    int rc;
+    MDB_stat mst;
+    size_t keySize = var.size();
+    MDB_cursor *cursor;
 
+    rc = mdb_txn_begin(m_env, NULL, 0, &txn);
+    lmdb_debug(rc, "txn", "resolveMultiMatches");
+    if (rc != 0) {
+        goto end_txn;
+    }
+
+    rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi);
+    lmdb_debug(rc, "dbi", "resolveMultiMatches");
+    if (rc != 0) {
+        goto end_dbi;
+    }
+
+    rc = mdb_cursor_open(txn, dbi, &cursor);
+    lmdb_debug(rc, "cursor_open", "resolveMultiMatches");
+    if (rc != 0) {
+        goto end_cursor_open;
+    }
+
+    while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
+        if (key.mv_size <= keySize + 1) {
+            continue;
+        }
+        char *a = reinterpret_cast<char *>(key.mv_data);
+        if (a[keySize] != ':') {
+            continue;
+        }
+        if (strncmp(var.c_str(), a, keySize) != 0) {
+            continue;
+        }
+        l->insert(l->begin(), new Variable(
+            std::string(reinterpret_cast<char *>(key.mv_data), key.mv_size),
+            std::string(reinterpret_cast<char *>(data.mv_data),
+                        data.mv_size)));
+    }
+
+    mdb_cursor_close(cursor);
+end_cursor_open:
+    mdb_dbi_close(m_env, dbi);
+end_dbi:
+    mdb_txn_abort(txn);
+end_txn:
+    return;
 }
 
 
@@ -72,10 +517,6 @@ void LMDB::resolveRegularExpression(const std::string& var,
 }
 
 
-std::string* LMDB::resolveFirst(const std::string& var) {
-}
-
-
 }  // namespace backend
 }  // namespace collection
 }  // namespace modsecurity

src/collection/backend/lmdb.h

@@ -24,6 +24,11 @@
 #endif
 
 #include <lmdb.h>
+#include <semaphore.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <pthread.h>
 
 #include "modsecurity/collection/variable.h"
 #include "modsecurity/collection/collection.h"
@@ -61,10 +66,10 @@ class LMDB :
         std::vector<const Variable *> *l) override;
 
  private:
-    MDB_env *m_env;
-    MDB_dbi m_dbi;
-    MDB_txn *m_txn;
+    void string2val(const std::string& str, MDB_val *val);
+    void inline lmdb_debug(int rc, std::string op, std::string scope);
 
+    MDB_env *m_env;
 };
 
 }  // namespace backend

src/modsecurity.cc

@@ -19,6 +19,7 @@
 #include "modsecurity/modsecurity.h"
 #include "modsecurity/rule.h"
 #include "src/collection/backend/in_memory-per_process.h"
+#include "src/collection/backend/lmdb.h"
 #include "src/config.h"
 #include "src/unique_id.h"
 #ifdef MSC_WITH_CURL
@@ -46,11 +47,19 @@ namespace modsecurity {
  */
 ModSecurity::ModSecurity()
     : m_connector(""),
+#ifdef WITH_LMDB
+    m_global_collection(new collection::backend::LMDB()),
+    m_resource_collection(new collection::backend::LMDB()),
+    m_ip_collection(new collection::backend::LMDB()),
+    m_session_collection(new collection::backend::LMDB()),
+    m_user_collection(new collection::backend::LMDB()),
+#else
     m_global_collection(new collection::backend::InMemoryPerProcess()),
     m_resource_collection(new collection::backend::InMemoryPerProcess()),
     m_ip_collection(new collection::backend::InMemoryPerProcess()),
     m_session_collection(new collection::backend::InMemoryPerProcess()),
     m_user_collection(new collection::backend::InMemoryPerProcess()),
+#endif
     m_logCb(NULL) {
     UniqueId::uniqueId();
     srand(time(NULL));

test/test-cases/regression/action-initcol.json

@@ -4,7 +4,7 @@
     "version_min":300000,
     "title":"Testing initcol action",
     "expected":{
-      "debug_log": "Saving variable: IP:auth_attempt with value: 3"
+      "debug_log": "Saving variable: IP:auth_attempt with value: "
     },
     "client":{
       "ip":"200.249.12.31",

test/test-cases/regression/action-setsid.json

@@ -4,7 +4,7 @@
     "version_min":300000,
     "title":"Testing setsid action",
     "expected":{
-      "debug_log": "Saving variable: SESSION:score with value: 5"
+      "debug_log": "Saving variable: SESSION:score with value: "
     },
     "client":{
       "ip":"200.249.12.31",

test/test-cases/regression/action-setuid.json

@@ -4,7 +4,7 @@
     "version_min":300000,
     "title":"Testing setuid action",
     "expected":{
-      "debug_log": "Saving variable: USER:score with value: 5"
+      "debug_log": "Saving variable: USER:score with value: "
     },
     "client":{
       "ip":"200.249.12.31",