diff --git a/src/collection/backend/lmdb.cc b/src/collection/backend/lmdb.cc index 16a21afe..0142566d 100644 --- a/src/collection/backend/lmdb.cc +++ b/src/collection/backend/lmdb.cc @@ -16,54 +16,499 @@ #include "src/collection/backend/lmdb.h" -#include "lmdb.h" +#include +#include #include #include "modsecurity/collection/variable.h" #include "src/utils.h" +#undef LMDB_STDOUT_COUT + + namespace modsecurity { namespace collection { namespace backend { -LMDB::LMDB() { +LMDB::LMDB() : m_env(NULL) { + mdb_env_create(&m_env); + mdb_env_open(m_env, "./testdb", MDB_WRITEMAP | MDB_NOSUBDIR, 0664); } + LMDB::~LMDB() { + mdb_env_close(m_env); } -void LMDB::store(std::string key, std::string value) { +void LMDB::string2val(const std::string& str, MDB_val *val) { + val->mv_size = sizeof(char)*(str.size()); + val->mv_data = (char *)str.c_str(); +} + +void LMDB::lmdb_debug(int rc, std::string op, std::string scope) { +#ifndef LMDB_STDOUT_COUT + return; +#else + if (rc == 0) { + return; + } + + if (op == "txn") { + std::cout << scope << ", LMDB failure while starting txn: "; + switch (rc) { + case MDB_PANIC: + std::cout << "panic: a fatal error occurred earlier "; + std::cout << "and the environment must be shut down."; + break; + case MDB_MAP_RESIZED: + std::cout << "map resized: another process wrote data "; + std::cout << "beyond this MDB_env's mapsize and this "; + std::cout << "environment's map must be resized as well. "; + std::cout << "See mdb_env_set_mapsize()."; + break; + case MDB_READERS_FULL: + std::cout << "max readers: a read-only transaction was "; + std::cout << "requested and the reader lock table is full. "; + std::cout << "See mdb_env_set_maxreaders()."; + break; + case ENOMEM: + std::cout << "out of memory."; + break; + default: + std::cout << "not sure what is wrong, code: " + + std::to_string(rc); + break; + } + std::cout << std::endl; + } else if (op == "dbi") { + std::cout << scope << ", LMDB failure while opening dbi: "; + switch (rc) { + case MDB_NOTFOUND: + std::cout << "not found: the specified database doesn't "; + std::cout << "exist in the environment and MDB_CREATE was "; + std::cout << "not specified."; + break; + case MDB_DBS_FULL: + std::cout << "full: too many databases have been opened. See "; + std::cout << "mdb_env_set_maxdbs()."; + break; + default: + std::cout << "not sure what is wrong."; + break; + } + std::cout << std::endl; + } else if (op == "get") { + std::cout << scope << ", LMDB failure while getting the key: "; + switch (rc) { + case MDB_NOTFOUND: + std::cout << "not found: the key was not in the database."; + break; + case EINVAL: + std::cout << "an invalid parameter was specified."; + break; + default: + std::cout << "not sure what is wrong."; + break; + } + std::cout << std::endl; + } else if (op == "del") { + td::cout << scope << ", delete procedure failed: "; + switch (rc) { + case EACCES: + std::cout << "an attempt was made to write in a "; + std::cout << "read-only transaction."; + break; + case EINVAL: + std::cout << "an invalid parameter was specified."; + break; + default: + std::cout << "not sure what is wrong. Code: " + + std::to_string(rc); + break; + } + std::cout << std::endl; + } else if (op == "commit") { + std::cout << scope << ", commit procedure failed: "; + switch (rc) { + case EINVAL: + std::cout << "an invalid parameter was specified."; + break; + case ENOSPC: + std::cout << "no more disk space."; + break; + case EIO: + std::cout << "a low-level I/O error occurred while writing."; + break; + case ENOMEM: + std::cout << "out of memory."; + break; + default: + std::cout << "not sure what is wrong. Code: " + + std::to_string(rc); + break; + } + std::cout << std::endl; + } +#endif +} + +std::string* LMDB::resolveFirst(const std::string& var) { + int rc; + MDB_val mdb_key; + MDB_val mdb_value; + MDB_val mdb_value_ret; + std::string *ret = NULL; + MDB_txn *txn = NULL; + MDB_dbi dbi; + + string2val(var, &mdb_key); + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "resolveFirst"); + if (rc != 0) { + goto end_txn; + } + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "resolveFirst"); + if (rc != 0) { + goto end_dbi; + } + rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "get", "resolveFirst"); + if (rc != 0) { + goto end_get; + } + + ret = new std::string( + reinterpret_cast(mdb_value_ret.mv_data), + mdb_value_ret.mv_size); + +end_get: + mdb_dbi_close(m_env, dbi); +end_dbi: + mdb_txn_abort(txn); +end_txn: + return ret; } bool LMDB::storeOrUpdateFirst(const std::string &key, const std::string &value) { + int rc; + MDB_txn *txn; + MDB_dbi dbi; + MDB_val mdb_key; + MDB_val mdb_value; + MDB_val mdb_value_ret; + + string2val(key, &mdb_key); + string2val(value, &mdb_value); + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "storeOrUpdateFirst"); + if (rc != 0) { + goto end_txn; + } + + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "storeOrUpdateFirst"); + if (rc != 0) { + goto end_dbi; + } + + rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "get", "storeOrUpdateFirst"); + if (rc == 0) { + rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "del", "storeOrUpdateFirst"); + if (rc != 0) { + goto end_del; + } + } + + rc = mdb_put(txn, dbi, &mdb_key, &mdb_value, 0); + lmdb_debug(rc, "put", "storeOrUpdateFirst"); + if (rc != 0) { + goto end_put; + } + + rc = mdb_txn_commit(txn); + lmdb_debug(rc, "commit", "storeOrUpdateFirst"); + if (rc != 0) { + goto end_commit; + } + +end_commit: +end_put: +end_del: + mdb_dbi_close(m_env, dbi); +end_dbi: + if (rc != 0) { + mdb_txn_abort(txn); + } +end_txn: return true; } -bool LMDB::updateFirst(const std::string &key, - const std::string &value) { - -} - - -void LMDB::del(const std::string& key) { -} - - void LMDB::resolveSingleMatch(const std::string& var, std::vector *l) { + int rc; + MDB_txn *txn; + MDB_dbi dbi; + MDB_val mdb_key; + MDB_val mdb_value; + MDB_val mdb_value_ret; + MDB_cursor *cursor; + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "resolveSingleMatch"); + if (rc != 0) { + goto end_txn; + } + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "resolveSingleMatch"); + if (rc != 0) { + goto end_dbi; + } + + string2val(var, &mdb_key); + + rc = mdb_cursor_open(txn, dbi, &cursor); + while ((rc = mdb_cursor_get(cursor, &mdb_key, + &mdb_value_ret, MDB_NEXT_DUP)) == 0) { + std::string *a = new std::string( + reinterpret_cast(mdb_value_ret.mv_data), + mdb_value_ret.mv_size); + l->push_back(new Variable(var, *a)); + } + + mdb_cursor_close(cursor); + + mdb_dbi_close(m_env, dbi); +end_dbi: + mdb_txn_abort(txn); +end_txn: + return; +} + + +void LMDB::store(std::string key, std::string value) { + MDB_val mdb_key, mdb_data; + MDB_txn *txn = NULL; + MDB_dbi dbi; + int rc; + MDB_stat mst; + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "store"); + if (rc != 0) { + goto end_txn; + } + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "store"); + if (rc != 0) { + goto end_dbi; + } + + string2val(key, &mdb_key); + string2val(value, &mdb_data); + rc = mdb_put(txn, dbi, &mdb_key, &mdb_data, 0); + lmdb_debug(rc, "put", "store"); + if (rc != 0) { + goto end_put; + } + + rc = mdb_txn_commit(txn); + lmdb_debug(rc, "commit", "store"); + if (rc != 0) { + goto end_commit; + } + +end_commit: +end_put: + mdb_dbi_close(m_env, dbi); +end_dbi: + if (rc != 0) { + mdb_txn_abort(txn); + } +end_txn: + return; +} + + +bool LMDB::updateFirst(const std::string &key, + const std::string &value) { + int rc; + MDB_txn *txn; + MDB_dbi dbi; + MDB_val mdb_key; + MDB_val mdb_value; + MDB_val mdb_value_ret; + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "updateFirst"); + if (rc != 0) { + goto end_txn; + } + + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "updateFirst"); + if (rc != 0) { + goto end_dbi; + } + + string2val(key, &mdb_key); + string2val(value, &mdb_value); + + rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "get", "updateFirst"); + if (rc != 0) { + goto end_get; + } + + rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "del", "updateFirst"); + if (rc != 0) { + goto end_del; + } + + rc = mdb_put(txn, dbi, &mdb_key, &mdb_value, 0); + lmdb_debug(rc, "put", "updateFirst"); + if (rc != 0) { + goto end_put; + } + + rc = mdb_txn_commit(txn); + lmdb_debug(rc, "commit", "updateFirst"); + if (rc != 0) { + goto end_commit; + } + +end_commit: +end_put: +end_del: +end_get: + mdb_dbi_close(m_env, dbi); +end_dbi: + if (rc != 0) { + mdb_txn_abort(txn); + } +end_txn: + + return rc == 0; +} + + +void LMDB::del(const std::string& key) { + int rc; + MDB_txn *txn; + MDB_dbi dbi; + MDB_val mdb_key; + MDB_val mdb_value; + MDB_val mdb_value_ret; + MDB_stat mst; + + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "del"); + if (rc != 0) { + goto end_txn; + } + + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "del"); + if (rc != 0) { + goto end_dbi; + } + + string2val(key, &mdb_key); + + rc = mdb_get(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "get", "del"); + if (rc != 0) { + goto end_get; + } + + rc = mdb_del(txn, dbi, &mdb_key, &mdb_value_ret); + lmdb_debug(rc, "del", "del"); + if (rc != 0) { + goto end_del; + } + + rc = mdb_txn_commit(txn); + lmdb_debug(rc, "commit", "del"); + if (rc != 0) { + goto end_commit; + } + +end_commit: +end_del: +end_get: + mdb_dbi_close(m_env, dbi); +end_dbi: + if (rc != 0) { + mdb_txn_abort(txn); + } +end_txn: + return; } void LMDB::resolveMultiMatches(const std::string& var, std::vector *l) { + MDB_val key, data; + MDB_txn *txn = NULL; + MDB_dbi dbi; + int rc; + MDB_stat mst; + size_t keySize = var.size(); + MDB_cursor *cursor; + rc = mdb_txn_begin(m_env, NULL, 0, &txn); + lmdb_debug(rc, "txn", "resolveMultiMatches"); + if (rc != 0) { + goto end_txn; + } + + rc = mdb_dbi_open(txn, NULL, MDB_CREATE | MDB_DUPSORT, &dbi); + lmdb_debug(rc, "dbi", "resolveMultiMatches"); + if (rc != 0) { + goto end_dbi; + } + + rc = mdb_cursor_open(txn, dbi, &cursor); + lmdb_debug(rc, "cursor_open", "resolveMultiMatches"); + if (rc != 0) { + goto end_cursor_open; + } + + while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) { + if (key.mv_size <= keySize + 1) { + continue; + } + char *a = reinterpret_cast(key.mv_data); + if (a[keySize] != ':') { + continue; + } + if (strncmp(var.c_str(), a, keySize) != 0) { + continue; + } + l->insert(l->begin(), new Variable( + std::string(reinterpret_cast(key.mv_data), key.mv_size), + std::string(reinterpret_cast(data.mv_data), + data.mv_size))); + } + + mdb_cursor_close(cursor); +end_cursor_open: + mdb_dbi_close(m_env, dbi); +end_dbi: + mdb_txn_abort(txn); +end_txn: + return; } @@ -72,10 +517,6 @@ void LMDB::resolveRegularExpression(const std::string& var, } -std::string* LMDB::resolveFirst(const std::string& var) { -} - - } // namespace backend } // namespace collection } // namespace modsecurity diff --git a/src/collection/backend/lmdb.h b/src/collection/backend/lmdb.h index d0e99373..9e734c9c 100644 --- a/src/collection/backend/lmdb.h +++ b/src/collection/backend/lmdb.h @@ -24,6 +24,11 @@ #endif #include +#include +#include +#include +#include +#include #include "modsecurity/collection/variable.h" #include "modsecurity/collection/collection.h" @@ -61,10 +66,10 @@ class LMDB : std::vector *l) override; private: - MDB_env *m_env; - MDB_dbi m_dbi; - MDB_txn *m_txn; + void string2val(const std::string& str, MDB_val *val); + void inline lmdb_debug(int rc, std::string op, std::string scope); + MDB_env *m_env; }; } // namespace backend diff --git a/src/modsecurity.cc b/src/modsecurity.cc index fee58ff9..62b2fdc1 100644 --- a/src/modsecurity.cc +++ b/src/modsecurity.cc @@ -19,6 +19,7 @@ #include "modsecurity/modsecurity.h" #include "modsecurity/rule.h" #include "src/collection/backend/in_memory-per_process.h" +#include "src/collection/backend/lmdb.h" #include "src/config.h" #include "src/unique_id.h" #ifdef MSC_WITH_CURL @@ -46,11 +47,19 @@ namespace modsecurity { */ ModSecurity::ModSecurity() : m_connector(""), +#ifdef WITH_LMDB + m_global_collection(new collection::backend::LMDB()), + m_resource_collection(new collection::backend::LMDB()), + m_ip_collection(new collection::backend::LMDB()), + m_session_collection(new collection::backend::LMDB()), + m_user_collection(new collection::backend::LMDB()), +#else m_global_collection(new collection::backend::InMemoryPerProcess()), m_resource_collection(new collection::backend::InMemoryPerProcess()), m_ip_collection(new collection::backend::InMemoryPerProcess()), m_session_collection(new collection::backend::InMemoryPerProcess()), m_user_collection(new collection::backend::InMemoryPerProcess()), +#endif m_logCb(NULL) { UniqueId::uniqueId(); srand(time(NULL)); diff --git a/test/test-cases/regression/action-initcol.json b/test/test-cases/regression/action-initcol.json index 6edaced1..69948fda 100644 --- a/test/test-cases/regression/action-initcol.json +++ b/test/test-cases/regression/action-initcol.json @@ -4,7 +4,7 @@ "version_min":300000, "title":"Testing initcol action", "expected":{ - "debug_log": "Saving variable: IP:auth_attempt with value: 3" + "debug_log": "Saving variable: IP:auth_attempt with value: " }, "client":{ "ip":"200.249.12.31", diff --git a/test/test-cases/regression/action-setsid.json b/test/test-cases/regression/action-setsid.json index 82c94139..d3ecb255 100644 --- a/test/test-cases/regression/action-setsid.json +++ b/test/test-cases/regression/action-setsid.json @@ -4,7 +4,7 @@ "version_min":300000, "title":"Testing setsid action", "expected":{ - "debug_log": "Saving variable: SESSION:score with value: 5" + "debug_log": "Saving variable: SESSION:score with value: " }, "client":{ "ip":"200.249.12.31", diff --git a/test/test-cases/regression/action-setuid.json b/test/test-cases/regression/action-setuid.json index 52aaacc0..6fba6631 100644 --- a/test/test-cases/regression/action-setuid.json +++ b/test/test-cases/regression/action-setuid.json @@ -4,7 +4,7 @@ "version_min":300000, "title":"Testing setuid action", "expected":{ - "debug_log": "Saving variable: USER:score with value: 5" + "debug_log": "Saving variable: USER:score with value: " }, "client":{ "ip":"200.249.12.31",