Adds first version of `processContentOffset'

This commit also includes an example application on how to use the
`processContentOffset' method.

configure.ac

@@ -316,9 +316,12 @@ AM_COND_IF([TEST_UTILITIES],
        [AC_CONFIG_FILES([test/Makefile test/benchmark/Makefile])])
 
 AM_COND_IF([EXAMPLES],
-       [AC_CONFIG_FILES([examples/Makefile \
+       [AC_CONFIG_FILES([ \
+            examples/Makefile \
             examples/simple_example_using_c/Makefile \
-            examples/multiprocess_c/Makefile])])
+            examples/multiprocess_c/Makefile \
+            examples/reading_logs_with_offset/Makefile \
+            ])])
 
 AM_COND_IF([AFL_FUZZER],
        [AC_CONFIG_FILES([test/fuzzer/Makefile])])

examples/Makefile.am

@@ -4,7 +4,8 @@ ACLOCAL_AMFLAGS = -I build
 
 SUBDIRS = \
 	simple_example_using_c \
-	multiprocess_c
+	multiprocess_c \
+	reading_logs_with_offset
 
 # make clean
 CLEANFILES = 

examples/reading_logs_with_offset/Makefile.am

@@ -0,0 +1,40 @@
+
+
+noinst_PROGRAMS = read
+
+read_SOURCES = \
+	read.cc
+
+read_LDADD = \
+	$(top_builddir)/src/.libs/libmodsecurity.a \
+	$(CURL_LDADD) \
+	$(GEOIP_LDFLAGS) $(GEOIP_LDADD) \
+	$(PCRE_LDADD) \
+	$(YAJL_LDFLAGS) $(YAJL_LDADD) \
+	$(LMDB_LDFLAGS) $(LMDB_LDADD) \
+	$(LIBXML2_LDADD) \
+	$(GLOBAL_LDADD)
+
+
+read_CPPFLAGS = \
+	$(GLOBAL_CFLAGS) \
+	-std=c++11 \
+	-I$(top_builddir)/headers \
+	-I$(top_builddir) \
+	-g \
+	-I../others \
+	-fPIC \
+	-O3 \
+	$(GEOIP_CFLAGS) \
+	$(GLOBAL_CPPFLAGS) \
+	$(MODSEC_NO_LOGS) \
+	$(YAJL_CFLAGS) \
+	$(LMDB_CFLAGS) \
+	$(PCRE_CFLAGS) \
+	$(LIBXML2_CFLAGS)
+
+
+MAINTAINERCLEANFILES = \
+	Makefile.in
+
+

examples/reading_logs_with_offset/read.cc

@@ -0,0 +1,34 @@
+
+#include <stdio.h>
+#include <string.h>
+
+#include <modsecurity/modsecurity.h>
+
+
+// Variable offset - REQUEST_HEADERS_NAMES
+
+const char *request = "" \
+    "GET /index.html?param1=value1&param2=value1&param3=value1 HTTP/\n" \
+    "AuThOrIzAtIoN: Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ==\n" \
+    "Host: localhost\n" \
+    "Content-Length: 27\n" \
+    "Content-Type: application/x-www-form-urlencoded\n";
+
+
+int main() {
+    modsecurity::ModSecurity msc;
+    std::string json("");
+    const char *err = NULL;
+    int ret = 0;
+
+    ret = msc.processContentOffset(request, strlen(request),
+        "o0,4v64,13v114,4v130,14v149,12t:lowercase", &json, &err);
+
+    if (ret >= 0) {
+        std::cout << json << std::endl;
+    } else {
+        std::cout << err << std::endl;
+    }
+
+    return ret;
+}

examples/simple_example_using_c/test.c

@@ -13,7 +13,7 @@
  *
  */
 
-#include <modsecurity/transaction.h>
+
 #include <stdio.h>
 #include <stdlib.h>
 

headers/modsecurity/modsecurity.h

@@ -221,6 +221,9 @@ class ModSecurity {
     void serverLog(void *data, const std::string& msg);
     const std::string& getConnectorInformation();
 
+    int processContentOffset(const char *content, size_t len,
+        const char *matchString, std::string *json, const char **err);
+
     collection::Collection *m_global_collection;
     collection::Collection *m_resource_collection;
     collection::Collection *m_ip_collection;

src/actions/transformations/lower_case.cc

@@ -35,18 +35,10 @@ std::string LowerCase::evaluate(std::string value,
     Transaction *transaction) {
     std::locale loc;
 
-    if (LowerCaseInstantCache::getInstance().count(value) > 0) {
-        return LowerCaseInstantCache::getInstance().at(value);
-    }
-
-    std::string orig_value = value;
-
     for (std::string::size_type i=0; i < value.length(); ++i) {
         value[i] = std::tolower(value[i], loc);
     }
 
-    LowerCaseInstantCache::getInstance().cache(orig_value, value);
-
     return value;
 }
 

src/actions/transformations/transformation.cc

@@ -58,6 +58,11 @@
 #include "src/actions/transformations/url_encode.h"
 #include "src/actions/transformations/utf8_to_unicode.h"
 
+
+#define IF_MATCH(b) \
+    if (a.compare(2, std::strlen(#b), #b) == 0)
+
+
 namespace modsecurity {
 namespace actions {
 namespace transformations {
@@ -68,6 +73,49 @@ std::string Transformation::evaluate(std::string value,
     return value;
 }
 
+Transformation* Transformation::instantiate(std::string a) {
+    IF_MATCH(base64DecodeExt) { return new Base64DecodeExt(a); }
+    IF_MATCH(base64Decode) { return new Base64Decode(a); }
+    IF_MATCH(base64Encode) { return new Base64Encode(a); }
+    IF_MATCH(cmd_line) { return new CmdLine(a); }
+    IF_MATCH(compress_whitespace) { return new CompressWhitespace(a); }
+    IF_MATCH(cssDecode) { return new CssDecode(a); }
+    IF_MATCH(escapeSeqDecode) { return new EscapeSeqDecode(a); }
+    IF_MATCH(hexDecode) { return new HexDecode(a); }
+    IF_MATCH(hexEncode) { return new HexEncode(a); }
+    IF_MATCH(htmlEntityDecode) { return new HtmlEntityDecode(a); }
+    IF_MATCH(jsDecode) { return new JsDecode(a); }
+    IF_MATCH(length) { return new Length(a); }
+    IF_MATCH(lowercase) { return new LowerCase(a); }
+    IF_MATCH(md5) { return new Md5(a); }
+    IF_MATCH(none) { return new None(a); }
+    IF_MATCH(normalizePathWin) { return new NormalisePathWin(a); }
+    IF_MATCH(normalisePathWin) { return new NormalisePathWin(a); }
+    IF_MATCH(normalizePath) { return new NormalisePath(a); }
+    IF_MATCH(normalisePath) { return new NormalisePath(a); }
+    IF_MATCH(parityEven7bit) { return new ParityEven7bit(a); }
+    IF_MATCH(parityOdd7bit) { return new ParityOdd7bit(a); }
+    IF_MATCH(parityZero7bit) { return new ParityZero7bit(a); }
+    IF_MATCH(removeCommentsChar) { return new RemoveCommentsChar(a); }
+    IF_MATCH(removeComments) { return new RemoveComments(a); }
+    IF_MATCH(removeNulls) { return new RemoveNulls(a); }
+    IF_MATCH(removeWhitespace) { return new RemoveWhitespace(a); }
+    IF_MATCH(compressWhitespace) { return new CompressWhitespace(a); }
+    IF_MATCH(replaceComments) { return new ReplaceComments(a); }
+    IF_MATCH(replaceNulls) { return new ReplaceNulls(a); }
+    IF_MATCH(sha1) { return new Sha1(a); }
+    IF_MATCH(sqlHexDecode) { return new SqlHexDecode(a); }
+    IF_MATCH(transformation) { return new Transformation(a); }
+    IF_MATCH(trimLeft) { return new TrimLeft(a); }
+    IF_MATCH(trimRight) { return new TrimRight(a); }
+    IF_MATCH(trim) { return new Trim(a); }
+    IF_MATCH(urlDecodeUni) { return new UrlDecodeUni(a); }
+    IF_MATCH(urlDecode) { return new UrlDecode(a); }
+    IF_MATCH(urlEncode) { return new UrlEncode(a); }
+    IF_MATCH(utf8ToUnicode) { return new Utf8ToUnicode(a); }
+
+    return new Transformation(a);
+}
 
 }  // namespace transformations
 }  // namespace actions

src/actions/transformations/transformation.h

@@ -20,6 +20,7 @@
 #ifndef SRC_ACTIONS_TRANSFORMATIONS_TRANSFORMATION_H_
 #define SRC_ACTIONS_TRANSFORMATIONS_TRANSFORMATION_H_
 
+
 namespace modsecurity {
 class Transaction;
 
@@ -36,6 +37,9 @@ class Transformation : public Action {
 
     std::string evaluate(std::string exp,
         Transaction *transaction) override;
+
+    static Transformation* instantiate(std::string a);
+
 };
 
 }  // namespace transformations

src/modsecurity.cc

@@ -22,11 +22,17 @@
 #include "src/collection/backend/lmdb.h"
 #include "src/config.h"
 #include "src/unique_id.h"
+#include "src/utils/regex.h"
 #ifdef MSC_WITH_CURL
 #include <curl/curl.h>
 #endif
+#ifdef WITH_YAJL
+#include <yajl/yajl_tree.h>
+#include <yajl/yajl_gen.h>
+#endif
 
 #include "src/utils/geo_lookup.h"
+#include "src/actions/transformations/transformation.h"
 
 namespace modsecurity {
 
@@ -170,6 +176,185 @@ void ModSecurity::serverLog(void *data, const std::string& msg) {
 }
 
 
+int ModSecurity::processContentOffset(const char *content, size_t len,
+    const char *matchString, std::string *json, const char **err) {
+#ifdef WITH_YAJL
+    Utils::Regex variables("v([0-9]+),([0-9]+)");
+    Utils::Regex operators("o([0-9]+),([0-9]+)");
+    Utils::Regex transformations("t:(?:(?!t:).)+");
+    int i;
+    yajl_gen g;
+    std::string varValue;
+    std::string opValue;
+    const unsigned char *buf;
+    size_t jsonSize;
+
+    std::list<Utils::SMatch> vars = variables.searchAll(matchString);
+    std::list<Utils::SMatch> ops = operators.searchAll(matchString);
+    std::list<Utils::SMatch> trans = transformations.searchAll(matchString);
+
+    g = yajl_gen_alloc(NULL);
+    if (g == NULL) {
+        *err = "Failed to allocate memory for the JSON creation.";
+        return -1;
+    }
+
+    yajl_gen_config(g, yajl_gen_beautify, 1);
+
+    yajl_gen_map_open(g);
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>("match"),
+        strlen("match"));
+
+    yajl_gen_array_open(g);
+    yajl_gen_map_open(g);
+
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>("variable"),
+            strlen("variable"));
+
+    yajl_gen_map_open(g);
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>("highlight"),
+            strlen("highlight"));
+
+        yajl_gen_array_open(g);
+    while (vars.size() > 0) {
+        std::string value;
+        yajl_gen_map_open(g);
+        vars.pop_back();
+        std::string startingAt = vars.back().match;
+        vars.pop_back();
+        std::string size = vars.back().match;
+        vars.pop_back();
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>("startingAt"),
+            strlen("startingAt"));
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(startingAt.c_str()),
+            startingAt.size());
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>("size"),
+            strlen("size"));
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(size.c_str()),
+            size.size());
+        yajl_gen_map_close(g);
+
+        if (stoi(startingAt) >= len) {
+            *err = "Offset is out of the content limits.";
+            return -1;
+        }
+
+        value = std::string(content, stoi(startingAt), stoi(size));
+        if (varValue.size() > 0) {
+            varValue.append(" " + value);
+        } else {
+            varValue.append(value);
+        }
+    }
+    yajl_gen_array_close(g);
+
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>("value"),
+            strlen("value"));
+
+    yajl_gen_array_open(g);
+
+    yajl_gen_map_open(g);
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>("value"),
+            strlen("value"));
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>(varValue.c_str()),
+            varValue.size());
+    yajl_gen_map_close(g);
+
+    while (trans.size() > 0) {
+        modsecurity::actions::transformations::Transformation *t;
+        std::string varValueRes;
+        yajl_gen_map_open(g);
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>("transformation"),
+            strlen("transformation"));
+
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(trans.back().match.c_str()),
+            trans.back().match.size());
+
+        t = modsecurity::actions::transformations::Transformation::instantiate(trans.back().match.c_str());
+        varValueRes = t->evaluate(varValue, NULL);
+        varValue.assign(varValueRes);
+        trans.pop_back();
+
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>("value"),
+            strlen("value"));
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>(varValue.c_str()),
+            varValue.size());
+        yajl_gen_map_close(g);
+    }
+
+    yajl_gen_array_close(g);
+
+    yajl_gen_string(g, reinterpret_cast<const unsigned char*>("operator"),
+            strlen("operator"));
+
+    yajl_gen_map_open(g);
+
+    while (ops.size() > 0) {
+        std::string value;
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>("highlight"),
+            strlen("highlight"));
+        yajl_gen_map_open(g);
+        ops.pop_back();
+        std::string startingAt = ops.back().match;
+        ops.pop_back();
+        std::string size = ops.back().match;
+        ops.pop_back();
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>("startingAt"),
+            strlen("startingAt"));
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(startingAt.c_str()),
+            startingAt.size());
+        yajl_gen_string(g, reinterpret_cast<const unsigned char*>("size"),
+            strlen("size"));
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(size.c_str()),
+            size.size());
+        yajl_gen_map_close(g);
+
+        if (stoi(startingAt) >= varValue.size()) {
+            *err = "Offset is out of the variable limits.";
+            return -1;
+        }
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>("value"),
+            strlen("value"));
+
+        value = std::string(varValue, stoi(startingAt), stoi(size));
+
+        yajl_gen_string(g,
+            reinterpret_cast<const unsigned char*>(value.c_str()),
+            value.size());
+    }
+
+    yajl_gen_map_close(g);
+
+
+    yajl_gen_map_close(g);
+    yajl_gen_array_close(g);
+
+    yajl_gen_map_close(g);
+    yajl_gen_array_close(g);
+    yajl_gen_map_close(g);
+
+    yajl_gen_get_buf(g, &buf, &jsonSize);
+
+    json->assign(reinterpret_cast<const char*>(buf), jsonSize);
+
+    yajl_gen_free(g);
+#else
+    *err = "Without YAJL support, we cannot generate JSON.";
+    return -1;
+#endif
+    return 0;
+}
+
+
 void ModSecurity::setServerLogCb(LogCb cb) {
     m_logCb = (LogCb) cb;
 }

test/unit/unit.cc

@@ -27,41 +27,7 @@
 #include "src/actions/transformations/transformation.h"
 #include "modsecurity/transaction.h"
 #include "modsecurity/actions/action.h"
-#include "src/actions/transformations/base64_decode_ext.h"
-#include "src/actions/transformations/base64_decode.h"
-#include "src/actions/transformations/base64_encode.h"
-#include "src/actions/transformations/cmd_line.h"
-#include "src/actions/transformations/compress_whitespace.h"
-#include "src/actions/transformations/css_decode.h"
-#include "src/actions/transformations/escape_seq_decode.h"
-#include "src/actions/transformations/hex_decode.h"
-#include "src/actions/transformations/hex_encode.h"
-#include "src/actions/transformations/html_entity_decode.h"
-#include "src/actions/transformations/js_decode.h"
-#include "src/actions/transformations/length.h"
-#include "src/actions/transformations/lower_case.h"
-#include "src/actions/transformations/md5.h"
-#include "src/actions/transformations/none.h"
-#include "src/actions/transformations/normalise_path.h"
-#include "src/actions/transformations/normalise_path_win.h"
-#include "src/actions/transformations/parity_even_7bit.h"
-#include "src/actions/transformations/parity_odd_7bit.h"
-#include "src/actions/transformations/parity_zero_7bit.h"
-#include "src/actions/transformations/remove_comments_char.h"
-#include "src/actions/transformations/remove_comments.h"
-#include "src/actions/transformations/remove_nulls.h"
-#include "src/actions/transformations/remove_whitespace.h"
-#include "src/actions/transformations/replace_comments.h"
-#include "src/actions/transformations/replace_nulls.h"
-#include "src/actions/transformations/sha1.h"
-#include "src/actions/transformations/sql_hex_decode.h"
-#include "src/actions/transformations/trim.h"
-#include "src/actions/transformations/trim_left.h"
-#include "src/actions/transformations/trim_right.h"
-#include "src/actions/transformations/url_decode.h"
-#include "src/actions/transformations/url_decode_uni.h"
-#include "src/actions/transformations/url_encode.h"
-#include "src/actions/transformations/utf8_to_unicode.h"
+#include "src/actions/transformations/transformation.h"
 
 
 #include "test/common/modsecurity_test.h"
@@ -70,8 +36,7 @@
 #include "test/unit/unit_test.h"
 #include "src/utils/string.h"
 
-#define IF_MATCH(b) \
-    if (a.compare(2, std::strlen(#b), #b) == 0)
+
 
 
 using modsecurity_test::UnitTest;
@@ -91,51 +56,6 @@ void print_help() {
 }
 
 
-Transformation* t_instantiate(std::string a) {
-    IF_MATCH(base64DecodeExt) { return new Base64DecodeExt(a); }
-    IF_MATCH(base64Decode) { return new Base64Decode(a); }
-    IF_MATCH(base64Encode) { return new Base64Encode(a); }
-    IF_MATCH(cmd_line) { return new CmdLine(a); }
-    IF_MATCH(compress_whitespace) { return new CompressWhitespace(a); }
-    IF_MATCH(cssDecode) { return new CssDecode(a); }
-    IF_MATCH(escapeSeqDecode) { return new EscapeSeqDecode(a); }
-    IF_MATCH(hexDecode) { return new HexDecode(a); }
-    IF_MATCH(hexEncode) { return new HexEncode(a); }
-    IF_MATCH(htmlEntityDecode) { return new HtmlEntityDecode(a); }
-    IF_MATCH(jsDecode) { return new JsDecode(a); }
-    IF_MATCH(length) { return new Length(a); }
-    IF_MATCH(lowercase) { return new LowerCase(a); }
-    IF_MATCH(md5) { return new Md5(a); }
-    IF_MATCH(none) { return new None(a); }
-    IF_MATCH(normalizePathWin) { return new NormalisePathWin(a); }
-    IF_MATCH(normalisePathWin) { return new NormalisePathWin(a); }
-    IF_MATCH(normalizePath) { return new NormalisePath(a); }
-    IF_MATCH(normalisePath) { return new NormalisePath(a); }
-    IF_MATCH(parityEven7bit) { return new ParityEven7bit(a); }
-    IF_MATCH(parityOdd7bit) { return new ParityOdd7bit(a); }
-    IF_MATCH(parityZero7bit) { return new ParityZero7bit(a); }
-    IF_MATCH(removeCommentsChar) { return new RemoveCommentsChar(a); }
-    IF_MATCH(removeComments) { return new RemoveComments(a); }
-    IF_MATCH(removeNulls) { return new RemoveNulls(a); }
-    IF_MATCH(removeWhitespace) { return new RemoveWhitespace(a); }
-    IF_MATCH(compressWhitespace) { return new CompressWhitespace(a); }
-    IF_MATCH(replaceComments) { return new ReplaceComments(a); }
-    IF_MATCH(replaceNulls) { return new ReplaceNulls(a); }
-    IF_MATCH(sha1) { return new Sha1(a); }
-    IF_MATCH(sqlHexDecode) { return new SqlHexDecode(a); }
-    IF_MATCH(transformation) { return new Transformation(a); }
-    IF_MATCH(trimLeft) { return new TrimLeft(a); }
-    IF_MATCH(trimRight) { return new TrimRight(a); }
-    IF_MATCH(trim) { return new Trim(a); }
-    IF_MATCH(urlDecodeUni) { return new UrlDecodeUni(a); }
-    IF_MATCH(urlDecode) { return new UrlDecode(a); }
-    IF_MATCH(urlEncode) { return new UrlEncode(a); }
-    IF_MATCH(utf8ToUnicode) { return new Utf8ToUnicode(a); }
-
-    return new Transformation(a);
-}
-
-
 void perform_unit_test(ModSecurityTest<UnitTest> *test, UnitTest *t,
     ModSecurityTestResults<UnitTest>* res) {
     std::string error;
@@ -173,7 +93,7 @@ void perform_unit_test(ModSecurityTest<UnitTest> *test, UnitTest *t,
         }
         delete op;
     } else if (t->type == "tfn") {
-        Transformation *tfn = t_instantiate("t:" + t->name);
+        Transformation *tfn = Transformation::instantiate("t:" + t->name);
         std::string ret = tfn->evaluate(t->input, NULL);
         t->obtained = 1;
         t->obtainedOutput = ret;