Adds support to load remote rules

build/curl.m4

@@ -0,0 +1,109 @@
+dnl Check for CURL Libraries
+dnl CHECK_CURL(ACTION-IF-FOUND [, ACTION-IF-NOT-FOUND])
+dnl Sets:
+dnl  CURL_CFLAGS
+dnl  CURL_LIBS
+
+CURL_CONFIG=""
+CURL_VERSION=""
+CURL_CPPFLAGS=""
+CURL_CFLAGS=""
+CURL_LDFLAGS=""
+CURL_LDADD=""
+CURL_MIN_VERSION="7.15.1"
+
+AC_DEFUN([CHECK_CURL],
+[dnl
+
+AC_ARG_WITH(
+    curl,
+    [AC_HELP_STRING([--with-curl=PATH],[Path to curl prefix or config script])],
+    [test_paths="${with_curl}"],
+    [test_paths="/usr/local/libcurl /usr/local/curl /usr/local /opt/libcurl /opt/curl /opt /usr"])
+
+AC_MSG_CHECKING([for libcurl config script])
+
+for x in ${test_paths}; do
+    dnl # Determine if the script was specified and use it directly
+    if test ! -d "$x" -a -e "$x"; then
+        CURL_CONFIG=$x
+        curl_path="no"
+        break
+    fi
+
+    dnl # Try known config script names/locations
+    for CURL_CONFIG in curl-config; do
+        if test -e "${x}/bin/${CURL_CONFIG}"; then
+            curl_path="${x}/bin"
+            break
+        elif test -e "${x}/${CURL_CONFIG}"; then
+            curl_path="${x}"
+            break
+        else
+            curl_path=""
+        fi
+    done
+    if test -n "$curl_path"; then
+        break
+    fi
+done
+
+if test -n "${curl_path}"; then
+    if test "${curl_path}" != "no"; then
+        CURL_CONFIG="${curl_path}/${CURL_CONFIG}"
+    fi
+    AC_MSG_RESULT([${CURL_CONFIG}])
+    CURL_VERSION=`${CURL_CONFIG} --version | sed 's/^[[^0-9]][[^[:space:]]][[^[:space:]]]*[[[:space:]]]*//'`
+    if test "$verbose_output" -eq 1; then AC_MSG_NOTICE(curl VERSION: $CURL_VERSION); fi
+    CURL_CFLAGS="`${CURL_CONFIG} --cflags`"
+    if test "$verbose_output" -eq 1; then AC_MSG_NOTICE(curl CFLAGS: $CURL_CFLAGS); fi
+    CURL_LDADD="`${CURL_CONFIG} --libs`"
+    if test "$verbose_output" -eq 1; then AC_MSG_NOTICE(curl LDADD: $CURL_LIBS); fi
+
+    dnl # Check version is ok
+    AC_MSG_CHECKING([if libcurl is at least v${CURL_MIN_VERSION}])
+    curl_min_ver=`echo ${CURL_MIN_VERSION} | awk -F. '{print (\$ 1 * 1000000) + (\$ 2 * 1000) + \$ 3}'`
+    curl_ver=`echo ${CURL_VERSION} | awk -F. '{print (\$ 1 * 1000000) + (\$ 2 * 1000) + \$ 3}'`
+    if test "$curl_min_ver" -le "$curl_ver"; then
+        AC_MSG_RESULT([yes, $CURL_VERSION])
+	curl_tlsv2_ver=`echo 7.34.0 | awk -F. '{print (\$ 1 * 1000000) + (\$ 2 * 1000) + \$ 3}'`
+	if test "$curl_tlsv2_ver" -le "$curl_ver"; then
+	    CURL_CFLAGS="${CURL_CFLAGS} -DWITH_CURL_SSLVERSION_TLSv1_2"
+	fi
+	CURL_CFLAGS="${CURL_CFLAGS} -DWITH_CURL"
+    else
+        AC_MSG_RESULT([no, $CURL_VERSION])
+        AC_MSG_NOTICE([NOTE: curl library may be too old])
+    fi
+
+    dnl # Check/warn if GnuTLS is used
+    AC_MSG_CHECKING([if libcurl is linked with gnutls])
+    curl_uses_gnutls=`echo ${CURL_LIBS} | grep gnutls | wc -l`
+    if test "$curl_uses_gnutls" -ne 0; then
+        AC_MSG_RESULT([yes])
+        AC_MSG_NOTICE([NOTE: curl linked with gnutls may be buggy, openssl recommended])
+        CURL_USES_GNUTLS=yes
+    else
+        AC_MSG_RESULT([no])
+        CURL_USES_GNUTLS=no
+    fi
+
+else
+    AC_MSG_RESULT([no])
+fi
+
+AC_SUBST(CURL_CONFIG)
+AC_SUBST(CURL_VERSION)
+AC_SUBST(CURL_CPPFLAGS)
+AC_SUBST(CURL_CFLAGS)
+AC_SUBST(CURL_LDFLAGS)
+AC_SUBST(CURL_LDADD)
+AC_SUBST(CURL_USES_GNUTLS)
+
+if test -z "${CURL_VERSION}"; then
+  AC_MSG_NOTICE([*** curl library not found.])
+else
+  AC_MSG_NOTICE([using curl v${CURL_VERSION}])
+  ifelse([$1], , , $1) 
+fi 
+])

configure.ac

@@ -44,6 +44,19 @@ PROG_GEOIP
 
 AM_CONDITIONAL([GEOIP_VERSION], [test "$GEOIP_VERSION" != ""])
 
+#
+# Check for curl
+#
+CHECK_CURL
+
+if test -z "${CURL_VERSION}"; then
+  AC_MSG_NOTICE([NOTE: curl was not found. SecRemoteRules support was disabled.])
+else
+  AC_DEFINE([MSC_WITH_CURL], [1], [Define if libcurl is available])
+fi
+
+
+
 # Checks for header files.
 AC_HEADER_STDC
 AC_CHECK_HEADERS([string])

examples/simple_example_using_c/basic_rules.conf

@@ -1,3 +1,7 @@
+SecRuleEngine On
+SecDebugLog /tmp/modsec_debug.log
+SecDebugLogLevel 9
+
 SecRule ARGS:test "@detectSQLi" "allow"
 SecRule ARGS|ARGS:test "!@contains asdfsafdt" "allow"
 SecRule ARGS "@detectSQLi" "allow"

examples/simple_example_using_c/test.c

@@ -24,6 +24,8 @@ char main_rule_uri[] = "basic_rules.conf";
 
 int main (int argc, char **argv)
 {
+    int ret = 1;
+    const char *error = NULL;
     ModSecurity *modsec = NULL;
     Assay *assay = NULL;
     Rules *rules = NULL;
@@ -34,7 +36,24 @@ int main (int argc, char **argv)
         "example on how to use ModSecurity API");
 
     rules = msc_create_rules_set();
-    msc_rules_add_file(rules, main_rule_uri);
+
+    ret = msc_rules_add_file(rules, main_rule_uri, &error);
+    if (ret == 0) {
+        fprintf(stderr, "Problems loading the rules --\n");
+        fprintf(stderr, "%s\n", error);
+        goto end;
+    }
+    msc_rules_dump(rules);
+
+    ret = msc_rules_add_remote(rules, "test",
+        "https://www.modsecurity.org/modsecurity-regression-test-secremoterules.txt",
+        &error);
+    if (ret == 0) {
+        fprintf(stderr, "Problems loading the rules --\n");
+        fprintf(stderr, "%s\n", error);
+        goto end;
+    }
+    msc_rules_dump(rules);
 
     assay = msc_new_assay(modsec, rules);
 
@@ -46,7 +65,7 @@ int main (int argc, char **argv)
     msc_process_request_body(assay);
     msc_process_response_headers(assay);
     msc_process_response_body(assay);
-
+end:
     msc_rules_cleanup(rules);
     msc_cleanup(modsec);
 

headers/modsecurity/rules.h

@@ -66,9 +66,12 @@ class Rules {
     * names should follow a patterner
     *
     */
-    int loadFromUri(char *uri);
+    bool loadFromUri(char *uri);
-    int loadRemote(char *key, char *uri);
+    bool loadRemote(char *key, char *uri);
-    int load(const char *rules);
+    bool load(const char *rules);
+    bool load(const char *rules, const std::string &ref);
+
+    void dump();
 
     int merge(Parser::Driver *driver);
     int merge(Rules *rules);
@@ -160,10 +163,12 @@ extern "C" {
 #endif
 
 Rules *msc_create_rules_set();
+void msc_rules_dump(Rules *rules);
 int msc_rules_merge(Rules *rules_dst, Rules *rules_from);
-int msc_rules_add_remote(Rules *rules, char *key, char *uri);
+int msc_rules_add_remote(Rules *rules, char *key, char *uri,
-int msc_rules_add_file(Rules *rules, char *file);
+    const char **error);
-int msc_rules_add(Rules *rules, const char *plain_rules);
+int msc_rules_add_file(Rules *rules, char *file, const char **error);
+int msc_rules_add(Rules *rules, const char *plain_rules, const char **error);
 int msc_rules_cleanup(Rules *rules);
 
 #ifdef __cplusplus

src/Makefile.am

@@ -94,10 +94,13 @@ ACTIONS = \
 	actions/transformations/url_encode.cc \
 	actions/transformations/utf8_to_unicode.cc
 
+
 UTILS = \
-	utils/sha1.cc \
+	utils/geo_lookup.cc \
+	utils/https_client.cc \
 	utils/md5.cc \
-	utils/geo_lookup.cc
+	utils/sha1.cc
+
 
 libmodsecurity_la_SOURCES = \
 	parser/seclang-parser.yy \
@@ -171,10 +174,12 @@ libmodsecurity_la_CPPFLAGS = \
 	-I ../headers
 
 libmodsecurity_la_LIBADD = \
-	@LEXLIB@ \
+        $(CURL_LDADD) \
         $(GEOIP_LDADD) \
+        @LEXLIB@ \
         $(YAJL_LDADD)
 
+
 libmodsecurity_la_LDFLAGS = \
 	-version-info @MSC_VERSION_INFO@
 

src/modsecurity.cc

@@ -20,6 +20,9 @@
 #include "src/config.h"
 #include "src/unique_id.h"
 
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
 
 namespace ModSecurity {
 
@@ -42,6 +45,9 @@ ModSecurity::ModSecurity()
     : m_connector("") {
     UniqueId::uniqueId();
     srand(time(NULL));
+#ifdef MSC_WITH_CURL
+    curl_global_init(CURL_GLOBAL_ALL);
+#endif
 }
 
 

src/parser/driver.cc

@@ -70,11 +70,12 @@ int Driver::addSecRule(Rule *rule) {
 }
 
 
-int Driver::parse(const std::string &f) {
+int Driver::parse(const std::string &f, const std::string &ref) {
+    this->ref = ref;
     buffer = f;
     scan_begin();
     yy::seclang_parser parser(*this);
-    parser.set_debug_level(0);
+    parser.set_debug_level(trace_parsing);
     int res = parser.parse();
 
     if (audit_log->init() == false) {
@@ -87,6 +88,7 @@ int Driver::parse(const std::string &f) {
 
 
 int Driver::parseFile(const std::string &f) {
+    this->ref = f;
     file = f;
     scan_begin();
     yy::seclang_parser parser(*this);
@@ -106,7 +108,7 @@ void Driver::error(const yy::location& l, const std::string& m,
     const std::string& c) {
     if (parserError.tellp() == 0) {
         parserError << "Parser error, ";
-        parserError << "Filename: " << file << ". ";
+        parserError << "File: " << ref << ". ";
         parserError << "Line: " << l.end.line << ". ";
         parserError << "Column: " << l.end.column << ". ";
     }

src/parser/driver.h

@@ -71,7 +71,7 @@ class Driver : public Rules {
     // Run the parser on file F.
     // Return 0 on success.
     int parseFile(const std::string& f);
-    int parse(const std::string& f);
+    int parse(const std::string& f, const std::string &ref);
 
     // The name of the file being parsed.
     // Used later to pass the file name to the location tracker.
@@ -87,6 +87,7 @@ class Driver : public Rules {
     void error(const yy::location& l, const std::string& m,
         const std::string& c);
 
+    std::string ref;
     std::string buffer;
 };
 

src/rules.cc

@@ -25,8 +25,10 @@
 #include "modsecurity/assay.h"
 #include "src/utils.h"
 #include "parser/driver.h"
+#include "utils/https_client.h"
 
 using ModSecurity::Parser::Driver;
+using ModSecurity::Utils::HttpsClient;
 
 namespace ModSecurity {
 
@@ -75,6 +77,11 @@ void Rules::decrementReferenceCount(void) {
 }
 
 
+Rules::~Rules() {
+    // audit_log->refCountDecreaseAndCheck();
+}
+
+
 /**
  * @name    loadFromUri
  * @brief   load rules from a give uri
@@ -90,11 +97,14 @@ void Rules::decrementReferenceCount(void) {
  * @retval false Problem loading the rules.
  *
  */
-int Rules::loadFromUri(char *uri) {
+bool Rules::loadFromUri(char *uri) {
-    std::cout << "Loading rules from: " << uri << std::endl;
-
     Driver *driver = new Driver();
-    driver->parse(uri);
+
+    if (driver->parseFile(uri) == false) {
+        parserError << driver->parserError.rdbuf();
+        return false;
+    }
+
     this->merge(driver);
     delete driver;
 
@@ -102,24 +112,30 @@ int Rules::loadFromUri(char *uri) {
 }
 
 
-Rules::~Rules() {
+bool Rules::loadRemote(char *key, char *uri) {
-    // audit_log->refCountDecreaseAndCheck();
+    HttpsClient client;
+    bool ret = client.download(uri);
+
+    if (ret) {
+        return this->load(client.content.c_str(), uri);
     }
 
-
-int Rules::loadRemote(char *key, char *uri) {
-    return true;
-}
-
-
-int Rules::load(const char *plain_rules) {
-    Driver *driver = new Driver();
-
-    if (driver->parse(plain_rules) == false) {
-        parserError << driver->parserError.rdbuf();
     return false;
 }
 
+
+bool Rules::load(const char *plain_rules) {
+    return this->load(plain_rules, "");
+}
+
+
+bool Rules::load(const char *plain_rules, const std::string &ref) {
+    Driver *driver = new Driver();
+
+    if (driver->parse(plain_rules, ref) == false) {
+        parserError << driver->parserError.str();
+        return false;
+    }
     this->merge(driver);
     delete driver;
 
@@ -225,6 +241,21 @@ int Rules::merge(Rules *from) {
 }
 
 
+void Rules::dump() {
+    std::cout << "Rules: " << std::endl;
+    for (int i = 0; i < ModSecurity::Phases::NUMBER_OF_PHASES; i++) {
+        std::vector<Rule *> rules = this->rules[i];
+        std::cout << "Phase: " << std::to_string(i);
+        std::cout << " (" << std::to_string(rules.size());
+        std::cout << " rules)" << std::endl;
+        for (int j = 0; j < rules.size(); j++) {
+            std::cout << "    Rule ID: " << std::to_string(rules[j]->rule_id);
+            std::cout << std::endl;
+        }
+    }
+}
+
+
 extern "C" Rules *msc_create_rules_set() {
     Rules *rules = new Rules();
 
@@ -232,6 +263,11 @@ extern "C" Rules *msc_create_rules_set() {
 }
 
 
+extern "C" void msc_rules_dump(Rules *rules) {
+    rules->dump();
+}
+
+
 extern "C" int msc_rules_merge(Rules *rules_dst,
     Rules *rules_from) {
     rules_dst->merge(rules_from);
@@ -241,24 +277,32 @@ extern "C" int msc_rules_merge(Rules *rules_dst,
 
 
 extern "C" int msc_rules_add_remote(Rules *rules,
-    char *key, char *uri) {
+    char *key, char *uri, const char **error) {
-    rules->loadRemote(key, uri);
+    int ret = rules->loadRemote(key, uri);
-
+    if (ret == 0) {
-    return 0;
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 
-extern "C" int msc_rules_add_file(Rules *rules, char *file) {
+extern "C" int msc_rules_add_file(Rules *rules, char *file,
-    rules->loadFromUri(file);
+    const char **error) {
-
+    int ret = rules->loadFromUri(file);
-    return 0;
+    if (ret == 0) {
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 
-extern "C" int msc_rules_add(Rules *rules, const char *plain_rules) {
+extern "C" int msc_rules_add(Rules *rules, const char *plain_rules,
-    rules->load(plain_rules);
+    const char **error) {
-
+    int ret = rules->load(plain_rules);
-    return 0;
+    if (ret == 0) {
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 

src/utils/https_client.cc

@@ -0,0 +1,106 @@
+/*
+ * ModSecurity, http://www.modsecurity.org/
+ * Copyright (c) 2015 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+ *
+ * You may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Trustwave Holdings, Inc.
+ * directly using the email address security@modsecurity.org.
+ *
+ */
+
+#include "utils/https_client.h"
+#include "src/config.h"
+
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <string>
+
+#include <fstream>
+#include <iostream>
+
+#include "src/unique_id.h"
+
+namespace ModSecurity {
+namespace Utils {
+
+
+size_t HttpsClient::handle(char * data, size_t size, size_t nmemb, void * p) {
+    return static_cast<HttpsClient*>(p)->handle_impl(data, size, nmemb);
+}
+
+
+size_t HttpsClient::handle_impl(char* data, size_t size, size_t nmemb) {
+    content.append(data, size * nmemb);
+    return size * nmemb;
+}
+
+
+#ifdef MSC_WITH_CURL
+bool HttpsClient::download(const std::string &uri) {
+    CURL *curl;
+    CURLcode res;
+    std::string uniqueId = "ModSec-unique-id: " + UniqueId::uniqueId();
+
+    curl = curl_easy_init();
+    if (!curl) {
+        error = "Not able to initialize libcurl";
+        return false;
+    }
+
+    struct curl_slist *headers_chunk = NULL;
+    curl_easy_setopt(curl, CURLOPT_URL, uri.c_str());
+
+    headers_chunk = curl_slist_append(headers_chunk, uniqueId.c_str());
+
+    /* Make it TLS 1.x only. */
+    curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
+
+    /* those are the default options, but lets make sure */
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1);
+
+    /* send all data to this function  */
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &HttpsClient::handle);
+
+    /* we pass our 'chunk' struct to the callback function */
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
+
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "modesecurity3");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers_chunk);
+
+    /* We want Curl to return error in case there is an HTTP error code */
+    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+
+    res = curl_easy_perform(curl);
+
+    curl_slist_free_all(headers_chunk);
+
+    if (res != CURLE_OK) {
+        error = curl_easy_strerror(res);
+    }
+
+    curl_easy_cleanup(curl);
+
+    return res == CURLE_OK;
+}
+#else
+bool HttpsClient::download(const std::string &uri) {
+    error = "Not compiled with libcurl support";
+    return false;
+}
+#endif
+
+}  // namespace Utils
+}  // namespace ModSecurity
+

src/utils/https_client.h

@@ -0,0 +1,55 @@
+/*
+ * ModSecurity, http://www.modsecurity.org/
+ * Copyright (c) 2015 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+ *
+ * You may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Trustwave Holdings, Inc.
+ * directly using the email address security@modsecurity.org.
+ *
+ */
+
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <functional>
+
+#include <GeoIPCity.h>
+
+#ifndef SRC_UTILS_HTTPS_CLIENT_H_
+#define SRC_UTILS_HTTPS_CLIENT_H_
+
+#include "modsecurity/assay.h"
+
+namespace ModSecurity {
+namespace Utils {
+
+
+class HttpsClient {
+ public:
+    HttpsClient()
+        : content(""),
+        error("") { }
+
+    bool download(const std::string &uri);
+    std::string content;
+
+    static size_t handle(char * data, size_t size, size_t nmemb, void * p);
+    size_t handle_impl(char * data, size_t size, size_t nmemb);
+
+    std::string error;
+};
+
+
+}  // namespace Utils
+}  // namespace ModSecurity
+
+#endif  // SRC_UTILS_HTTPS_CLIENT_H_

test/Makefile.am

@@ -16,6 +16,7 @@ MAINTAINERCLEANFILES = \
 
 bin_PROGRAMS = unit-tests regression-tests
 
+
 # unit_tests
 
 unit_tests_SOURCES = \
@@ -24,6 +25,7 @@ unit_tests_SOURCES = \
 
 unit_tests_LDADD = \
 	$(top_builddir)/src/.libs/libmodsecurity.a \
+	$(CURL_LDADD) \
 	$(GEOIP_LDADD) \
 	$(YAJL_LDADD)
 
@@ -33,6 +35,7 @@ unit_tests_CPPFLAGS = \
 	-O0 \
 	-g \
 	-I$(top_builddir)/headers \
+	$(CURL_CFLAGS) \
 	$(GEOIP_CFLAGS) \
 	$(YAJL_CFLAGS)
 
@@ -46,6 +49,7 @@ regression_tests_SOURCES = \
 
 regression_tests_LDADD = \
 	$(top_builddir)/src/.libs/libmodsecurity.a \
+	$(CURL_LDADD) \
 	$(GEOIP_LDADD) \
 	$(YAJL_LDADD)
 
@@ -55,5 +59,6 @@ regression_tests_CPPFLAGS = \
 	-O0 \
 	-g \
 	-I$(top_builddir)/headers \
+	$(CURL_CFLAGS) \
 	$(GEOIP_CFLAGS) \
 	$(YAJL_CFLAGS)

test/benchmark/Makefile.am

@@ -7,6 +7,7 @@ benchmark_SOURCES = \
 
 benchmark_LDADD = \
 	$(top_builddir)/src/.libs/libmodsecurity.a \
+	$(CURL_LDADD) \
 	$(GEOIP_LDADD) \
 	$(YAJL_LDADD)