Adds support to load remote rules

src/Makefile.am

@@ -94,10 +94,13 @@ ACTIONS = \
 	actions/transformations/url_encode.cc \
 	actions/transformations/utf8_to_unicode.cc
 
+
 UTILS = \
-	utils/sha1.cc \
+	utils/geo_lookup.cc \
+	utils/https_client.cc \
 	utils/md5.cc \
-	utils/geo_lookup.cc
+	utils/sha1.cc
+
 
 libmodsecurity_la_SOURCES = \
 	parser/seclang-parser.yy \
@@ -171,9 +174,11 @@ libmodsecurity_la_CPPFLAGS = \
 	-I ../headers
 
 libmodsecurity_la_LIBADD = \
-	@LEXLIB@ \
-	$(GEOIP_LDADD) \
-    $(YAJL_LDADD)
+        $(CURL_LDADD) \
+        $(GEOIP_LDADD) \
+        @LEXLIB@ \
+        $(YAJL_LDADD)
+
 
 libmodsecurity_la_LDFLAGS = \
 	-version-info @MSC_VERSION_INFO@

src/modsecurity.cc

@@ -20,6 +20,9 @@
 #include "src/config.h"
 #include "src/unique_id.h"
 
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
 
 namespace ModSecurity {
 
@@ -42,6 +45,9 @@ ModSecurity::ModSecurity()
     : m_connector("") {
     UniqueId::uniqueId();
     srand(time(NULL));
+#ifdef MSC_WITH_CURL
+    curl_global_init(CURL_GLOBAL_ALL);
+#endif
 }
 
 

src/parser/driver.cc

@@ -70,11 +70,12 @@ int Driver::addSecRule(Rule *rule) {
 }
 
 
-int Driver::parse(const std::string &f) {
+int Driver::parse(const std::string &f, const std::string &ref) {
+    this->ref = ref;
     buffer = f;
     scan_begin();
     yy::seclang_parser parser(*this);
-    parser.set_debug_level(0);
+    parser.set_debug_level(trace_parsing);
     int res = parser.parse();
 
     if (audit_log->init() == false) {
@@ -87,6 +88,7 @@ int Driver::parse(const std::string &f) {
 
 
 int Driver::parseFile(const std::string &f) {
+    this->ref = f;
     file = f;
     scan_begin();
     yy::seclang_parser parser(*this);
@@ -106,7 +108,7 @@ void Driver::error(const yy::location& l, const std::string& m,
     const std::string& c) {
     if (parserError.tellp() == 0) {
         parserError << "Parser error, ";
-        parserError << "Filename: " << file << ". ";
+        parserError << "File: " << ref << ". ";
         parserError << "Line: " << l.end.line << ". ";
         parserError << "Column: " << l.end.column << ". ";
     }

src/parser/driver.h

@@ -71,7 +71,7 @@ class Driver : public Rules {
     // Run the parser on file F.
     // Return 0 on success.
     int parseFile(const std::string& f);
-    int parse(const std::string& f);
+    int parse(const std::string& f, const std::string &ref);
 
     // The name of the file being parsed.
     // Used later to pass the file name to the location tracker.
@@ -87,6 +87,7 @@ class Driver : public Rules {
     void error(const yy::location& l, const std::string& m,
         const std::string& c);
 
+    std::string ref;
     std::string buffer;
 };
 

src/rules.cc

@@ -25,8 +25,10 @@
 #include "modsecurity/assay.h"
 #include "src/utils.h"
 #include "parser/driver.h"
+#include "utils/https_client.h"
 
 using ModSecurity::Parser::Driver;
+using ModSecurity::Utils::HttpsClient;
 
 namespace ModSecurity {
 
@@ -75,6 +77,11 @@ void Rules::decrementReferenceCount(void) {
 }
 
 
+Rules::~Rules() {
+    // audit_log->refCountDecreaseAndCheck();
+}
+
+
 /**
  * @name    loadFromUri
  * @brief   load rules from a give uri
@@ -90,11 +97,14 @@ void Rules::decrementReferenceCount(void) {
  * @retval false Problem loading the rules.
  *
  */
-int Rules::loadFromUri(char *uri) {
-    std::cout << "Loading rules from: " << uri << std::endl;
-
+bool Rules::loadFromUri(char *uri) {
     Driver *driver = new Driver();
-    driver->parse(uri);
+
+    if (driver->parseFile(uri) == false) {
+        parserError << driver->parserError.rdbuf();
+        return false;
+    }
+
     this->merge(driver);
     delete driver;
 
@@ -102,24 +112,30 @@ int Rules::loadFromUri(char *uri) {
 }
 
 
-Rules::~Rules() {
-    // audit_log->refCountDecreaseAndCheck();
-}
+bool Rules::loadRemote(char *key, char *uri) {
+    HttpsClient client;
+    bool ret = client.download(uri);
 
-
-int Rules::loadRemote(char *key, char *uri) {
-    return true;
-}
-
-
-int Rules::load(const char *plain_rules) {
-    Driver *driver = new Driver();
-
-    if (driver->parse(plain_rules) == false) {
-        parserError << driver->parserError.rdbuf();
-        return false;
+    if (ret) {
+        return this->load(client.content.c_str(), uri);
     }
 
+    return false;
+}
+
+
+bool Rules::load(const char *plain_rules) {
+    return this->load(plain_rules, "");
+}
+
+
+bool Rules::load(const char *plain_rules, const std::string &ref) {
+    Driver *driver = new Driver();
+
+    if (driver->parse(plain_rules, ref) == false) {
+        parserError << driver->parserError.str();
+        return false;
+    }
     this->merge(driver);
     delete driver;
 
@@ -225,6 +241,21 @@ int Rules::merge(Rules *from) {
 }
 
 
+void Rules::dump() {
+    std::cout << "Rules: " << std::endl;
+    for (int i = 0; i < ModSecurity::Phases::NUMBER_OF_PHASES; i++) {
+        std::vector<Rule *> rules = this->rules[i];
+        std::cout << "Phase: " << std::to_string(i);
+        std::cout << " (" << std::to_string(rules.size());
+        std::cout << " rules)" << std::endl;
+        for (int j = 0; j < rules.size(); j++) {
+            std::cout << "    Rule ID: " << std::to_string(rules[j]->rule_id);
+            std::cout << std::endl;
+        }
+    }
+}
+
+
 extern "C" Rules *msc_create_rules_set() {
     Rules *rules = new Rules();
 
@@ -232,6 +263,11 @@ extern "C" Rules *msc_create_rules_set() {
 }
 
 
+extern "C" void msc_rules_dump(Rules *rules) {
+    rules->dump();
+}
+
+
 extern "C" int msc_rules_merge(Rules *rules_dst,
     Rules *rules_from) {
     rules_dst->merge(rules_from);
@@ -241,24 +277,32 @@ extern "C" int msc_rules_merge(Rules *rules_dst,
 
 
 extern "C" int msc_rules_add_remote(Rules *rules,
-    char *key, char *uri) {
-    rules->loadRemote(key, uri);
-
-    return 0;
+    char *key, char *uri, const char **error) {
+    int ret = rules->loadRemote(key, uri);
+    if (ret == 0) {
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 
-extern "C" int msc_rules_add_file(Rules *rules, char *file) {
-    rules->loadFromUri(file);
-
-    return 0;
+extern "C" int msc_rules_add_file(Rules *rules, char *file,
+    const char **error) {
+    int ret = rules->loadFromUri(file);
+    if (ret == 0) {
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 
-extern "C" int msc_rules_add(Rules *rules, const char *plain_rules) {
-    rules->load(plain_rules);
-
-    return 0;
+extern "C" int msc_rules_add(Rules *rules, const char *plain_rules,
+    const char **error) {
+    int ret = rules->load(plain_rules);
+    if (ret == 0) {
+        *error = rules->getParserError().c_str();
+    }
+    return ret;
 }
 
 

src/utils/https_client.cc

@@ -0,0 +1,106 @@
+/*
+ * ModSecurity, http://www.modsecurity.org/
+ * Copyright (c) 2015 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+ *
+ * You may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Trustwave Holdings, Inc.
+ * directly using the email address security@modsecurity.org.
+ *
+ */
+
+#include "utils/https_client.h"
+#include "src/config.h"
+
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
+
+#include <sys/socket.h>
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <string>
+
+#include <fstream>
+#include <iostream>
+
+#include "src/unique_id.h"
+
+namespace ModSecurity {
+namespace Utils {
+
+
+size_t HttpsClient::handle(char * data, size_t size, size_t nmemb, void * p) {
+    return static_cast<HttpsClient*>(p)->handle_impl(data, size, nmemb);
+}
+
+
+size_t HttpsClient::handle_impl(char* data, size_t size, size_t nmemb) {
+    content.append(data, size * nmemb);
+    return size * nmemb;
+}
+
+
+#ifdef MSC_WITH_CURL
+bool HttpsClient::download(const std::string &uri) {
+    CURL *curl;
+    CURLcode res;
+    std::string uniqueId = "ModSec-unique-id: " + UniqueId::uniqueId();
+
+    curl = curl_easy_init();
+    if (!curl) {
+        error = "Not able to initialize libcurl";
+        return false;
+    }
+
+    struct curl_slist *headers_chunk = NULL;
+    curl_easy_setopt(curl, CURLOPT_URL, uri.c_str());
+
+    headers_chunk = curl_slist_append(headers_chunk, uniqueId.c_str());
+
+    /* Make it TLS 1.x only. */
+    curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
+
+    /* those are the default options, but lets make sure */
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 1);
+    curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 1);
+
+    /* send all data to this function  */
+    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &HttpsClient::handle);
+
+    /* we pass our 'chunk' struct to the callback function */
+    curl_easy_setopt(curl, CURLOPT_WRITEDATA, this);
+
+    curl_easy_setopt(curl, CURLOPT_USERAGENT, "modesecurity3");
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers_chunk);
+
+    /* We want Curl to return error in case there is an HTTP error code */
+    curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1);
+
+    res = curl_easy_perform(curl);
+
+    curl_slist_free_all(headers_chunk);
+
+    if (res != CURLE_OK) {
+        error = curl_easy_strerror(res);
+    }
+
+    curl_easy_cleanup(curl);
+
+    return res == CURLE_OK;
+}
+#else
+bool HttpsClient::download(const std::string &uri) {
+    error = "Not compiled with libcurl support";
+    return false;
+}
+#endif
+
+}  // namespace Utils
+}  // namespace ModSecurity
+

src/utils/https_client.h

@@ -0,0 +1,55 @@
+/*
+ * ModSecurity, http://www.modsecurity.org/
+ * Copyright (c) 2015 Trustwave Holdings, Inc. (http://www.trustwave.com/)
+ *
+ * You may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * If any of the files related to licensing are missing or if you have any
+ * other questions related to licensing please contact Trustwave Holdings, Inc.
+ * directly using the email address security@modsecurity.org.
+ *
+ */
+
+#ifdef MSC_WITH_CURL
+#include <curl/curl.h>
+#endif
+
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <functional>
+
+#include <GeoIPCity.h>
+
+#ifndef SRC_UTILS_HTTPS_CLIENT_H_
+#define SRC_UTILS_HTTPS_CLIENT_H_
+
+#include "modsecurity/assay.h"
+
+namespace ModSecurity {
+namespace Utils {
+
+
+class HttpsClient {
+ public:
+    HttpsClient()
+        : content(""),
+        error("") { }
+
+    bool download(const std::string &uri);
+    std::string content;
+
+    static size_t handle(char * data, size_t size, size_t nmemb, void * p);
+    size_t handle_impl(char * data, size_t size, size_t nmemb);
+
+    std::string error;
+};
+
+
+}  // namespace Utils
+}  // namespace ModSecurity
+
+#endif  // SRC_UTILS_HTTPS_CLIENT_H_