Rename default.conf to default.conf

Oriane 23.12.24 adding new composes

README.md

@@ -74,7 +74,7 @@ For Linux, if you’ve built your own package use the following commands:
 
 ```bash
 $ install-cp-nano-agent.sh --install --hybrid_mode
-$ install-cp-nano-service-http-transaction-handler.sh –install
+$ install-cp-nano-service-http-transaction-handler.sh --install
 $ install-cp-nano-attachment-registration-manager.sh --install
 ```
 You can add the ```--token <token>``` and ```--email <email address>``` options to the first command, to get a token follow [documentation](https://docs.openappsec.io/getting-started/using-the-web-ui-saas/connect-deployed-agents-to-saas-management-k8s-and-linux). 

components/CMakeLists.txt

@@ -7,3 +7,4 @@ add_subdirectory(pending_key)
 add_subdirectory(utils)
 add_subdirectory(attachment-intakers)
 add_subdirectory(security_apps)
+add_subdirectory(nginx_message_reader)

components/include/central_nginx_manager.h

@@ -0,0 +1,45 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __CENTRAL_NGINX_MANAGER_H__
+#define __CENTRAL_NGINX_MANAGER_H__
+
+#include "component.h"
+#include "singleton.h"
+#include "i_messaging.h"
+#include "i_rest_api.h"
+#include "i_mainloop.h"
+#include "i_agent_details.h"
+
+class CentralNginxManager
+        :
+    public Component,
+    Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Messaging>,
+    Singleton::Consume<I_MainLoop>,
+    Singleton::Consume<I_AgentDetails>
+{
+public:
+    CentralNginxManager();
+    ~CentralNginxManager();
+
+    void preload() override;
+    void init() override;
+    void fini() override;
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif // __CENTRAL_NGINX_MANAGER_H__

components/include/nginx_message_reader.h

@@ -0,0 +1,28 @@
+#ifndef __NGINX_MESSAGE_READER_H__
+#define __NGINX_MESSAGE_READER_H__
+
+#include "singleton.h"
+#include "i_mainloop.h"
+#include "i_socket_is.h"
+#include "component.h"
+
+class NginxMessageReader
+        :
+    public Component,
+    Singleton::Consume<I_MainLoop>,
+    Singleton::Consume<I_Socket>
+{
+public:
+    NginxMessageReader();
+    ~NginxMessageReader();
+
+    void init() override;
+    void fini() override;
+    void preload() override;
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif //__NGINX_MESSAGE_READER_H__

components/include/nginx_utils.h

@@ -0,0 +1,51 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __NGINX_UTILS_H__
+#define __NGINX_UTILS_H__
+
+#include <string>
+
+#include "maybe_res.h"
+#include "singleton.h"
+#include "i_shell_cmd.h"
+
+class NginxConfCollector
+{
+public:
+    NginxConfCollector(const std::string &nginx_conf_input_path, const std::string &nginx_conf_output_path);
+    Maybe<std::string> generateFullNginxConf() const;
+
+private:
+    std::vector<std::string> expandIncludes(const std::string &includePattern) const;
+    void processConfigFile(
+        const std::string &path,
+        std::ostringstream &conf_output,
+        std::vector<std::string> &errors
+    ) const;
+
+    std::string main_conf_input_path;
+    std::string main_conf_output_path;
+    std::string main_conf_directory_path;
+};
+
+class NginxUtils : Singleton::Consume<I_ShellCmd>
+{
+public:
+    static std::string getModulesPath();
+    static std::string getMainNginxConfPath();
+    static Maybe<void> validateNginxConf(const std::string &nginx_conf_path);
+    static Maybe<void> reloadNginx(const std::string &nginx_conf_path);
+};
+
+#endif // __NGINX_UTILS_H__

components/include/waap.h

@@ -33,7 +33,6 @@ class I_WaapAssetStatesManager;
 class I_Messaging;
 class I_AgentDetails;
 class I_Encryptor;
-class I_WaapModelResultLogger;
 
 const std::string WAAP_APPLICATION_NAME = "waap application";
 
@@ -51,8 +50,7 @@ class WaapComponent
     Singleton::Consume<I_AgentDetails>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_Encryptor>,
-    Singleton::Consume<I_Environment>,
-    Singleton::Consume<I_WaapModelResultLogger>
+    Singleton::Consume<I_Environment>
 {
 public:
     WaapComponent();

components/nginx_message_reader/CMakeLists.txt

@@ -0,0 +1,3 @@
+link_directories(${BOOST_ROOT}/lib)
+
+add_library(nginx_message_reader nginx_message_reader.cc)

components/nginx_message_reader/nginx_message_reader.cc

@@ -0,0 +1,735 @@
+#include "nginx_message_reader.h"
+
+#include <string>
+#include <boost/regex.hpp>
+#include <boost/algorithm/string.hpp>
+#include <boost/algorithm/string/regex.hpp>
+
+#include "config.h"
+#include "singleton.h"
+#include "i_mainloop.h"
+#include "enum_array.h"
+#include "log_generator.h"
+#include "maybe_res.h"
+#include "http_transaction_data.h"
+#include "generic_rulebase/rulebase_config.h"
+#include "generic_rulebase/evaluators/asset_eval.h"
+#include "generic_rulebase/triggers_config.h"
+#include "agent_core_utilities.h"
+#include "rate_limit_config.h"
+
+USE_DEBUG_FLAG(D_NGINX_MESSAGE_READER);
+
+using namespace std;
+
+static const string syslog_regex_string = (
+    "<[0-9]+>([A-Z][a-z][a-z]\\s{1,2}\\d{1,2}\\s\\d{2}"
+    "[:]\\d{2}[:]\\d{2})\\s([\\w][\\w\\d\\.@-]*)\\s(nginx:)"
+);
+
+static const boost::regex socket_address_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+):(\\d+)");
+static const boost::regex syslog_regex(syslog_regex_string);
+static const boost::regex alert_log_regex(
+    "("
+    + syslog_regex_string + ") "
+    + "(.+?\\[alert\\] )(.+?)"
+    ", (client: .+?)"
+    ", (server: .+?)"
+    ", (request: \".+?\")"
+    ", (upstream: \".+?\")"
+    ", (host: \".+?\")$"
+);
+
+static const boost::regex error_log_regex(
+    "("
+    + syslog_regex_string + ") "
+    + "(.+?\\[error\\] )(.+?)"
+    ", (client: .+?)"
+    ", (server: .+?)"
+    ", (request: \".+?\")"
+    ", (upstream: \".+?\")"
+    ", (host: \".+?\")$"
+);
+
+static const boost::regex server_regex("(\\d+\\.\\d+\\.\\d+\\.\\d+)|(\\w+\\.\\w+)");
+static const boost::regex uri_regex("^/");
+static const boost::regex port_regex("\\d+");
+static const boost::regex response_code_regex("[0-9]{3}");
+static const boost::regex http_method_regex("[A-Za-z]+");
+
+class NginxMessageReader::Impl
+{
+public:
+    void
+    init()
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<NginxMessageReader>();
+        mainloop->addOneTimeRoutine(
+            I_MainLoop::RoutineType::System,
+            [this] ()
+            {
+                initSyslogServerSocket();
+                handleNginxLogs();
+            },
+            "Initialize nginx syslog",
+            true
+        );
+    }
+
+    void
+    preload()
+    {
+        registerConfigLoadCb([this]() { loadNginxMessageReaderConfig(); });
+    }
+
+    void
+    fini()
+    {
+        I_Socket *i_socket = Singleton::Consume<I_Socket>::by<NginxMessageReader>();
+        i_socket->closeSocket(syslog_server_socket);
+    }
+
+    void
+    loadNginxMessageReaderConfig()
+    {
+        rate_limit_status_code = getProfileAgentSettingWithDefault<string>(
+            "429",
+            "accessControl.rateLimit.returnCode"
+        );
+        dbgTrace(D_NGINX_MESSAGE_READER) << "Selected rate-limit status code: " << rate_limit_status_code;
+    }
+
+private:
+    enum class LogInfo {
+        HTTP_METHOD,
+        URI,
+        RESPONSE_CODE,
+        HOST,
+        SOURCE,
+        DESTINATION_IP,
+        DESTINATION_PORT,
+        EVENT_MESSAGE,
+        ASSET_ID,
+        ASSET_NAME,
+        RULE_NAME,
+        RULE_ID,
+        COUNT
+    };
+
+    void
+    initSyslogServerSocket()
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<NginxMessageReader>();
+        I_Socket *i_socket = Singleton::Consume<I_Socket>::by<NginxMessageReader>();
+        string nginx_syslog_server_address = getProfileAgentSettingWithDefault<string>(
+            "127.0.0.1:1514",
+            "reverseProxy.nginx.syslogAddress"
+        );
+        dbgInfo(D_NGINX_MESSAGE_READER) << "Attempting to open a socket: " << nginx_syslog_server_address;
+        do {
+            Maybe<I_Socket::socketFd> new_socket = i_socket->genSocket(
+                I_Socket::SocketType::UDP,
+                false,
+                true,
+                nginx_syslog_server_address
+            );
+            if (!new_socket.ok()) {
+                dbgError(D_NGINX_MESSAGE_READER) << "Failed to open a socket. Error: " << new_socket.getErr();
+                mainloop->yield(chrono::milliseconds(500));
+                continue;
+            }
+
+            if (new_socket.unpack() < 0) {
+                dbgError(D_NGINX_MESSAGE_READER)<< "Generated socket is OK yet negative";
+                mainloop->yield(chrono::milliseconds(500));
+                continue;
+            }
+            syslog_server_socket = new_socket.unpack();
+            dbgInfo(D_NGINX_MESSAGE_READER)
+                << "Opened socket for nginx logs over syslog. Socket: "
+                << syslog_server_socket;
+        } while (syslog_server_socket < 0);
+    }
+
+    void
+    handleNginxLogs()
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        I_MainLoop::Routine read_logs =
+        [this] ()
+        {
+            Maybe<string> logs = getLogsFromSocket(syslog_server_socket);
+
+            if (!logs.ok()) {
+                dbgWarning(D_NGINX_MESSAGE_READER)
+                    << "Failed to get NGINX logs from the socket. Error: "
+                    << logs.getErr();
+                return;
+            }
+            string raw_logs_to_parse = logs.unpackMove();
+            vector<string> logs_to_parse = separateLogs(raw_logs_to_parse);
+
+            for (auto const &log: logs_to_parse) {
+                bool log_sent;
+                if (isAccessLog(log)) {
+                    log_sent = sendAccessLog(log);
+                } else if (isAlertErrorLog(log) || isErrorLog(log)) {
+                    log_sent = sendErrorLog(log);
+                } else {
+                    dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
+                    continue;
+                }
+                if (!log_sent) {
+                    dbgWarning(D_NGINX_MESSAGE_READER) << "Failed to send Log to Infinity Portal";
+                } else {
+                    dbgTrace(D_NGINX_MESSAGE_READER) << "Succesfully sent nginx log to Infinity Portal";
+                }
+            }
+        };
+        I_MainLoop *mainloop = Singleton::Consume<I_MainLoop>::by<NginxMessageReader>();
+        mainloop->addFileRoutine(
+            I_MainLoop::RoutineType::RealTime,
+            syslog_server_socket,
+            read_logs,
+            "Process nginx logs",
+            true
+        );
+    }
+
+    bool
+    sendAccessLog(const string &log)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Access log" << log;
+        Maybe<EnumArray<LogInfo, string>> log_info = parseAccessLog(log);
+        if (!log_info.ok()) {
+            dbgWarning(D_NGINX_MESSAGE_READER)
+                << "Failed parsing the NGINX logs. Error: "
+                << log_info.getErr();
+            return false;
+        }
+        auto unpacked_log_info = log_info.unpack();
+
+        if (unpacked_log_info[LogInfo::RESPONSE_CODE] == rate_limit_status_code) {
+            return sendRateLimitLog(unpacked_log_info);
+        }
+        return sendLog(unpacked_log_info);
+    }
+
+    bool
+    sendErrorLog(const string &log)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Error log" << log;
+        Maybe<EnumArray<LogInfo, string>> log_info = parseErrorLog(log);
+        if (!log_info.ok()) {
+            dbgWarning(D_NGINX_MESSAGE_READER)
+                << "Failed parsing the NGINX logs. Error: "
+                << log_info.getErr();
+            return false;
+        }
+        return sendLog(log_info.unpack());
+    }
+
+    bool
+    isAccessLog(const string &log) const
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Chekck if string contains \"accessLog\"" << log;
+        return log.find("accessLog") != string::npos;
+    }
+
+    bool
+    isAlertErrorLog(const string &log) const
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'error log'. Log: " << log;
+        return log.find("[alert]") != string::npos;
+    }
+
+    bool
+    isErrorLog(const string &log) const
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Check if log is of type 'error log'. Log: " << log;
+        return log.find("[error]") != string::npos;
+    }
+
+    bool
+    sendLog(const EnumArray<LogInfo, string> &log_info)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        string event_name;
+        switch (log_info[LogInfo::RESPONSE_CODE][0]) {
+            case '4': {
+                event_name = "Invalid request or incorrect reverse proxy configuration - Request dropped."
+                " Please check the reverse proxy configuration of your relevant assets";
+                break;
+            }
+            case '5': {
+                event_name = "AppSec Gateway reverse proxy error - Request dropped. "
+                "Please verify the reverse proxy configuration of your relevant assets. "
+                "If the issue persists please contact Check Point Support";
+                break;
+            }
+            default: {
+                dbgError(D_NGINX_MESSAGE_READER) << "Irrelevant status code";
+                return false;
+            }
+        }
+
+        dbgTrace(D_NGINX_MESSAGE_READER)
+            << "Nginx log's event name and response code: "
+            << event_name
+            << ", "
+            << log_info[LogInfo::RESPONSE_CODE];
+        LogGen log(
+            event_name,
+            ReportIS::Audience::SECURITY,
+            ReportIS::Severity::INFO,
+            ReportIS::Priority::LOW,
+            ReportIS::Tags::REVERSE_PROXY
+        );
+        log << LogField("eventConfidence", "High");
+
+        for (LogInfo field : makeRange<LogInfo>()) {
+            Maybe<string> string_field = convertLogFieldToString(field);
+            if (!string_field.ok()) {
+                dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " <<  string_field.getErr();
+                return false;
+            }
+
+            if (field != LogInfo::DESTINATION_PORT) {
+                log << LogField(string_field.unpack(), log_info[field]);
+                continue;
+            }
+
+            try {
+                log << LogField(string_field.unpack(), stoi(log_info[field]));
+            } catch (const exception &e) {
+                dbgError(D_NGINX_MESSAGE_READER)
+                    << "Unable to convert port to numeric value: "
+                    << e.what();
+                log << LogField(string_field.unpack(), 0);
+            }
+        }
+        return true;
+    }
+
+    bool
+    sendRateLimitLog(const EnumArray<LogInfo, string> &log_info)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Getting rate-limit rules of asset ID: " << log_info[LogInfo::ASSET_ID];
+
+        ScopedContext rate_limit_ctx;
+
+        rate_limit_ctx.registerValue<GenericConfigId>(AssetMatcher::ctx_key, log_info[LogInfo::ASSET_ID]);
+        auto rate_limit_config = getConfiguration<RateLimitConfig>("rulebase", "rateLimit");
+        if (!rate_limit_config.ok()) {
+            dbgTrace(D_NGINX_MESSAGE_READER)
+                << "Rate limit context does not match asset ID: " << log_info[LogInfo::ASSET_ID];
+            return false;
+        }
+        RateLimitConfig unpacked_rate_limit_config = rate_limit_config.unpack();
+
+        string nginx_uri = log_info[LogInfo::URI];
+        const LogTriggerConf &rate_limit_trigger = unpacked_rate_limit_config.getRateLimitTrigger(nginx_uri);
+
+        dbgTrace(D_NGINX_MESSAGE_READER)<< "About to generate NGINX rate-limit log";
+
+        string event_name = "Rate limit";
+        string security_action = "Drop";
+        bool is_log_required = false;
+
+        // Prevent events checkbox (in triggers)
+        if (rate_limit_trigger.isPreventLogActive(LogTriggerConf::SecurityType::AccessControl)) {
+            is_log_required = true;
+        }
+
+        if (!is_log_required) {
+            dbgTrace(D_NGINX_MESSAGE_READER) << "Not sending NGINX rate-limit log as it is not required";
+            return false;
+        }
+
+        ostringstream src_ip;
+        ostringstream dst_ip;
+        src_ip << log_info[LogInfo::SOURCE];
+        dst_ip << log_info[LogInfo::DESTINATION_IP];
+
+        ReportIS::Severity log_severity = ReportIS::Severity::MEDIUM;
+        ReportIS::Priority log_priority = ReportIS::Priority::MEDIUM;
+
+        LogGen log = rate_limit_trigger(
+            event_name,
+            LogTriggerConf::SecurityType::AccessControl,
+            log_severity,
+            log_priority,
+            true, // is drop
+            LogField("practiceType", "Rate Limit"),
+            ReportIS::Tags::RATE_LIMIT
+        );
+
+        for (LogInfo field : makeRange<LogInfo>()) {
+            Maybe<string> string_field = convertLogFieldToString(field);
+            if (!string_field.ok()) {
+                dbgDebug(D_NGINX_MESSAGE_READER) << "Enum field was not converted: " <<  string_field.getErr();
+                return false;
+            }
+            
+            if (
+                field == LogInfo::HOST ||
+                field == LogInfo::URI ||
+                field == LogInfo::HTTP_METHOD ||
+                field == LogInfo::SOURCE ||
+                field == LogInfo::DESTINATION_IP ||
+                field == LogInfo::ASSET_ID ||
+                field == LogInfo::ASSET_NAME ||
+                field == LogInfo::RESPONSE_CODE
+            ) {
+                if (!log_info[field].empty()) {
+                    log << LogField(string_field.unpack(), log_info[field]);
+                    continue;
+                }
+            }
+
+            if (field == LogInfo::DESTINATION_PORT) {
+                try {
+                    int numeric_dst_port = stoi(log_info[field]);
+                    log << LogField(string_field.unpack(), numeric_dst_port);
+                } catch (const exception &e) {
+                    dbgWarning(D_NGINX_MESSAGE_READER)
+                        << "Unable to convert dst port: "
+                        << log_info[field]
+                        << " to numberic value. Error: "
+                        << e.what();
+                }
+            }
+        }
+
+        return true;
+    }
+
+    Maybe<string>
+    convertLogFieldToString(LogInfo field)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        switch (field) {
+            case LogInfo::HTTP_METHOD:
+                return string("httpMethod");
+            case LogInfo::URI:
+                return string("httpUriPath");
+            case LogInfo::RESPONSE_CODE:
+                return string("httpResponseCode");
+            case LogInfo::HOST:
+                return string("httpHostName");
+            case LogInfo::SOURCE:
+                return string("httpSourceId");
+            case LogInfo::DESTINATION_IP:
+                return string("destinationIp");
+            case LogInfo::DESTINATION_PORT:
+                return string("destinationPort");
+            case LogInfo::ASSET_ID:
+                return string("assetId");
+            case LogInfo::ASSET_NAME:
+                return string("assetName");
+            case LogInfo::EVENT_MESSAGE:
+                return string("httpResponseBody");
+            case LogInfo::RULE_ID:
+                return string("ruleId");
+            case LogInfo::RULE_NAME:
+                return string("ruleName");
+            case LogInfo::COUNT:
+                dbgError(D_NGINX_MESSAGE_READER) << "LogInfo::COUNT is not allowed";
+                return genError("LogInfo::COUNT is not allowed");
+        }
+        dbgError(D_NGINX_MESSAGE_READER) << "No Enum found, int value: " << static_cast<int>(field);
+        return genError("No Enum found");
+    }
+
+    static vector<string>
+    separateLogs(const string &raw_logs_to_parse)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "separating logs. logs: " << raw_logs_to_parse;
+        dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs start of function. Logs to parse: " << raw_logs_to_parse;
+        boost::smatch matcher;
+        vector<string> logs;
+
+        if (raw_logs_to_parse.empty()) return logs;
+
+        size_t pos = 0;
+        while (NGEN::Regex::regexSearch(__FILE__, __LINE__, raw_logs_to_parse.substr(pos), matcher, syslog_regex)) {
+            if (pos == 0) {
+                dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs pos = 0";
+                pos++;
+                continue;
+            }
+            auto log_length = matcher.position();
+            logs.push_back(raw_logs_to_parse.substr(pos - 1, log_length));
+
+            pos += log_length + 1;
+        }
+        logs.push_back(raw_logs_to_parse.substr(pos - 1));
+        dbgTrace(D_NGINX_MESSAGE_READER) << "separateLogs end of function";
+
+        return logs;
+    }
+
+    static pair<string, string>
+    parseErrorLogRequestField(const string &request)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "parsing request field. request: " << request;
+        string formatted_request = request;
+        vector<string> result;
+        boost::erase_all(formatted_request, "\"");
+        boost::erase_all(formatted_request, "\n");
+        boost::split(result, formatted_request, boost::is_any_of(" "), boost::token_compress_on);
+
+        const int http_method_index = 1;
+        const int uri_index = 2;
+        return pair<string, string>(result[http_method_index], result[uri_index]);
+    }
+
+    static string
+    parseErrorLogField(const string &field)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "parsing error log field " << field;
+        string formatted_field = field;
+        vector<string> result;
+        boost::erase_all(formatted_field, "\"");
+        boost::erase_all(formatted_field, "\n");
+        boost::split(result, formatted_field, boost::is_any_of(" "), boost::token_compress_on);
+
+        const int field_index = 1;
+        return result[field_index];
+    }
+
+    void
+    addContextFieldsToLogInfo(EnumArray<LogInfo, string> &log_info)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+        ScopedContext ctx;
+
+        try {
+            ctx.registerValue<uint16_t>(
+                HttpTransactionData::listening_port_ctx,
+                static_cast<uint16_t>(stoi(log_info[LogInfo::DESTINATION_PORT]))
+            );
+        } catch (const exception &e) {
+            dbgError(D_NGINX_MESSAGE_READER) << "Failed register values for context " << e.what();
+        }
+        ctx.registerValue<string>(HttpTransactionData::host_name_ctx, log_info[LogInfo::HOST]);
+        ctx.registerValue<string>(HttpTransactionData::uri_ctx, log_info[LogInfo::URI]);
+        auto rule_by_ctx = getConfiguration<BasicRuleConfig>("rulebase", "rulesConfig");
+        if (!rule_by_ctx.ok()) {
+            dbgWarning(D_NGINX_MESSAGE_READER)
+                << "AssetId was not found by the given context. Reason: "
+                << rule_by_ctx.getErr();
+            return;
+        }
+
+        BasicRuleConfig context = rule_by_ctx.unpack();
+        log_info[LogInfo::ASSET_ID] = context.getAssetId();
+        log_info[LogInfo::ASSET_NAME] = context.getAssetName();
+        log_info[LogInfo::RULE_ID] = context.getRuleId();
+        log_info[LogInfo::RULE_NAME] = context.getRuleName();
+    }
+
+    Maybe<EnumArray<LogInfo, string>>
+    parseErrorLog(const string &log_line)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Handling log line:" << log_line;
+        string port;
+        EnumArray<LogInfo, string> log_info(EnumArray<LogInfo, string>::Fill(), string(""));
+
+        boost::smatch matcher;
+        vector<string> result;
+        if (
+            !NGEN::Regex::regexSearch(
+                __FILE__,
+                __LINE__,
+                log_line,
+                matcher,
+                isAlertErrorLog(log_line) ? alert_log_regex : error_log_regex
+            )
+        ) {
+            dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
+            return genError("Unexpected nginx log format");
+        }
+
+        const int event_message_index = 6;
+        const int source_index = 7;
+        const int request_index = 9;
+        const int host_index = 11;
+        string host = string(matcher[host_index].first, matcher[host_index].second);
+        string source = string(matcher[source_index].first, matcher[source_index].second);
+        string event_message = string(matcher[event_message_index].first, matcher[event_message_index].second);
+        string request = string(matcher[request_index].first, matcher[request_index].second);
+
+        host = parseErrorLogField(host);
+        source = parseErrorLogField(source);
+        pair<string, string> parsed_request = parseErrorLogRequestField(request);
+        string http_method = parsed_request.first;
+        string uri = parsed_request.second;
+
+        if (NGEN::Regex::regexSearch(__FILE__, __LINE__, host, matcher, socket_address_regex)) {
+            int host_index = 1;
+            int port_index = 2;
+            host = string(matcher[host_index].first, matcher[host_index].second);
+            port = string(matcher[port_index].first, matcher[port_index].second);
+        } else if (NGEN::Regex::regexSearch(__FILE__, __LINE__, host, matcher, boost::regex("https://"))) {
+            port = "443";
+        } else {
+            port = "80";
+        }
+
+        log_info[LogInfo::HOST] = host;
+        log_info[LogInfo::URI] = uri;
+        log_info[LogInfo::RESPONSE_CODE] = "500";
+        log_info[LogInfo::HTTP_METHOD] = http_method;
+        log_info[LogInfo::SOURCE] = source;
+        log_info[LogInfo::DESTINATION_IP] = host;
+        log_info[LogInfo::DESTINATION_PORT] = port;
+        log_info[LogInfo::EVENT_MESSAGE] = event_message;
+
+        addContextFieldsToLogInfo(log_info);
+
+        if (!validateLog(log_info)) {
+            dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
+            return genError("Unexpected nginx log format");
+        }
+
+        return log_info;
+    }
+
+    Maybe<EnumArray<LogInfo, string>>
+    parseAccessLog(const string &log_line)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Parsing log line: " << log_line;
+        string formatted_log = log_line;
+        EnumArray<LogInfo, string> log_info(EnumArray<LogInfo, string>::Fill(), string(""));
+        vector<string> result;
+        boost::erase_all(formatted_log, "\"");
+        boost::erase_all(formatted_log, "\n");
+        boost::split(result, formatted_log, boost::is_any_of(" "), boost::token_compress_on);
+
+        const int valid_log_size = 20;
+
+        if (result.size() < valid_log_size) {
+            dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
+            return genError("Unexpected nginx log format");
+        }
+
+        const int host_index = 6;
+        const int host_port_index = 7;
+        const int http_method_index = 13;
+        const int uri_index = 14;
+        const int response_cod_index = 16;
+        const int source_index = 8;
+
+        log_info[LogInfo::HOST] = result[host_index];
+        log_info[LogInfo::URI] = result[uri_index];
+        log_info[LogInfo::RESPONSE_CODE] = result[response_cod_index];
+        log_info[LogInfo::HTTP_METHOD] = result[http_method_index];
+        log_info[LogInfo::SOURCE] = result[source_index];
+        log_info[LogInfo::DESTINATION_IP] = result[host_index];
+        log_info[LogInfo::DESTINATION_PORT] = result[host_port_index];
+        log_info[LogInfo::EVENT_MESSAGE] = "Invalid request or incorrect reverse proxy configuration - "
+        "Request dropped. Please check the reverse proxy configuration of your relevant assets";
+
+        addContextFieldsToLogInfo(log_info);
+
+        if (!validateLog(log_info)) {
+            dbgWarning(D_NGINX_MESSAGE_READER) << "Unexpected nginx log format";
+            return genError("Unexpected nginx log format");
+        }
+        return log_info;
+    }
+
+    static bool
+    validateLog(const EnumArray<LogInfo, string> &log_info)
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER);
+
+        boost::smatch matcher;
+        if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::HOST], matcher, server_regex)) {
+            dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate server (Host): " << log_info[LogInfo::HOST];
+            return false;
+        }
+        if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::URI], matcher, uri_regex)) {
+            dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate Uri: " << log_info[LogInfo::URI];
+            return false;
+        }
+
+        if (
+            !NGEN::Regex::regexSearch(
+                __FILE__,
+                __LINE__,
+                log_info[LogInfo::RESPONSE_CODE],
+                matcher, response_code_regex
+            )
+        ) {
+            dbgTrace(D_NGINX_MESSAGE_READER)
+                << "Could not validate response code: "
+                << log_info[LogInfo::RESPONSE_CODE];
+            return false;
+        }
+
+        if (
+            !NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::HTTP_METHOD], matcher, http_method_regex)
+        ) {
+            dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate HTTP method: " << log_info[LogInfo::HTTP_METHOD];
+            return false;
+        }
+
+        if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::DESTINATION_PORT], matcher, port_regex)) {
+            dbgTrace(D_NGINX_MESSAGE_READER)
+                << "Could not validate destination port : "
+                << log_info[LogInfo::DESTINATION_PORT];
+            return false;
+        }
+
+        if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, log_info[LogInfo::SOURCE], matcher, server_regex)) {
+            dbgTrace(D_NGINX_MESSAGE_READER) << "Could not validate source : " << log_info[LogInfo::SOURCE];
+            return false;
+        }
+
+        return true;
+    }
+
+    Maybe<string>
+    getLogsFromSocket(const I_Socket::socketFd &client_socket) const
+    {
+        dbgFlow(D_NGINX_MESSAGE_READER) << "Reading logs from socket. fd: " << client_socket;
+        I_Socket *i_socket = Singleton::Consume<I_Socket>::by<NginxMessageReader>();
+        Maybe<vector<char>> raw_log_data = i_socket->receiveData(client_socket, 0, false);
+        if (!raw_log_data.ok()) {
+            dbgWarning(D_NGINX_MESSAGE_READER) << "Error receiving data from socket";
+            return genError("Error receiving data from socket");
+        }
+
+        string raw_log(raw_log_data.unpack().begin(), raw_log_data.unpack().end());
+        return move(raw_log);
+    }
+
+    I_Socket::socketFd syslog_server_socket = -1;
+    string rate_limit_status_code = "429";
+};
+
+NginxMessageReader::NginxMessageReader() : Component("NginxMessageReader"), pimpl(make_unique<Impl>()) {}
+
+NginxMessageReader::~NginxMessageReader() {}
+
+void
+NginxMessageReader::init()
+{
+    pimpl->init();
+}
+
+void
+NginxMessageReader::preload()
+{
+    pimpl->preload();
+}
+
+void
+NginxMessageReader::fini()
+{
+    pimpl->fini();
+}

components/security_apps/CMakeLists.txt

@@ -5,3 +5,4 @@ add_subdirectory(local_policy_mgmt_gen)
 add_subdirectory(orchestration)
 add_subdirectory(rate_limit)
 add_subdirectory(waap)
+add_subdirectory(central_nginx_manager)

components/security_apps/central_nginx_manager/CMakeLists.txt

@@ -0,0 +1,3 @@
+include_directories(include)
+
+add_library(central_nginx_manager central_nginx_manager.cc lets_encrypt_listener.cc)

components/security_apps/central_nginx_manager/central_nginx_manager.cc

@@ -0,0 +1,390 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "central_nginx_manager.h"
+#include "lets_encrypt_listener.h"
+
+#include <string>
+#include <vector>
+#include <cereal/external/base64.hpp>
+
+#include "debug.h"
+#include "config.h"
+#include "rest.h"
+#include "log_generator.h"
+#include "nginx_utils.h"
+#include "agent_core_utilities.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+class CentralNginxConfig
+{
+public:
+    void load(cereal::JSONInputArchive &ar)
+    {
+        try {
+            string nginx_conf_base64;
+            ar(cereal::make_nvp("id", file_id));
+            ar(cereal::make_nvp("name", file_name));
+            ar(cereal::make_nvp("data", nginx_conf_base64));
+            nginx_conf_content = cereal::base64::decode(nginx_conf_base64);
+            central_nginx_conf_path = getCentralNginxConfPath();
+            shared_config_path = getSharedConfigPath();
+            if (!nginx_conf_content.empty()) configureCentralNginx();
+        } catch (const cereal::Exception &e) {
+            dbgDebug(D_NGINX_MANAGER) << "Could not load Central Management Config JSON. Error: " << e.what();
+            ar.setNextName(nullptr);
+        }
+    }
+
+    const string & getFileId() const { return file_id; }
+    const string & getFileName() const { return file_name; }
+    const string & getFileContent() const { return nginx_conf_content; }
+
+    static string
+    getCentralNginxConfPath()
+    {
+        string central_nginx_conf_path = getProfileAgentSettingWithDefault<string>(
+            string("/tmp/central_nginx.conf"),
+            "centralNginxManagement.confDownloadPath"
+        );
+        dbgInfo(D_NGINX_MANAGER) << "Central NGINX configuration path: " << central_nginx_conf_path;
+
+        return central_nginx_conf_path;
+    }
+
+    static string
+    getSharedConfigPath()
+    {
+        string central_shared_conf_path = getConfigurationWithDefault<string>(
+            "/etc/cp/conf",
+            "Config Component",
+            "configuration path"
+        );
+        central_shared_conf_path += "/centralNginxManager/shared/central_nginx_shared.conf";
+        dbgInfo(D_NGINX_MANAGER) << "Shared NGINX configuration path: " << central_shared_conf_path;
+
+        return central_shared_conf_path;
+    }
+
+private:
+    void
+    loadAttachmentModule()
+    {
+        string attachment_module_path = NginxUtils::getModulesPath() + "/ngx_cp_attachment_module.so";
+        if (!NGEN::Filesystem::exists(attachment_module_path)) {
+            dbgTrace(D_NGINX_MANAGER) << "Attachment module " << attachment_module_path << " does not exist";
+            return;
+        }
+
+        string attachment_module_conf = "load_module " + attachment_module_path + ";";
+        if (nginx_conf_content.find(attachment_module_conf) != string::npos) {
+            dbgTrace(D_NGINX_MANAGER) << "Attachment module " << attachment_module_path << " already loaded";
+            return;
+        }
+
+        nginx_conf_content = attachment_module_conf + "\n" + nginx_conf_content;
+    }
+
+    Maybe<void>
+    loadSharedDirective(const string &directive)
+    {
+        dbgFlow(D_NGINX_MANAGER) << "Loading shared directive into the servers " << directive;
+
+        if (!NGEN::Filesystem::copyFile(shared_config_path, shared_config_path + ".bak", true)) {
+            return genError("Could not create a backup of the shared NGINX configuration file");
+        }
+
+        ifstream shared_config(shared_config_path);
+        if (!shared_config.is_open()) {
+            return genError("Could not open shared NGINX configuration file");
+        }
+
+        string shared_config_content((istreambuf_iterator<char>(shared_config)), istreambuf_iterator<char>());
+        shared_config.close();
+
+        if (shared_config_content.find(directive) != string::npos) {
+            dbgTrace(D_NGINX_MANAGER) << "Shared directive " << directive << " already loaded";
+            return {};
+        }
+
+        ofstream new_shared_config(shared_config_path, ios::app);
+        if (!new_shared_config.is_open()) {
+            return genError("Could not open shared NGINX configuration file");
+        }
+
+        dbgTrace(D_NGINX_MANAGER) << "Adding shared directive " << directive;
+        new_shared_config << directive << "\n";
+        new_shared_config.close();
+
+        auto validation = NginxUtils::validateNginxConf(central_nginx_conf_path);
+        if (!validation.ok()) {
+            if (!NGEN::Filesystem::copyFile(shared_config_path + ".bak", shared_config_path, true)) {
+                return genError("Could not restore the shared NGINX configuration file");
+            }
+            return genError("Could not validate shared NGINX configuration file. Error: " + validation.getErr());
+        }
+
+        return {};
+    }
+
+    Maybe<void>
+    loadSharedConfig()
+    {
+        dbgFlow(D_NGINX_MANAGER) << "Loading shared configuration into the servers";
+
+        ofstream shared_config(shared_config_path);
+        if (!shared_config.is_open()) {
+            return genError("Could not create shared NGINX configuration file");
+        }
+        shared_config.close();
+
+        string shared_config_directive = "include " + shared_config_path + ";\n";
+        boost::regex server_regex("server\\s*\\{");
+        nginx_conf_content = NGEN::Regex::regexReplace(
+            __FILE__,
+            __LINE__,
+            nginx_conf_content,
+            server_regex,
+            "server {\n" + shared_config_directive
+        );
+
+        ofstream nginx_conf_file(central_nginx_conf_path);
+        if (!nginx_conf_file.is_open()) {
+            return genError("Could not open a temporary central NGINX configuration file");
+        }
+        nginx_conf_file << nginx_conf_content;
+        nginx_conf_file.close();
+
+        auto validation = NginxUtils::validateNginxConf(central_nginx_conf_path);
+        if (!validation.ok()) {
+            return genError("Could not validate central NGINX configuration file. Error: " + validation.getErr());
+        }
+
+        return {};
+    }
+
+    Maybe<void>
+    configureSyslog()
+    {
+        if (!getProfileAgentSettingWithDefault<bool>(true, "centralNginxManagement.syslogEnabled")) {
+            dbgTrace(D_NGINX_MANAGER) << "Syslog is disabled via settings";
+            return {};
+        }
+
+        string syslog_directive = "error_log syslog:server=127.0.0.1:1514 warn;";
+        auto load_shared_directive_result = loadSharedDirective(syslog_directive);
+        if (!load_shared_directive_result.ok()) {
+            return genError("Could not configure syslog directive, error: " + load_shared_directive_result.getErr());
+        }
+
+        return {};
+    }
+
+    Maybe<void>
+    saveBaseCentralNginxConf()
+    {
+        ofstream central_nginx_conf_base_file(central_nginx_conf_path + ".base");
+        if (!central_nginx_conf_base_file.is_open()) {
+            return genError("Could not open a temporary central NGINX configuration file");
+        }
+        central_nginx_conf_base_file << nginx_conf_content;
+        central_nginx_conf_base_file.close();
+
+        return {};
+    }
+
+    void
+    configureCentralNginx()
+    {
+        loadAttachmentModule();
+        auto save_base_nginx_conf = saveBaseCentralNginxConf();
+        if (!save_base_nginx_conf.ok()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not save base NGINX configuration. Error: "
+                << save_base_nginx_conf.getErr();
+            return;
+        }
+
+        string nginx_conf_content_backup = nginx_conf_content;
+        auto shared_config_result = loadSharedConfig();
+        if (!shared_config_result.ok()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not load shared configuration. Error: "
+                << shared_config_result.getErr();
+            nginx_conf_content = nginx_conf_content_backup;
+            return;
+        }
+
+        auto syslog_result = configureSyslog();
+        if (!syslog_result.ok()) {
+            dbgWarning(D_NGINX_MANAGER) << "Could not configure syslog. Error: " << syslog_result.getErr();
+        }
+    }
+
+    string file_id;
+    string file_name;
+    string nginx_conf_content;
+    string central_nginx_conf_path;
+    string shared_config_path;
+};
+
+class CentralNginxManager::Impl
+{
+public:
+    void
+    init()
+    {
+        dbgInfo(D_NGINX_MANAGER) << "Starting Central NGINX Manager";
+
+        string main_nginx_conf_path = NginxUtils::getMainNginxConfPath();
+        if (
+            NGEN::Filesystem::exists(main_nginx_conf_path)
+            && !NGEN::Filesystem::exists(main_nginx_conf_path + ".orig")
+        ) {
+            dbgInfo(D_NGINX_MANAGER) << "Creating a backup of the original main NGINX configuration file";
+            NGEN::Filesystem::copyFile(main_nginx_conf_path, main_nginx_conf_path + ".orig", true);
+        }
+
+        i_mainloop = Singleton::Consume<I_MainLoop>::by<CentralNginxManager>();
+        if (!lets_encrypt_listener.init()) {
+            dbgWarning(D_NGINX_MANAGER) << "Could not start Lets Encrypt Listener, scheduling retry";
+            i_mainloop->addOneTimeRoutine(
+                I_MainLoop::RoutineType::System,
+                [this] ()
+                {
+                    while(!lets_encrypt_listener.init()) {
+                        dbgWarning(D_NGINX_MANAGER) << "Could not start Lets Encrypt Listener, will retry";
+                        i_mainloop->yield(chrono::seconds(5));
+                    }
+                },
+                "Lets Encrypt Listener initializer",
+                false
+            );
+        }
+    }
+
+    void
+    loadPolicy()
+    {
+        auto central_nginx_config = getSetting<vector<CentralNginxConfig>>("centralNginxManagement");
+        if (!central_nginx_config.ok() || central_nginx_config.unpack().empty()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not load Central NGINX Management settings. Error: "
+                << central_nginx_config.getErr();
+            return;
+        }
+
+        auto &config = central_nginx_config.unpack().front();
+        if (config.getFileContent().empty()) {
+            dbgWarning(D_NGINX_MANAGER) << "Empty NGINX configuration file";
+            return;
+        }
+
+        dbgTrace(D_NGINX_MANAGER)
+            << "Handling Central NGINX Management settings: "
+            << config.getFileId()
+            << ", "
+            << config.getFileName()
+            << ", "
+            << config.getFileContent();
+
+        string central_nginx_conf_path = config.getCentralNginxConfPath();
+        ofstream central_nginx_conf_file(central_nginx_conf_path);
+        if (!central_nginx_conf_file.is_open()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not open central NGINX configuration file: "
+                << central_nginx_conf_path;
+            return;
+        }
+        central_nginx_conf_file << config.getFileContent();
+        central_nginx_conf_file.close();
+
+        auto validation_result = NginxUtils::validateNginxConf(central_nginx_conf_path);
+        if (!validation_result.ok()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not validate central NGINX configuration file. Error: "
+                << validation_result.getErr();
+            logError(validation_result.getErr());
+            return;
+        }
+
+        dbgTrace(D_NGINX_MANAGER) << "Validated central NGINX configuration file";
+
+        auto reload_result = NginxUtils::reloadNginx(central_nginx_conf_path);
+        if (!reload_result.ok()) {
+            dbgWarning(D_NGINX_MANAGER)
+                << "Could not reload central NGINX configuration. Error: "
+                << reload_result.getErr();
+            logError("Could not reload central NGINX configuration. Error: " + reload_result.getErr());
+            return;
+        }
+    }
+
+    void
+    fini()
+    {
+        string central_nginx_base_path = CentralNginxConfig::getCentralNginxConfPath() + ".base";
+        if (!NGEN::Filesystem::exists(central_nginx_base_path)) {
+            dbgWarning(D_NGINX_MANAGER) << "Could not find base NGINX configuration file: " << central_nginx_base_path;
+            return;
+        }
+
+        NginxUtils::reloadNginx(central_nginx_base_path);
+    }
+
+private:
+    void
+    logError(const string &error)
+    {
+        LogGen log(
+            error,
+            ReportIS::Audience::SECURITY,
+            ReportIS::Severity::CRITICAL,
+            ReportIS::Priority::HIGH,
+            ReportIS::Tags::POLICY_INSTALLATION
+        );
+    }
+
+    I_MainLoop *i_mainloop = nullptr;
+    LetsEncryptListener lets_encrypt_listener;
+};
+
+CentralNginxManager::CentralNginxManager()
+    :
+    Component("Central NGINX Manager"),
+    pimpl(make_unique<CentralNginxManager::Impl>()) {}
+
+CentralNginxManager::~CentralNginxManager() {}
+
+void
+CentralNginxManager::init()
+{
+    pimpl->init();
+}
+
+void
+CentralNginxManager::fini()
+{
+    pimpl->fini();
+}
+
+void
+CentralNginxManager::preload()
+{
+    registerExpectedSetting<vector<CentralNginxConfig>>("centralNginxManagement");
+    registerExpectedConfiguration<string>("Config Component", "configuration path");
+    registerConfigLoadCb([this]() { pimpl->loadPolicy(); });
+}

components/security_apps/central_nginx_manager/include/lets_encrypt_listener.h

@@ -0,0 +1,30 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __LETS_ENCRYPT_HANDLER_H__
+#define __LETS_ENCRYPT_HANDLER_H__
+
+#include <string>
+
+#include "maybe_res.h"
+
+class LetsEncryptListener
+{
+public:
+    bool init();
+
+private:
+    Maybe<std::string> getChallengeValue(const std::string &uri) const;
+};
+
+#endif // __LETS_ENCRYPT_HANDLER_H__

components/security_apps/central_nginx_manager/lets_encrypt_listener.cc

@@ -0,0 +1,76 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "lets_encrypt_listener.h"
+
+#include <string>
+
+#include "central_nginx_manager.h"
+#include "debug.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+bool
+LetsEncryptListener::init()
+{
+    dbgInfo(D_NGINX_MANAGER) << "Starting Lets Encrypt Listener";
+    return Singleton::Consume<I_RestApi>::by<CentralNginxManager>()->addWildcardGetCall(
+        ".well-known/acme-challenge/",
+        [&] (const string &uri) -> string
+        {
+            Maybe<string> maybe_challenge_value = getChallengeValue(uri);
+            if (!maybe_challenge_value.ok()) {
+                dbgWarning(D_NGINX_MANAGER)
+                    << "Could not get challenge value for uri: "
+                    << uri
+                    << ", error: "
+                    << maybe_challenge_value.getErr();
+                return string{""};
+            };
+
+            dbgTrace(D_NGINX_MANAGER) << "Got challenge value: " << maybe_challenge_value.unpack();
+            return maybe_challenge_value.unpack();
+        }
+    );
+}
+
+Maybe<string>
+LetsEncryptListener::getChallengeValue(const string &uri) const
+{
+    string challenge_key = uri.substr(uri.find_last_of('/') + 1);
+    string api_query = "/api/lets-encrypt-challenge?http_challenge_key=" + challenge_key;
+
+    dbgInfo(D_NGINX_MANAGER) << "Getting challenge value via: " << api_query;
+
+    MessageMetadata md;
+    md.insertHeader("X-Tenant-Id", Singleton::Consume<I_AgentDetails>::by<CentralNginxManager>()->getTenantId());
+    Maybe<HTTPResponse, HTTPResponse> maybe_http_challenge_value =
+        Singleton::Consume<I_Messaging>::by<CentralNginxManager>()->sendSyncMessage(
+            HTTPMethod::GET,
+            api_query,
+            string("{}"),
+            MessageCategory::GENERIC,
+            md
+        );
+
+    if (!maybe_http_challenge_value.ok()) return genError(maybe_http_challenge_value.getErr().getBody());
+
+    string challenge_value = maybe_http_challenge_value.unpack().getBody();
+    if (!challenge_value.empty() && challenge_value.front() == '"' && challenge_value.back() == '"') {
+        challenge_value = challenge_value.substr(1, challenge_value.size() - 2);
+    }
+
+    return challenge_value;
+}

components/security_apps/orchestration/health_check_manager/health_check_manager.cc

@@ -266,10 +266,10 @@ private:
             case OrchestrationStatusFieldType::COUNT : return "Count";
         }
 
-        dbgAssert(false)
+        dbgAssertOpt(false)
             << AlertInfo(AlertTeam::CORE, "orchestration health")
             << "Trying to convert unknown orchestration status field to string.";
-        return "";
+        return "Unknown Field";
     }
 
     HealthCheckStatus
@@ -282,7 +282,7 @@ private:
             case UpdatesProcessResult::DEGRADED : return HealthCheckStatus::DEGRADED;
         }
 
-        dbgAssert(false)
+        dbgAssertOpt(false)
             << AlertInfo(AlertTeam::CORE, "orchestration health")
             << "Trying to convert unknown update process result field to health check status.";
         return HealthCheckStatus::IGNORED;

components/security_apps/orchestration/modules/orchestration_status.cc

@@ -429,7 +429,7 @@ public:
                 status.insertServiceSetting(service_name, path);
                 return;
             case OrchestrationStatusConfigType::MANIFEST:
-                dbgAssert(false)
+                dbgAssertOpt(false)
                     << AlertInfo(AlertTeam::CORE, "sesrvice configuration")
                     << "Manifest is not a service configuration file type";
                 break;
@@ -438,7 +438,9 @@ public:
             case OrchestrationStatusConfigType::COUNT:
                 break;
         }
-        dbgAssert(false) << AlertInfo(AlertTeam::CORE, "sesrvice configuration") << "Unknown configuration file type";
+        dbgAssertOpt(false)
+            << AlertInfo(AlertTeam::CORE, "service configuration")
+            << "Unknown configuration file type";
     }
 
     void

components/security_apps/orchestration/orchestration_comp.cc

@@ -1587,6 +1587,7 @@ private:
         }
 
         setDelayedUpgradeTime();
+
         while (true) {
             Singleton::Consume<I_Environment>::by<OrchestrationComp>()->startNewTrace(false);
             if (shouldReportAgentDetailsMetadata()) {
@@ -1695,13 +1696,19 @@ private:
         auto backup_installation_file = current_installation_file + backup_ext;
         auto temp_ext = getConfigurationWithDefault<string>("_temp", "orchestration", "Temp file extension");
 
-        dbgAssert(i_orchestration_tools->doesFileExist(backup_installation_file))
+        if (!i_orchestration_tools->doesFileExist(backup_installation_file)) {
+            dbgAssertOpt(false)
                 << AlertInfo(AlertTeam::CORE, "orchestration backup")
                 << "There is no backup installation package";
+            return;
+        }
 
-        dbgAssert(i_orchestration_tools->copyFile(backup_installation_file, current_installation_file))
+        if (!i_orchestration_tools->copyFile(backup_installation_file, current_installation_file)) {
+            dbgAssertOpt(false)
                 << AlertInfo(AlertTeam::CORE, "orchestration backup")
                 << "Failed to copy backup installation package";
+            return;
+        }
 
         // Copy the backup manifest file to the default manifest file path.
         auto manifest_file_path = getConfigurationWithDefault<string>(
@@ -1716,12 +1723,18 @@ private:
 
         auto package_handler = Singleton::Consume<I_PackageHandler>::by<OrchestrationComp>();
         // Install the backup orchestration service installation package.
-        dbgAssert(package_handler->preInstallPackage(service_name, current_installation_file))
+        if (!package_handler->preInstallPackage(service_name, current_installation_file)) {
+            dbgAssertOpt(false)
                 << AlertInfo(AlertTeam::CORE, "orchestration backup")
                 << "Failed to restore from backup, pre install test failed";
-        dbgAssert(package_handler->installPackage(service_name, current_installation_file, true))
+            return;
+        }
+        if (!package_handler->installPackage(service_name, current_installation_file, true)) {
+            dbgAssertOpt(false)
                 << AlertInfo(AlertTeam::CORE, "orchestration backup")
                 << "Failed to restore from backup, installation failed";
+            return;
+        }
     }
     // LCOV_EXCL_STOP
 

components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc

@@ -386,7 +386,7 @@ OrchestrationTools::Impl::calculateChecksum(Package::ChecksumTypes checksum_type
         return genError("Error while reading file " + path + ", " + e.what());
     }
 
-    dbgAssert(false)
+    dbgAssertOpt(false)
         << AlertInfo(AlertTeam::CORE, "service configuration")
         << "Checksum type is not supported. Checksum type: "
         << static_cast<unsigned int>(checksum_type);

components/security_apps/orchestration/package_handler/package_handler.cc

@@ -141,11 +141,11 @@ packageHandlerActionsToString(PackageHandlerActions action)
         }
     }
 
-    dbgAssert(false)
+    dbgAssertOpt(false)
         << AlertInfo(AlertTeam::CORE, "service configuration")
         << "Package handler action is not supported. Action: "
         << static_cast<unsigned int>(action);
-    return string();
+    return string("--UNSUPPORTED");
 }
 
 void

components/security_apps/orchestration/update_communication/fog_authenticator.cc

@@ -467,9 +467,9 @@ getDeplymentType()
         case EnvType::COUNT: break;
     }
 
-    dbgAssert(false)
+    dbgAssertOpt(false)
         << AlertInfo(AlertTeam::CORE, "fog communication")
-        << "Failed to get a legitimate deplyment type: "
+        << "Failed to get a legitimate deployment type: "
         << static_cast<uint>(deplyment_type);
     return "Embedded";
 }

components/utils/CMakeLists.txt

@@ -5,3 +5,5 @@ add_subdirectory(ip_utilities)
 add_subdirectory(keywords)
 add_subdirectory(pm)
 add_subdirectory(service_health_status)
+add_subdirectory(nginx_utils)
+add_subdirectory(utilities)

components/utils/nginx_utils/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(nginx_utils nginx_utils.cc)

components/utils/nginx_utils/nginx_utils.cc

@@ -0,0 +1,281 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "nginx_utils.h"
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <dirent.h>
+#include <boost/regex.hpp>
+
+#include "debug.h"
+#include "maybe_res.h"
+#include "config.h"
+#include "agent_core_utilities.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+NginxConfCollector::NginxConfCollector(const string &input_path, const string &output_path)
+        :
+    main_conf_input_path(input_path),
+    main_conf_output_path(output_path)
+{
+    main_conf_directory_path = main_conf_input_path.substr(0, main_conf_input_path.find_last_of('/'));
+}
+
+vector<string>
+NginxConfCollector::expandIncludes(const string &include_pattern) const {
+    vector<string> matching_files;
+    string absolute_include_pattern = include_pattern;
+    string maybe_directory = include_pattern.substr(0, include_pattern.find_last_of('/'));
+    if (!maybe_directory.empty() && maybe_directory.front() != '/') {
+        dbgTrace(D_NGINX_MANAGER) << "Include pattern is a relative path: " << include_pattern;
+        maybe_directory = main_conf_directory_path + '/' + maybe_directory;
+        absolute_include_pattern = main_conf_directory_path + '/' + include_pattern;
+    }
+
+    if (!NGEN::Filesystem::exists(maybe_directory)) {
+        dbgTrace(D_NGINX_MANAGER) << "Include pattern directory/file does not exist: " << maybe_directory;
+        return matching_files;
+    }
+
+    string filename_pattern = absolute_include_pattern.substr(absolute_include_pattern.find_last_of('/') + 1);
+    boost::regex wildcard_regex("\\*");
+    boost::regex pattern(
+        NGEN::Regex::regexReplace(__FILE__, __LINE__, filename_pattern, wildcard_regex, string("[^/]*"))
+    );
+
+    if (!NGEN::Filesystem::isDirectory(maybe_directory)) {
+        dbgTrace(D_NGINX_MANAGER) << "Include pattern is a file: " << absolute_include_pattern;
+        matching_files.push_back(absolute_include_pattern);
+        return matching_files;
+    }
+
+    DIR* dir = opendir(maybe_directory.c_str());
+    if (!dir) {
+        dbgTrace(D_NGINX_MANAGER) << "Could not open directory: " << maybe_directory;
+        return matching_files;
+    }
+
+    struct dirent *entry;
+    while ((entry = readdir(dir)) != nullptr) {
+        if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0) continue;
+
+        if (NGEN::Regex::regexMatch(__FILE__, __LINE__, entry->d_name, pattern)) {
+            matching_files.push_back(maybe_directory + "/" + entry->d_name);
+            dbgTrace(D_NGINX_MANAGER) << "Matched file: " << maybe_directory << '/' << entry->d_name;
+        }
+    }
+    closedir(dir);
+
+    return matching_files;
+}
+
+void
+NginxConfCollector::processConfigFile(const string &path, ostringstream &conf_output, vector<string> &errors) const
+{
+    ifstream file(path);
+    if (!file.is_open()) return;
+
+    string content((istreambuf_iterator<char>(file)), istreambuf_iterator<char>());
+    file.close();
+
+    dbgTrace(D_NGINX_MANAGER) << "Processing file: " << path;
+
+    if (content.empty()) return;
+
+    try {
+        boost::regex include_regex(R"(^\s*include\s+([^;]+);)");
+        boost::smatch match;
+
+        while (NGEN::Regex::regexSearch(__FILE__, __LINE__, content, match, include_regex)) {
+            string include_pattern = match[1].str();
+            include_pattern = NGEN::Strings::trim(include_pattern);
+            dbgTrace(D_NGINX_MANAGER) << "Include pattern: " << include_pattern;
+
+            vector<string> included_files = expandIncludes(include_pattern);
+            if (included_files.empty()) {
+                dbgTrace(D_NGINX_MANAGER) << "No files matched the include pattern: " << include_pattern;
+                content.replace(match.position(), match.length(), "");
+                continue;
+            }
+
+            ostringstream included_content;
+            for (const string &included_file : included_files) {
+                dbgTrace(D_NGINX_MANAGER) << "Processing included file: " << included_file;
+                processConfigFile(included_file, included_content, errors);
+            }
+            content.replace(match.position(), match.length(), included_content.str());
+        }
+    } catch (const boost::regex_error &e) {
+        errors.emplace_back(e.what());
+        return;
+    } catch (const exception &e) {
+        errors.emplace_back(e.what());
+        return;
+    }
+
+    conf_output << content;
+}
+
+Maybe<string>
+NginxConfCollector::generateFullNginxConf() const
+{
+    if (!NGEN::Filesystem::exists(main_conf_input_path)) {
+        return genError("Input file does not exist: " + main_conf_input_path);
+    }
+
+    ostringstream conf_output;
+    vector<string> errors;
+    processConfigFile(main_conf_input_path, conf_output, errors);
+
+    if (!errors.empty()) {
+        for (const string &error : errors) dbgWarning(D_NGINX_MANAGER) << error;
+        return genError("Errors occurred while processing configuration files");
+    }
+
+    ofstream single_nginx_conf_file(main_conf_output_path);
+    if (!single_nginx_conf_file.is_open()) return genError("Could not create output file: " + main_conf_output_path);
+
+    single_nginx_conf_file << conf_output.str();
+    single_nginx_conf_file.close();
+
+    return NGEN::Filesystem::resolveFullPath(main_conf_output_path);
+}
+
+string
+NginxUtils::getMainNginxConfPath()
+{
+    static string main_nginx_conf_path;
+    if (!main_nginx_conf_path.empty()) return main_nginx_conf_path;
+
+    auto main_nginx_conf_path_setting = getProfileAgentSetting<string>("centralNginxManagement.mainConfPath");
+    if (main_nginx_conf_path_setting.ok()) {
+        main_nginx_conf_path = main_nginx_conf_path_setting.unpack();
+        return main_nginx_conf_path;
+    }
+
+    string default_main_nginx_conf_path = "/etc/nginx/nginx.conf";
+    string command = "nginx -V 2>&1";
+    auto result = Singleton::Consume<I_ShellCmd>::by<NginxUtils>()->getExecOutputAndCode(command);
+    if (!result.ok()) return default_main_nginx_conf_path;
+
+    string output = result.unpack().first;
+    boost::regex conf_regex(R"(--conf-path=([^ ]+))");
+    boost::smatch match;
+    if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, output, match, conf_regex)) {
+        main_nginx_conf_path = default_main_nginx_conf_path;
+        return main_nginx_conf_path;
+    }
+
+    string conf_path = match[1].str();
+    conf_path = NGEN::Strings::trim(conf_path);
+    if (conf_path.empty()) {
+        main_nginx_conf_path = default_main_nginx_conf_path;
+        return main_nginx_conf_path;
+    }
+
+    main_nginx_conf_path = conf_path;
+    return main_nginx_conf_path;
+}
+
+string
+NginxUtils::getModulesPath()
+{
+    static string main_modules_path;
+    if (!main_modules_path.empty()) return main_modules_path;
+
+    auto modules_path_setting = getProfileAgentSetting<string>("centralNginxManagement.modulesPath");
+    if (modules_path_setting.ok()) {
+        main_modules_path = modules_path_setting.unpack();
+        return main_modules_path;
+    }
+
+    string default_modules_path = "/usr/share/nginx/modules";
+    string command = "nginx -V 2>&1";
+    auto result = Singleton::Consume<I_ShellCmd>::by<NginxUtils>()->getExecOutputAndCode(command);
+    if (!result.ok()) return default_modules_path;
+
+    string output = result.unpack().first;
+    boost::regex modules_regex(R"(--modules-path=([^ ]+))");
+    boost::smatch match;
+    if (!NGEN::Regex::regexSearch(__FILE__, __LINE__, output, match, modules_regex)) {
+        main_modules_path = default_modules_path;
+        return main_modules_path;
+    }
+
+    string modules_path = match[1].str();
+    modules_path = NGEN::Strings::trim(modules_path);
+    if (modules_path.empty()) {
+        main_modules_path = default_modules_path;
+        return main_modules_path;
+    }
+
+    main_modules_path = modules_path;
+    return modules_path;
+}
+
+Maybe<void>
+NginxUtils::validateNginxConf(const string &nginx_conf_path)
+{
+    dbgTrace(D_NGINX_MANAGER) << "Validating NGINX configuration file: " << nginx_conf_path;
+    if (!NGEN::Filesystem::exists(nginx_conf_path)) return genError("Nginx configuration file does not exist");
+
+    string command = "nginx -t -c " + nginx_conf_path + " 2>&1";
+    auto result = Singleton::Consume<I_ShellCmd>::by<NginxUtils>()->getExecOutputAndCode(command);
+    if (!result.ok()) return genError(result.getErr());
+    if (result.unpack().second != 0) return genError(result.unpack().first);
+
+    dbgTrace(D_NGINX_MANAGER) << "NGINX configuration file is valid";
+
+    return {};
+}
+
+Maybe<void>
+NginxUtils::reloadNginx(const string &nginx_conf_path)
+{
+    dbgTrace(D_NGINX_MANAGER) << "Applying and reloading new NGINX configuration file: " << nginx_conf_path;
+    string main_nginx_conf_path = getMainNginxConfPath();
+
+    string backup_conf_path = main_nginx_conf_path + ".bak";
+    if (
+        NGEN::Filesystem::exists(main_nginx_conf_path)
+        && !NGEN::Filesystem::copyFile(main_nginx_conf_path, backup_conf_path, true)
+    ) {
+        return genError("Could not create backup of NGINX configuration file");
+    }
+
+    dbgTrace(D_NGINX_MANAGER) << "Copying new NGINX configuration file to: " << main_nginx_conf_path;
+    if (!NGEN::Filesystem::copyFile(nginx_conf_path, main_nginx_conf_path, true)) {
+        return genError("Could not copy new NGINX configuration file");
+    }
+
+    string command = "nginx -s reload 2>&1";
+    auto result = Singleton::Consume<I_ShellCmd>::by<NginxUtils>()->getExecOutputAndCode(command);
+    if (!result.ok() || result.unpack().second != 0) {
+        if (!NGEN::Filesystem::copyFile(backup_conf_path, main_nginx_conf_path, true)) {
+            return genError("Could not restore backup of NGINX configuration file");
+        }
+        dbgTrace(D_NGINX_MANAGER) << "Successfully restored backup of NGINX configuration file";
+        return result.ok() ? genError(result.unpack().first) : genError(result.getErr());
+    }
+
+    dbgInfo(D_NGINX_MANAGER) << "Successfully reloaded NGINX configuration file";
+
+    return {};
+}

components/utils/pm/debugpm.cc

@@ -46,7 +46,7 @@ panicCFmt(const string &func, uint line, const char *fmt, ...)
 {
     va_list va;
     va_start(va, fmt);
-    Debug("PM", func, line).getStreamAggr() << CFmtPrinter(fmt, va);
+    Debug("PM", func, line, true).getStreamAggr() << CFmtPrinter(fmt, va);
     va_end(va);
 }
 

components/utils/utilities/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory(nginx_conf_collector)

components/utils/utilities/nginx_conf_collector/CMakeLists.txt

@@ -0,0 +1,37 @@
+include_directories(${PROJECT_SOURCE_DIR}/core/include/)
+
+link_directories(${Boost_LIBRARY_DIRS})
+link_directories(${ZLIB_ROOT}/lib)
+
+link_directories(${ZLIB_ROOT}/lib)
+link_directories(${CMAKE_BINARY_DIR}/core)
+link_directories(${CMAKE_BINARY_DIR}/core/compression)
+
+SET(EXECUTABLE_NAME "nginx_conf_collector_bin")
+add_executable(${EXECUTABLE_NAME} nginx_conf_collector.cc)
+target_compile_definitions(${EXECUTABLE_NAME} PRIVATE "NGINX_CONF_COLLECTOR_VERSION=\"$ENV{CI_PIPELINE_ID}\"")
+
+target_link_libraries(${EXECUTABLE_NAME}
+        shell_cmd
+        mainloop
+        messaging
+        event_is
+        metric
+        compression_utils
+        z
+        nginx_utils
+        time_proxy
+        debug_is
+        version
+        report
+        config
+        environment
+        singleton
+        rest
+        boost_context
+        boost_regex
+        pthread
+)
+
+install(TARGETS ${EXECUTABLE_NAME} DESTINATION bin)
+install(PROGRAMS ${EXECUTABLE_NAME} DESTINATION central_nginx_manager/bin RENAME cp-nano-nginx-conf-collector)

components/utils/utilities/nginx_conf_collector/nginx_conf_collector.cc

@@ -0,0 +1,148 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <iostream>
+#include <unistd.h>
+
+#include "agent_core_utilities.h"
+#include "debug.h"
+#include "internal/shell_cmd.h"
+#include "mainloop.h"
+#include "nginx_utils.h"
+#include "time_proxy.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_NGINX_MANAGER);
+
+class MainComponent
+{
+public:
+    MainComponent()
+    {
+        time_proxy.init();
+        environment.init();
+        mainloop.init();
+        shell_cmd.init();
+    }
+
+    ~MainComponent()
+    {
+        shell_cmd.fini();
+        mainloop.fini();
+        environment.fini();
+        time_proxy.fini();
+    }
+private:
+    ShellCmd shell_cmd;
+    MainloopComponent mainloop;
+    Environment environment;
+    TimeProxyComponent time_proxy;
+};
+
+void
+printVersion()
+{
+#ifdef NGINX_CONF_COLLECTOR_VERSION
+    cout << "Check Point NGINX configuration collector version: " << NGINX_CONF_COLLECTOR_VERSION << '\n';
+#else
+    cout << "Check Point NGINX configuration collector version: Private" << '\n';
+#endif
+}
+
+void
+printUsage(const char *prog_name)
+{
+    cout << "Usage: " << prog_name << " [-v] [-i /path/to/nginx.conf] [-o /path/to/output.conf]" << '\n';
+    cout << "  -V              Print version" << '\n';
+    cout << "  -v              Enable verbose output" << '\n';
+    cout << "  -i input_file   Specify input file (default is /etc/nginx/nginx.conf)" << '\n';
+    cout << "  -o output_file  Specify output file (default is ./full_nginx.conf)" << '\n';
+    cout << "  -h              Print this help message" << '\n';
+}
+
+int
+main(int argc, char *argv[])
+{
+    string nginx_input_file = "/etc/nginx/nginx.conf";
+    string nginx_output_file = "full_nginx.conf";
+
+    int opt;
+    while ((opt = getopt(argc, argv, "Vvhi:o:h")) != -1) {
+        switch (opt) {
+            case 'V':
+                printVersion();
+                return 0;
+            case 'v':
+                Debug::setUnitTestFlag(D_NGINX_MANAGER, Debug::DebugLevel::TRACE);
+                break;
+            case 'i':
+                nginx_input_file = optarg;
+                break;
+            case 'o':
+                nginx_output_file = optarg;
+                break;
+            case 'h':
+                printUsage(argv[0]);
+                return 0;
+            default:
+                printUsage(argv[0]);
+                return 1;
+        }
+    }
+
+    for (int i = optind; i < argc;) {
+        cerr << "Unknown argument: " << argv[i] << '\n';
+        printUsage(argv[0]);
+        return 1;
+    }
+
+    dbgTrace(D_NGINX_MANAGER) << "Starting nginx configuration collector";
+
+    MainComponent main_component;
+    auto validation_result = NginxUtils::validateNginxConf(nginx_input_file);
+    if (!validation_result.ok()) {
+        cerr
+            << "Could not validate nginx configuration file: "
+            << nginx_input_file
+            << '\n'
+            << validation_result.getErr();
+        return 1;
+    }
+
+    NginxConfCollector nginx_collector(nginx_input_file, nginx_output_file);
+    auto result = nginx_collector.generateFullNginxConf();
+    if (!result.ok()) {
+        cerr << "Could not generate full nginx configuration file, error: " << result.getErr() << '\n';
+        return 1;
+    }
+
+    if (result.unpack().empty() || !NGEN::Filesystem::exists(result.unpack())) {
+        cerr << "Generated nginx configuration file does not exist: " << result.unpack() << '\n';
+        return 1;
+    }
+
+    validation_result = NginxUtils::validateNginxConf(result.unpack());
+    if (!validation_result.ok()) {
+        cerr
+            << "Could not validate generated nginx configuration file: "
+            << nginx_output_file
+            << '\n'
+            << validation_result.getErr();
+        return 1;
+    }
+
+    cout << "Full nginx configuration file was successfully generated: " << result.unpack() << '\n';
+
+    return 0;
+}

config/k8s/v1beta2/open-appsec-k8s-default-config-v1beta2.yaml

@@ -14,7 +14,7 @@ spec:
     - default-threat-prevention-practice
     accessControlPractices:
     - default-access-control-practice
-    customResponses: default-web-user-response
+    customResponse: default-web-user-response
     triggers:
     - default-log-trigger
   specificRules:
@@ -62,7 +62,7 @@ spec:
     files: []
     # relevant for docker and linux embedded deployments
     # 0 or 1 files supported in array
-  openapiSchemaValidation: # schema validation requires "Premium Edition"
+  schemaValidation: # schema validation requires "Premium Edition"
     overrideMode: inherited
     configmap: []
     # relevant for deployments on kubernetes

config/k8s/v1beta2/open-appsec-k8s-full-example-config-v1beta2.yaml

@@ -0,0 +1,163 @@
+apiVersion: openappsec.io/v1beta2
+kind: AccessControlPractice
+metadata:
+  name: access-control-practice-example
+spec:
+  practiceMode: prevent
+  rateLimit:
+    overrideMode: inherited
+    rules:
+    - action: prevent
+      comment: Limiting access to the resource
+      limit: 100
+      triggers:
+      - log-trigger-example
+      unit: minute
+      uri: /api/resource
+    - action: inherited
+      comment: Rate limiting for authentication requests
+      limit: 50
+      triggers:
+      - log-trigger-example
+      unit: second
+      uri: /api/auth
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: CustomResponse
+metadata:
+  name: custom-response-block-page-example
+spec:
+  mode: block-page
+  messageTitle: "Access Denied"
+  messageBody: "Your request was blocked for security reasons."
+  httpResponseCode: 403
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: Exception
+metadata:
+  name: exception-example
+spec:
+  action: accept
+  condition:
+    - key: countryCode
+      value: US
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: LogTrigger
+metadata:
+  name: log-trigger-example
+spec:
+  accessControlLogging:
+    allowEvents: false
+    dropEvents: true
+  appsecLogging:
+    detectEvents: true
+    preventEvents: true
+    allWebRequests: false
+  additionalSuspiciousEventsLogging:
+    enabled: true
+    minSeverity: high # {high|critical}
+    responseBody: false
+    responseCode: true
+  extendedLogging:
+    urlPath: true
+    urlQuery: true
+    httpHeaders: false
+    requestBody: false
+  logDestination:
+    cloud: true
+    logToAgent: true
+    stdout:
+      format: json-formatted
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: Policy
+metadata:
+    name: policy-example
+spec:
+    default:
+        mode: prevent-learn
+        accessControlPractices: [access-control-practice-example]
+        threatPreventionPractices: [threat-prevention-practice-example]
+        triggers: [log-trigger-example]
+        customResponse: custom-response-block-page-example
+        sourceIdentifiers: sources-identifier-example
+        trustedSources: trusted-sources-example
+        exceptions:
+        - exception-example
+---
+apiVersion: openappsec.io/v1beta2
+kind: ThreatPreventionPractice
+metadata:
+  name: threat-prevention-practice-example
+spec:
+  practiceMode: inherited
+  webAttacks:
+    overrideMode: inherited
+    minimumConfidence: high
+  intrusionPrevention:
+  # intrusion prevention (IPS) requires "Premium Edition"
+    overrideMode: inherited
+    maxPerformanceImpact: medium
+    minSeverityLevel: medium
+    minCveYear: 2016
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  fileSecurity:
+  # file security requires "Premium Edition"
+    overrideMode: inherited
+    minSeverityLevel: medium
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  snortSignatures:
+    # you must specify snort signatures in configmap or file to activate snort inspection
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  schemaValidation: # schema validation requires "Premium Edition"
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  antiBot: # antibot requires "Premium Edition"
+    overrideMode: inherited
+    injectedUris: []
+    validatedUris: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: TrustedSource
+metadata:
+  name: trusted-sources-example
+spec:
+  minNumOfSources: 3
+  sourcesIdentifiers:
+    - 1.0.0.27
+    - 1.0.0.28
+    - 1.0.0.29
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: SourcesIdentifier
+metadata:
+  name: sources-identifier-example
+spec:
+  sourcesIdentifiers:
+    - identifier: sourceip
+      value:
+        - "192.168.1.1" 
+        - "10.0.0.1"
+

config/k8s/v1beta2/open-appsec-k8s-prevent-config-v1beta2.yaml

@@ -14,7 +14,7 @@ spec:
     - default-threat-prevention-practice
     accessControlPractices:
     - default-access-control-practice
-    customResponses: default-web-user-response
+    customResponse: default-web-user-response
     triggers:
     - default-log-trigger
   specificRules:
@@ -62,7 +62,7 @@ spec:
     files: []
     # relevant for docker and linux embedded deployments
     # 0 or 1 files supported in array
-  openapiSchemaValidation: # schema validation requires "Premium Edition"
+  schemaValidation: # schema validation requires "Premium Edition"
     overrideMode: inherited
     configmap: []
     # relevant for deployments on kubernetes

core/CMakeLists.txt

@@ -40,7 +40,7 @@ target_link_libraries(
     "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest"
     "compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey"
     "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities"
-    "report_messaging;env_details"
+    "report_messaging;env_details;version"
     -Wl,-no-whole-archive
 )
 

core/agent_core_utilities/agent_core_utilities.cc

@@ -203,6 +203,18 @@ deleteFile(const string &path)
     return true;
 }
 
+string
+resolveFullPath(const string &input_path) {
+    dbgTrace(D_INFRA_UTILS) << "Resolving absolute path: " << input_path;
+    char resolved_path[PATH_MAX];
+    if (!realpath(input_path.c_str(), resolved_path)) {
+        dbgWarning(D_INFRA_UTILS) << "Error resolving path: " << input_path << ", errno: " << errno;
+        return "";
+    }
+
+    return string(resolved_path);
+}
+
 bool
 deleteDirectory(const string &path, bool delete_content)
 {
@@ -510,6 +522,23 @@ removeTrailingWhitespaces(string str)
     return str;
 }
 
+string
+removeLeadingWhitespaces(string str)
+{
+    str.erase(
+        str.begin(),
+        find_if(str.begin(), str.end(), [] (char c) { return !isspace(c); })
+    );
+
+    return str;
+}
+
+string
+trim(string str)
+{
+    return removeLeadingWhitespaces(removeTrailingWhitespaces(str));
+}
+
 } // namespace Strings
 
 } // namespace NGEN

core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc

@@ -184,3 +184,27 @@ TEST_F(AgentCoreUtilUT, removeTrailingWhitespacesTest)
     string str_with_trailing_whitespace = "str_with_trailing_whitespace\n\n\n\r    \n\n\r";
     EXPECT_EQ(NGEN::Strings::removeTrailingWhitespaces(str_with_trailing_whitespace), "str_with_trailing_whitespace");
 }
+
+TEST_F(AgentCoreUtilUT, removeLeadingWhitespacesTest)
+{
+    string str_with_leading_whitespace = "\n\n\n\r    \n\n\rstr_with_leading_whitespace";
+    EXPECT_EQ(NGEN::Strings::removeLeadingWhitespaces(str_with_leading_whitespace), "str_with_leading_whitespace");
+}
+
+TEST_F(AgentCoreUtilUT, trimTest)
+{
+    string str_with_leading_and_trailing_whitespace = "\n\n \r  \rstr_with_whitespace\n\r \n\n\r";
+    EXPECT_EQ(NGEN::Strings::trim(str_with_leading_and_trailing_whitespace), "str_with_whitespace");
+}
+
+TEST_F(AgentCoreUtilUT, resolveFullPathTest)
+{
+    string working_dir = cptestFnameInExeDir("");
+    ofstream file(working_dir + "test.txt");
+    ASSERT_TRUE(file.is_open());
+    file.close();
+    string relative_path = "test.txt";
+    string full_path = NGEN::Filesystem::resolveFullPath(relative_path);
+    EXPECT_EQ(full_path, working_dir + "test.txt");
+    ASSERT_TRUE(NGEN::Filesystem::deleteFile(working_dir + "test.txt"));
+}

core/agent_details/agent_details.cc

@@ -388,8 +388,9 @@ AgentDetails::convertProxyProtocolToString(ProxyProtocol proto) const
         case ProxyProtocol::HTTP: return "http";
         case ProxyProtocol::HTTPS: return "https";
     }
-    dbgAssert(false) << alert << "Unsupported Proxy Protocol " << static_cast<int>(proto);
-    return "";
+    dbgAssertOpt(false) << alert << "Unsupported Proxy Protocol " << static_cast<int>(proto);
+    dbgWarning(D_ORCHESTRATOR) << "Using https proxy as default";
+    return "https";
 }
 
 Maybe<void>
@@ -475,11 +476,14 @@ Maybe<void>
 AgentDetails::loadProxyType(ProxyProtocol protocol)
 {
     dbgFlow(D_ORCHESTRATOR) << "Loading proxy type: " << convertProxyProtocolToString(protocol);
-    dbgAssert(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS)
+    if (!(protocol == ProxyProtocol::HTTP || protocol == ProxyProtocol::HTTPS)) {
+        dbgAssertOpt(false)
             << alert
             << "Unsupported Proxy Protocol "
             << static_cast<int>(protocol);
-
+        protocol = ProxyProtocol::HTTPS;
+        dbgWarning(D_ORCHESTRATOR) << "Using https proxy as default";
+    }
     static const map<ProxyProtocol, string> env_var_name = {
         {ProxyProtocol::HTTPS, "https_proxy"},
         {ProxyProtocol::HTTP, "http_proxy"}

core/buffers/buffer.cc

@@ -144,8 +144,8 @@ Buffer::operator+(const Buffer &other) const
 Buffer
 Buffer::getSubBuffer(uint start, uint end) const
 {
-    dbgAssert(start<=end && end<=len) << alert << "Buffer::getSubBuffer() returned: Illegal scoping of buffer";
-    if (start == end) return Buffer();
+    dbgAssertOpt(start<=end && end<=len) << alert << "Buffer::getSubBuffer() returned: Illegal scoping of buffer";
+    if (start >= end || end > len) return Buffer();
 
     Buffer res;
     uint offset = 0;
@@ -178,8 +178,12 @@ Buffer::getSubBuffer(uint start, uint end) const
 Maybe<uint>
 Buffer::findFirstOf(char ch, uint start) const
 {
-    dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end";
-
+    if (start > len) {
+        dbgAssertOpt(start <= len)
+            << alert
+            << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end";
+        return genError("Cannot set a start point after buffer's end");
+    }
     for (; start < len; ++start) {
         if ((*this)[start] == ch) return start;
     }
@@ -189,8 +193,12 @@ Buffer::findFirstOf(char ch, uint start) const
 Maybe<uint>
 Buffer::findFirstOf(const Buffer &buf, uint start) const
 {
-    dbgAssert(start <= len) << alert << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end";
-
+    if (start > len) {
+        dbgAssertOpt(start <= len)
+            << alert
+            << "Buffer::findFirstOf() returned: Cannot set a start point after buffer's end";
+        return genError("Cannot set a start point after buffer's end");
+    }
     for (; start + buf.size() <= len; ++start) {
         auto sub_buffer = getSubBuffer(start, start + buf.size());
         if (sub_buffer == buf) return start;
@@ -201,9 +209,13 @@ Buffer::findFirstOf(const Buffer &buf, uint start) const
 Maybe<uint>
 Buffer::findFirstNotOf(char ch, uint start) const
 {
-    dbgAssert(start <= len)
+    if (start > len) {
+        dbgAssertOpt(start <= len)
             << alert
             << "Buffer::findFirstNotOf() returned: Cannot set a start point after buffer's end";
+        return genError("Cannot set a start point after buffer's end");
+    }
+
     for (; start < len; ++start) {
         if ((*this)[start] != ch) return start;
     }
@@ -213,7 +225,12 @@ Buffer::findFirstNotOf(char ch, uint start) const
 Maybe<uint>
 Buffer::findLastOf(char ch, uint start) const
 {
-    dbgAssert(start <= len) << alert << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end";
+    if (start > len) {
+        dbgAssertOpt(start <= len)
+            << alert
+            << "Buffer::findLastOf() returned: Cannot set a start point after buffer's end";
+        return genError("Cannot set a start point after buffer's end");
+    }
     for (; 0 < start; --start) {
         if ((*this)[start - 1] == ch) return start - 1;
     }
@@ -223,9 +240,12 @@ Buffer::findLastOf(char ch, uint start) const
 Maybe<uint>
 Buffer::findLastNotOf(char ch, uint start) const
 {
-    dbgAssert(start <= len)
+    if (start > len) {
+        dbgAssertOpt(start <= len)
             << alert
             << "Buffer::findLastNotOf() returned: Cannot set a start point after buffer's end";
+        return genError("Cannot set a start point after buffer's end");
+    }
     for (; 0 < start; --start) {
         if ((*this)[start - 1] != ch) return start - 1;
     }
@@ -235,8 +255,8 @@ Buffer::findLastNotOf(char ch, uint start) const
 void
 Buffer::truncateHead(uint size)
 {
-    dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end";
-    if (size == 0) return;
+    dbgAssertOpt(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end";
+    if (size == 0 || size > len) return;
     if (size == len) {
         clear();
         return;
@@ -261,8 +281,8 @@ Buffer::truncateHead(uint size)
 void
 Buffer::truncateTail(uint size)
 {
-    dbgAssert(size <= len) << alert << "Cannot set a new end of buffer after the buffer's end";
-    if (size == 0) return;
+    dbgAssertOpt(size <= len) << alert << "Cannot set a new end of buffer after the buffer's end";
+    if (size == 0 || size > len) return;
     if (size == len) {
         clear();
         return;
@@ -285,14 +305,20 @@ Buffer::truncateTail(uint size)
 void
 Buffer::keepHead(uint size)
 {
-    dbgAssert(size <= len) << alert << "Cannot set a new end of buffer before the buffer's start";
+    if (size > len) {
+        dbgAssertOpt(size <= len) << alert << "Cannot set a new end of buffer before the buffer's start";
+        return;
+    }
     truncateTail(len - size);
 }
 
 void
 Buffer::keepTail(uint size)
 {
-    dbgAssert(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end";
+    if (size > len) {
+        dbgAssertOpt(size <= len) << alert << "Cannot set a new start of buffer after the buffer's end";
+        return;
+    }
     truncateHead(len - size);
 }
 

core/compression/CMakeLists.txt

@@ -2,8 +2,11 @@ include_directories(${ng_module_osrc_zlib_path}/include)
 add_definitions(-DZLIB_CONST)
 
 add_library(compression_utils SHARED compression_utils.cc)
+add_library(static_compression_utils compression_utils.cc)
 
 add_subdirectory(compression_utils_ut)
 
 install(TARGETS compression_utils DESTINATION lib)
 install(TARGETS compression_utils DESTINATION http_transaction_handler_service/lib)
+
+install(TARGETS static_compression_utils DESTINATION lib)

core/connkey/connkey.cc

@@ -64,12 +64,12 @@ IPAddr::print(ostream &os) const
     switch (type) {
         case IPType::V4: {
             formatted_addr = inet_ntop(AF_INET, &v4, buf, sizeof(buf));
-            dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv4 address";
+            dbgAssertOpt(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv4 address";
             break;
         }
         case IPType::V6: {
             formatted_addr = inet_ntop(AF_INET6, &v6, buf, sizeof(buf));
-            dbgAssert(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv6 address";
+            dbgAssertOpt(formatted_addr == buf) << alert("conversion error") << "Failed to convert an IPv6 address";
             break;
         }
         case IPType::UNINITIALIZED: {
@@ -116,7 +116,7 @@ ConnKey::reverse()
 size_t
 ConnKey::hash() const
 {
-    dbgAssert(src.type != IPType::UNINITIALIZED)
+    dbgAssertOpt(src.type != IPType::UNINITIALIZED)
         << alert("hashing")
         << "ConnKey::hash was called on an uninitialized object";
     size_t seed = 0;

core/debug_is/debug.cc

@@ -27,6 +27,7 @@
 #include "i_instance_awareness.h"
 #include "i_signal_handler.h"
 #include "hash_combine.h"
+#include "version.h"
 
 using namespace std;
 
@@ -298,14 +299,19 @@ AlertInfo::evalParams()
 Debug::Debug(
     const string &file_name,
     const string &func_name,
-    const uint &line)
+    const uint &line,
+    bool force_assert)
 {
-    if (Singleton::exists<Config::I_Config>()) {
-        do_assert = getConfigurationWithDefault<bool>(true, "Debug I/S", "Abort on assertion");
+    if (!force_assert && !should_assert_optional) {
+        do_assert = false;
     } else {
         do_assert = true;
     }
 
+    if (Singleton::exists<Config::I_Config>()) {
+        do_assert = getConfigurationWithDefault<bool>(do_assert, "Debug I/S", "Abort on assertion");
+    }
+
     auto current_configuration =
         Singleton::exists<Config::I_Config>() ? getConfigurationWithDefault(default_config, "Debug") : default_config;
 
@@ -519,6 +525,13 @@ Debug::preload()
 
     active_streams["STDOUT"] = make_shared<Debug::DebugStream>(&cout);
     active_streams["FOG"] = make_shared<DebugFogStream>();
+
+    string branch = Version::getBranch();
+    if (branch == "open-source" || branch == "master" || branch.substr(0, 6) == "hotfix") {
+        should_assert_optional = false;
+    } else {
+        should_assert_optional = true;
+    }
 }
 
 void
@@ -844,3 +857,4 @@ bool Debug::is_fail_open_mode = false;
 bool Debug::debug_override_exist = false;
 string Debug::default_debug_file_stream_path = "";
 vector<string> Debug::streams_from_mgmt;
+bool Debug::should_assert_optional = true;

core/debug_is/debug_streams.cc

@@ -396,14 +396,18 @@ LogLevel
 DebugFogStream::getLogLevel() const
 {
     switch (level) {
-        case Debug::DebugLevel::NOISE:     dbgAssert(false) << alert << "Impossible LogLevel 'Noise'"; break;
+        case Debug::DebugLevel::NOISE:
+            dbgAssertOpt(false) << alert << "Impossible LogLevel 'Noise'";
+            return LogLevel::TRACE;
         case Debug::DebugLevel::TRACE:     return LogLevel::TRACE;
         case Debug::DebugLevel::DEBUG:     return LogLevel::DEBUG;
         case Debug::DebugLevel::WARNING:   return LogLevel::WARNING;
         case Debug::DebugLevel::INFO:      return LogLevel::INFO;
         case Debug::DebugLevel::ERROR:     return LogLevel::ERROR;
         case Debug::DebugLevel::ASSERTION: return LogLevel::ERROR;
-        case Debug::DebugLevel::NONE:      dbgAssert(false) << alert << "Impossible LogLevel 'None'"; break;
+        case Debug::DebugLevel::NONE:
+            dbgAssertOpt(false) << alert << "Impossible LogLevel 'None'";
+            return LogLevel::ERROR;
     }
 
     return LogLevel::INFO;

core/environment/context.cc

@@ -60,10 +60,11 @@ Context::convertToString(MetaDataType type)
         case MetaDataType::Direction: return "direction";
         case MetaDataType::Email: return "email";
         case MetaDataType::COUNT:
-            dbgAssert(false) << alert << "COUNT is not a valid meta data type";
+            dbgAssertOpt(false) << alert << "COUNT is not a valid meta data type";
+            return "invalid_count";
     }
-    dbgAssert(false) << alert << "Reached impossible case with type=" << static_cast<int>(type);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached impossible case with type=" << static_cast<int>(type);
+    return "invalid_metadata_type";
 }
 
 map<string, uint64_t>

core/environment/span.cc

@@ -97,8 +97,8 @@ Span::convertSpanContextTypeToString(ContextType type)
             return "Follows from";
         }
     }
-    dbgAssert(false) << AlertInfo(AlertTeam::CORE, "tracing") << "Span context not supported";
-    return string();
+    dbgAssertOpt(false) << AlertInfo(AlertTeam::CORE, "tracing") << "Span context not supported";
+    return "Invalid context type";
 }
 
 SpanWrapper::SpanWrapper(string _trace_id, Span::ContextType _type, string _prev_span)

core/include/general/debug.h

@@ -159,7 +159,8 @@ public:
     Debug(
         const std::string &file_name,
         const std::string &func_name,
-        const uint &line
+        const uint &line,
+        bool force_assert
     );
 
     Debug(
@@ -273,6 +274,7 @@ private:
     static bool debug_override_exist;
     static std::string default_debug_file_stream_path;
     static std::vector<std::string> streams_from_mgmt;
+    static bool should_assert_optional;
 
     bool do_assert;
     bool is_communication = false;
@@ -328,7 +330,11 @@ getBaseName(const char *iter, const char *base)
 
 #define dbgAssert(cond) \
     if (CP_LIKELY(cond)) { \
-    } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__).getStreamAggr()
+    } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__, true).getStreamAggr()
+
+#define dbgAssertOpt(cond) \
+    if (CP_LIKELY(cond)) { \
+    } else Debug::DebugAlert(__FILENAME__, __FUNCTION__, __LINE__, false).getStreamAggr()
 
 // Macros to allow simple debug messaging
 #define DBG_GENERIC(level, ...) \

core/include/services_sdk/interfaces/i_rest_api.h

@@ -49,6 +49,10 @@ public:
     }
 
     virtual bool addGetCall(const std::string &uri, const std::function<std::string()> &callback) = 0;
+    virtual bool addWildcardGetCall(
+        const std::string &uri,
+        const std::function<std::string(const std::string &)> &callback
+    ) = 0;
 
     virtual uint16_t getListeningPort() const = 0;
 

core/include/services_sdk/interfaces/mock/mock_rest_api.h

@@ -10,6 +10,11 @@ class MockRestApi : public Singleton::Provide<I_RestApi>::From<MockProvider<I_Re
 public:
     MOCK_CONST_METHOD0(getListeningPort, uint16_t());
     MOCK_METHOD2(addGetCall, bool(const std::string &, const std::function<std::string()> &));
+    MOCK_METHOD2(
+        addWildcardGetCall,
+        bool(const std::string &, const std::function<std::string(const std::string &)> &)
+    );
+
     // You can't mock a function with an R-value reference. So mock a slightly different one
     MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr<RestInit> &));
 

core/include/services_sdk/resources/debug_flags.h

@@ -114,6 +114,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT)
     DEFINE_FLAG(D_RATE_LIMIT, D_COMPONENT)
     DEFINE_FLAG(D_ROLLBACK_TESTING, D_COMPONENT)
+    DEFINE_FLAG(D_NGINX_MANAGER, D_COMPONENT)
 
     DEFINE_FLAG(D_PARSER, D_COMPONENT)
     DEFINE_FLAG(D_WS, D_COMPONENT)

core/include/services_sdk/resources/report/base_field.h

@@ -162,7 +162,7 @@ class LogField : Singleton::Consume<I_Environment>
         void
         addFields(const LogField &)
         {
-            dbgAssert(false)
+            dbgAssertOpt(false)
                 << AlertInfo(AlertTeam::CORE, "report i/s")
                 << "Trying to add a log field to a 'type'ed field";
         }

core/include/services_sdk/resources/report/report_bulks.h

@@ -26,7 +26,10 @@ public:
     void
     setBulkSize(uint size)
     {
-        dbgAssert(size > 0) << AlertInfo(AlertTeam::CORE, "report i/s") << "Bulk size must be larger than 0";
+        if (size <= 0) {
+            dbgAssertOpt(size > 0) << AlertInfo(AlertTeam::CORE, "report i/s") << "Bulk size must be larger than 0";
+            size = 100;
+        }
         dbgDebug(D_REPORT_BULK) << "Bulk size is set to " << size;
         bulk_size = size;
     }

core/include/services_sdk/utilities/agent_core_utilities.h

@@ -33,6 +33,7 @@ bool makeDir(const std::string &path, mode_t permission = S_IRWXU);
 bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU);
 bool deleteDirectory(const std::string &path, bool delete_content = false);
 bool touchFile(const std::string &path);
+std::string resolveFullPath(const std::string &input_path);
 
 bool
 copyFile(
@@ -43,11 +44,8 @@ copyFile(
 );
 
 bool deleteFile(const std::string &path);
-
 std::string convertToHumanReadable(uint64_t size_in_bytes);
-
 std::string getFileName(const std::string &path);
-
 bool copyDirectory(const std::string &src_dir_path, const std::string &dst_dir_path);
 
 }// namespace Filesystem
@@ -85,6 +83,8 @@ namespace Strings
 {
 
 std::string removeTrailingWhitespaces(std::string str);
+std::string removeLeadingWhitespaces(std::string str);
+std::string trim(std::string str);
 
 } // namespace Strings
 

core/include/services_sdk/utilities/connkey.h

@@ -87,9 +87,12 @@ public:
     bool
     operator==(const IPAddr &other) const
     {
-        dbgAssert(type!=IPType::UNINITIALIZED && other.type!=IPType::UNINITIALIZED)
+        if (type == IPType::UNINITIALIZED || other.type == IPType::UNINITIALIZED) {
+            dbgAssertOpt(type!=IPType::UNINITIALIZED && other.type!=IPType::UNINITIALIZED)
                 << AlertInfo(AlertTeam::CORE, "connkey")
                 << "Called on an uninitialized IPType object";
+            return false;
+        }
         // Always compairing as if IPv6, in case of Ipv4 the rest of the address is zeroed out.
         int ip_len = (other.type == IPType::V4) ? sizeof(v4.s_addr) : sizeof(v6.s6_addr);
         return (type == other.type) && (memcmp(v6.s6_addr, other.v6.s6_addr, ip_len) == 0);
@@ -308,9 +311,12 @@ public:
     IPType
     getType() const
     {
-        dbgAssert(src.type == dst.type)
+        if (src.type != dst.type) {
+            dbgAssertOpt(src.type == dst.type)
                 << AlertInfo(AlertTeam::CORE, "connkey")
                 << "Mismatch in connection types (Src and Dst types are not identical)";
+            return IPType::V6;
+        }
         return src.type;
     }
 

core/logging/cef_stream.cc

@@ -59,7 +59,14 @@ CefStream::sendLog(const Report &log)
 void
 CefStream::init()  {
     updateSettings();
-    maintainConnection();
+    mainloop->addOneTimeRoutine(
+        I_MainLoop::RoutineType::Offline,
+        [this] ()
+        {
+            dbgTrace(D_REPORT) << FIRST_CEF_CONNECT_NAME;
+        },
+        FIRST_CEF_CONNECT_NAME
+    );
 
     auto ceflog_retry_interval = getProfileAgentSettingWithDefault<uint>(
         RETRY_CONNECT_INTERVAL,

core/logging/log_streams.h

@@ -26,7 +26,9 @@
 #include "logging_comp.h"
 
 static const int RETRY_CONNECT_INTERVAL = 120;
+static const std::string FIRST_SYSLOG_CONNECT_NAME = "first connecting to Syslog server";
 static const std::string SYSLOG_CONNECT_NAME = "connecting to Syslog server";
+static const std::string FIRST_CEF_CONNECT_NAME = "first connecting to CEF server";
 static const std::string CEF_CONNECT_NAME = "connecting to CEF server";
 static const int NUMBER_OF_LOGS_PER_SEND = 15;
 static size_t MAX_LOG_QUEUE = 1000;

core/logging/logging_ut/logging_ut.cc

@@ -134,6 +134,16 @@ public:
             DoAll(SaveArg<1>(&sysog_routine), Return(0))
         );
 
+        EXPECT_CALL(
+            mock_mainloop,
+            addOneTimeRoutine(_, _, "first connecting to Syslog server", _)
+        ).WillRepeatedly(DoAll(SaveArg<1>(&first_connect_syslog_routine), Return(0)));
+
+        EXPECT_CALL(
+            mock_mainloop,
+            addOneTimeRoutine(_, _, "first connecting to CEF server", _)
+        ).WillRepeatedly(DoAll(SaveArg<1>(&first_connect_cef_routine), Return(0)));
+
         EXPECT_CALL(
                 mock_mainloop,
                 addRecurringRoutine(_, _, _, "connecting to Syslog server", _)
@@ -303,6 +313,8 @@ public:
     ConfigComponent           config;
     vector<string>            capture_syslog_cef_data;
     I_MainLoop::Routine       sysog_routine = nullptr;
+    I_MainLoop::Routine       first_connect_syslog_routine = nullptr;
+    I_MainLoop::Routine       first_connect_cef_routine = nullptr;
     I_MainLoop::Routine       connect_syslog_routine = nullptr;
     I_MainLoop::Routine       connect_cef_routine = nullptr;
     StrictMock<MockShellCmd>  mock_shell_cmd;
@@ -1517,6 +1529,8 @@ TEST_F(LogTest, ObfuscationCefSysLogTest)
     I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
     // for cef
     CefStream cef_stream(address, port, protocol);
+    ASSERT_NE(first_connect_cef_routine, nullptr);
+    first_connect_cef_routine();
     ASSERT_NE(connect_cef_routine, nullptr);
     connect_cef_routine();
     cef_stream.sendLog(CreateReport(tag1, tag2));
@@ -1525,6 +1539,8 @@ TEST_F(LogTest, ObfuscationCefSysLogTest)
     SyslogStream syslog_stream(address, port, protocol);
 
     // connection to socket before send log
+    ASSERT_NE(first_connect_syslog_routine, nullptr);
+    first_connect_syslog_routine();
     ASSERT_NE(connect_syslog_routine, nullptr);
     connect_syslog_routine();
     
@@ -1554,6 +1570,8 @@ TEST_F(LogTest, SysLogWriteFailTest)
     I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
     SyslogStream syslog_stream(address, port, protocol);
 
+    ASSERT_NE(first_connect_syslog_routine, nullptr);
+    first_connect_syslog_routine();
     ASSERT_NE(connect_syslog_routine, nullptr);
     connect_syslog_routine();
 
@@ -1599,6 +1617,8 @@ TEST_F(LogTest, CefWriteFailTest)
     I_Socket::SocketType protocol = I_Socket::SocketType::TCP;
     CefStream cef_stream(address, port, protocol);
 
+    ASSERT_NE(first_connect_cef_routine, nullptr);
+    first_connect_cef_routine();
     ASSERT_NE(connect_cef_routine, nullptr);
     connect_cef_routine();
 

core/logging/syslog_stream.cc

@@ -71,7 +71,14 @@ SyslogStream::sendLog(const vector<char> &data)
 void
 SyslogStream::init() {
     updateSettings();
-    maintainConnection();
+    mainloop->addOneTimeRoutine(
+        I_MainLoop::RoutineType::Offline,
+        [this] ()
+        {
+            dbgTrace(D_REPORT) << FIRST_SYSLOG_CONNECT_NAME;
+        },
+        FIRST_SYSLOG_CONNECT_NAME
+    );
 
     auto syslog_retry_interval = getProfileAgentSettingWithDefault<uint>(
         RETRY_CONNECT_INTERVAL,

core/mainloop/coroutine.cc

@@ -76,10 +76,12 @@ RoutineWrapper::resume()
 void
 RoutineWrapper::invoke(pull_type &pull, I_MainLoop::Routine func)
 {
-    dbgAssert(active != nullptr)
+    if (!active) {
+        dbgAssertOpt(active != nullptr)
             << AlertInfo(AlertTeam::CORE, "mainloop i/s")
             << "Trying to invoke without an active routine";
-
+        return;
+    }
     active->pull = move(pull); // First invokation (other invokaction will start inside `func`), set the `pull` object
     func();
 }

core/mainloop/mainloop.cc

@@ -227,7 +227,10 @@ MainloopComponent::Impl::reportStartupEvent()
 void
 MainloopComponent::Impl::run()
 {
-    dbgAssert(!is_running) << alert << "MainloopComponent::Impl::run was called while it was already running";
+    if (is_running) {
+        dbgAssertOpt(!is_running) << alert << "MainloopComponent::Impl::run was called while it was already running";
+        return;
+    }
     is_running = true;
 
     bool has_primary_routines = true;
@@ -467,7 +470,10 @@ MainloopComponent::Impl::getCurrentRoutineId() const
 void
 MainloopComponent::Impl::yield(bool force)
 {
-    dbgAssert(curr_iter != routines.end()) << alert << "Calling 'yield' without a running current routine";
+    if (curr_iter == routines.end()) {
+        dbgAssertOpt(curr_iter != routines.end()) << alert << "Calling 'yield' without a running current routine";
+        return;
+    }
     if (do_stop) throw MainloopStop();
     if (!force && getTimer()->getMonotonicTime() < stop_time) return;
 
@@ -508,7 +514,10 @@ MainloopComponent::Impl::stopAll()
 void
 MainloopComponent::Impl::stop()
 {
-    dbgAssert(curr_iter != routines.end()) << alert << "Attempting to stop a routine when none is running";
+    if (curr_iter == routines.end()) {
+        dbgAssertOpt(curr_iter != routines.end()) << alert << "Attempting to stop a routine when none is running";
+        return;
+    }
     stop(curr_iter);
 }
 
@@ -526,7 +535,10 @@ MainloopComponent::Impl::stop(RoutineID id)
 void
 MainloopComponent::Impl::halt()
 {
-    dbgAssert(curr_iter != routines.end()) << alert << "Calling 'halt' without a running current routine";
+    if (curr_iter == routines.end()) {
+        dbgAssertOpt(curr_iter != routines.end()) << alert << "Calling 'halt' without a running current routine";
+        return;
+    }
     curr_iter->second.halt();
     yield(true);
 }
@@ -535,7 +547,10 @@ void
 MainloopComponent::Impl::halt(RoutineID id)
 {
     auto iter = routines.find(id);
-    dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to halt";
+    if (iter == routines.end()) {
+        dbgAssertOpt(iter != routines.end()) << alert << "No routine " << id << " to halt";
+        return;
+    }
     iter->second.halt();
     if (iter == curr_iter) yield(true);
 }
@@ -544,7 +559,10 @@ void
 MainloopComponent::Impl::resume(RoutineID id)
 {
     auto iter = routines.find(id);
-    dbgAssert(iter != routines.end()) << alert << "No routine " << id << " to resume";
+    if (iter == routines.end()) {
+        dbgAssertOpt(iter != routines.end()) << alert << "No routine " << id << " to resume";
+        return;
+    }
     iter->second.resume();
 }
 

core/messaging/messaging_comp/http_response.cc

@@ -94,10 +94,13 @@ string
 HTTPResponse::toString() const
 {
     auto code = status_code_to_string.find(status_code);
-    dbgAssert(code != status_code_to_string.end())
+    if (code == status_code_to_string.end()) {
+        dbgAssertOpt(code != status_code_to_string.end())
             << AlertInfo(AlertTeam::CORE, "messaging i/s")
             << "Unknown status code "
             << int(status_code);
+        return "[Status-code]: 500 - HTTP_INTERNAL_SERVER_ERROR, [Body]: " + (body.empty() ? "{}" : body);
+    }
 
     return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body);
 }

core/report/tag_and_enum_management.cc

@@ -160,8 +160,8 @@ TagAndEnumManagement::convertToString(const StreamType &stream_type)
         case StreamType::COUNT: break;
     }
 
-    dbgAssert(false) << alert << "Unknown log stream type. Type: " << static_cast<int>(stream_type);
-    return "";
+    dbgAssertOpt(false) << alert << "Unknown log stream type. Type: " << static_cast<int>(stream_type);
+    return "Unknown stream";
 }
 
 string
@@ -175,8 +175,8 @@ TagAndEnumManagement::convertToString(const Severity &severity)
         case Severity::INFO:     return "Info";
     }
 
-    dbgAssert(false) << alert << "Reached an impossible severity value of: " << static_cast<int>(severity);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached an impossible severity value of: " << static_cast<int>(severity);
+    return "Unknown severity";
 }
 
 string
@@ -188,8 +188,8 @@ TagAndEnumManagement::convertToString(const Type &type)
         case Type::CODE:     return "Code Related";
     }
 
-    dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast<int>(type);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast<int>(type);
+    return "Unknown type";
 }
 
 string
@@ -203,8 +203,8 @@ TagAndEnumManagement::convertToString(const Level &level)
         case Level::CUSTOM:   return "Custom";
     }
 
-    dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast<int>(level);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast<int>(level);
+    return "Unknown Level";
 }
 
 string
@@ -218,8 +218,8 @@ TagAndEnumManagement::convertToString(const LogLevel &log_level)
         case LogLevel::ERROR:   return "error";
     }
 
-    dbgAssert(false) << alert << "Reached an impossible type value of: " << static_cast<int>(log_level);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached an impossible type value of: " << static_cast<int>(log_level);
+    return "Unknown log level";
 }
 
 string
@@ -230,8 +230,8 @@ TagAndEnumManagement::convertToString(const Audience &audience)
         case Audience::INTERNAL: return "Internal";
     }
 
-    dbgAssert(false) << alert << "Reached an impossible audience value of: " << static_cast<int>(audience);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached an impossible audience value of: " << static_cast<int>(audience);
+    return "Unknown audience";
 }
 
 string
@@ -244,8 +244,8 @@ TagAndEnumManagement::convertToString(const Priority &priority)
         case Priority::LOW:    return "Low";
     }
 
-    dbgAssert(false) << alert << "Reached impossible priority value of: " << static_cast<int>(priority);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached impossible priority value of: " << static_cast<int>(priority);
+    return "Unknown priority";
 }
 
 string
@@ -263,8 +263,8 @@ TagAndEnumManagement::convertToString(const Notification &notification)
         case Notification::SDWAN_POLICY_WARNING_LOG: return "c58d490e-6aa0-43da-bfaa-7edad0a57b7a";
     }
 
-    dbgAssert(false) << alert << "Reached impossible notification value of: " << static_cast<int>(notification);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached impossible notification value of: " << static_cast<int>(notification);
+    return "00000000-0000-0000-0000-000000000000";
 }
 
 string
@@ -281,8 +281,8 @@ TagAndEnumManagement::convertToString(const IssuingEngine &issuing_engine)
         case IssuingEngine::HORIZON_TELEMETRY_METRICS: return "horizonTelemetryMetrics";
     }
 
-    dbgAssert(false) << alert << "Reached impossible engine value of: " << static_cast<int>(issuing_engine);
-    return "";
+    dbgAssertOpt(false) << alert << "Reached impossible engine value of: " << static_cast<int>(issuing_engine);
+    return "Unknown Issuer";
 }
 
 

core/rest/rest_server.cc

@@ -50,6 +50,7 @@ public:
     bool bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range);
     bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
     bool addGetCall(const string &uri, const function<string()> &cb) override;
+    bool addWildcardGetCall(const string &uri, const function<string(const string &)> &callback);
     uint16_t getListeningPort() const override { return listening_port; }
     Maybe<string> getSchema(const string &uri) const override;
     Maybe<string> invokeRest(const string &uri, istream &in) const override;
@@ -67,6 +68,7 @@ private:
     I_MainLoop *mainloop;
     map<string, unique_ptr<RestInit>> rest_calls;
     map<string, function<string()>> get_calls;
+    map<string, function<string(const string &)>> wildcard_get_calls;
     uint16_t listening_port = 0;
     vector<uint16_t> port_range;
 };
@@ -128,11 +130,14 @@ RestServer::Impl::prepareConfiguration()
     } else {
         auto range_start = getPortConfig("Nano service API Port Range start");
         auto range_end = getPortConfig("Nano service API Port Range end");
-        dbgAssert(range_start.ok() && range_end.ok()) << alert << "Rest port configuration was not provided";
-        dbgAssert(*range_start < *range_end)
+        if (!(range_start.ok() && range_end.ok()) || !(*range_start < *range_end)) {
+            dbgAssertOpt(range_start.ok() && range_end.ok()) << alert << "Rest port configuration was not provided";
+            dbgAssertOpt(*range_start < *range_end)
                 << alert
                 << "Rest port range corrupted (lower bound higher then upper bound)";
-
+            range_start = 0;
+            range_end = 1;
+        }
         port_range.resize(*range_end - *range_start);
         for (uint16_t i = 0, port = *range_start; i < port_range.size(); i++, port++) {
             port_range[i] = port;
@@ -283,6 +288,13 @@ RestServer::Impl::addGetCall(const string &uri, const function<string()> &callba
     return get_calls.emplace(uri, callback).second;
 }
 
+bool
+RestServer::Impl::addWildcardGetCall(const string &uri, const function<string(const string&)> &callback)
+{
+    if (rest_calls.find(uri) != rest_calls.end()) return false;
+    return wildcard_get_calls.emplace(uri, callback).second;
+}
+
 Maybe<string>
 RestServer::Impl::getSchema(const string &uri) const
 {
@@ -307,14 +319,26 @@ RestServer::Impl::invokeRest(const string &uri, istream &in) const
 bool
 RestServer::Impl::isGetCall(const string &uri) const
 {
-    return get_calls.find(uri) != get_calls.end();
+    if (get_calls.find(uri) != get_calls.end()) return true;
+
+    for (const auto &wildcard : wildcard_get_calls) {
+        if (!uri.find(wildcard.first)) return true;
+    }
+
+    return false;
 }
 
 string
 RestServer::Impl::invokeGet(const string &uri) const
 {
     auto instance = get_calls.find(uri);
-    return instance != get_calls.end() ? instance->second() : "";
+    if (instance != get_calls.end()) return instance->second();
+
+    for (const auto &wildcard : wildcard_get_calls) {
+        if (!uri.find(wildcard.first)) return wildcard.second(uri);
+    }
+
+    return "";
 }
 
 string
@@ -334,8 +358,8 @@ RestServer::Impl::changeActionToString(RestAction oper)
             return "delete-";
         }
         default: {
-            dbgAssert(false) << alert << "Unknown REST action";
-            return "";
+            dbgAssertOpt(false) << alert << "Unknown REST action";
+            return "unknown-";
         }
     }
 }

core/rest/rest_ut/CMakeLists.txt

@@ -4,5 +4,5 @@ link_directories(${ng_module_osrc_zlib_path}/lib)
 add_unit_test(
     rest_server_ut
     "rest_schema_ut.cc;rest_must_param_ut.cc;rest_config_ut.cc"
-    "singleton;messaging;tenant_manager;rest;environment;-lz;shell_cmd;-lboost_filesystem;instance_awareness;-lz;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey"
+    "singleton;messaging;tenant_manager;rest;environment;-lz;shell_cmd;-lboost_filesystem;instance_awareness;-lz;version;debug_is;time_proxy;mainloop;agent_details;encryptor;event_is;metric;-lboost_context;-lboost_regex;-lboost_system;-lssl;-lcrypto;connkey"
 )

core/rest/rest_ut/rest_config_ut.cc

@@ -171,11 +171,16 @@ TEST_F(RestConfigTest, basic_flow)
     auto i_rest = Singleton::Consume<I_RestApi>::from(rest_server);
     ASSERT_TRUE(i_rest->addRestCall<TestServer>(RestAction::ADD, "test"));
     ASSERT_TRUE(i_rest->addGetCall("stuff", [] () { return string("blabla"); }));
+    ASSERT_TRUE(
+        i_rest->addWildcardGetCall("api/", [] (const string &uri) { return uri.substr(uri.find_last_of('/') + 1); })
+    );
 
     int file_descriptor1 = socket(AF_INET, SOCK_STREAM, 0);
     EXPECT_NE(file_descriptor1, -1);
     int file_descriptor2 = socket(AF_INET, SOCK_STREAM, 0);
     EXPECT_NE(file_descriptor2, -1);
+    int file_descriptor3 = socket(AF_INET, SOCK_STREAM, 0);
+    EXPECT_NE(file_descriptor3, -1);
 
     auto primary_port = getConfiguration<uint>("connection", "Nano service API Port Alternative");
     struct sockaddr_in sa;
@@ -185,6 +190,7 @@ TEST_F(RestConfigTest, basic_flow)
     int socket_enable = 1;
     EXPECT_EQ(setsockopt(file_descriptor1, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
     EXPECT_EQ(setsockopt(file_descriptor2, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
+    EXPECT_EQ(setsockopt(file_descriptor3, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)), 0);
 
     EXPECT_CALL(messaging, sendSyncMessage(_, _, _, _, _))
         .WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "")));
@@ -203,6 +209,11 @@ TEST_F(RestConfigTest, basic_flow)
         string msg2 = "POST /add-test HTTP/1.1\r\nContent-Length: 10\r\n\r\n{\"num\": 5}";
         EXPECT_EQ(write(file_descriptor2, msg2.data(), msg2.size()), static_cast<int>(msg2.size()));
 
+        EXPECT_EQ(connect(file_descriptor3, (struct sockaddr*)&sa, sizeof(struct sockaddr)), 0)
+            << "file_descriptor3 Error: "
+            << strerror(errno);
+        string msg3 = "GET /api/123 HTTP/1.1\r\n\r\n";
+        EXPECT_EQ(write(file_descriptor3, msg3.data(), msg3.size()), static_cast<int>(msg3.size()));
         while(!TestServer::g_num) {
             mainloop->yield(true);
         }
@@ -215,6 +226,14 @@ TEST_F(RestConfigTest, basic_flow)
             mainloop->yield(true);
         }
 
+        struct pollfd s_poll3;
+        s_poll3.fd = file_descriptor3;
+        s_poll3.events = POLLIN;
+        s_poll3.revents = 0;
+        while(poll(&s_poll3, 1, 0) <= 0) {
+            mainloop->yield(true);
+        }
+
         mainloop->stopAll();
     };
     mainloop->addOneTimeRoutine(
@@ -233,6 +252,11 @@ TEST_F(RestConfigTest, basic_flow)
         string(respose, 76),
         "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 6\r\n\r\nblabla"
     );
+    EXPECT_EQ(read(file_descriptor3, respose, 1000), 73);
+    EXPECT_EQ(
+        string(respose, 73),
+        "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 3\r\n\r\n123"
+    );
 }
 
 string

core/socket_is/socket_is.cc

@@ -182,9 +182,11 @@ public:
     Maybe<unique_ptr<SocketInternal>>
     acceptConn(bool is_blocking, const string &authorized_ip = "")
     {
-        dbgAssert(is_server_socket) << alert << "Failed to accept new connections from a client socket";
-        dbgAssert(socket_int > 0) << alert << "Called with uninitialized server socket";
-
+        if (!(is_server_socket) || !(socket_int > 0)) {
+            dbgAssertOpt(is_server_socket) << alert << "Failed to accept new connections from a client socket";
+            dbgAssertOpt(socket_int > 0) << alert << "Called with uninitialized server socket";
+            return genError("Failed due to internal error");
+        }
         dbgDebug(D_SOCKET) << "Attempt to accept new socket. Server Socket FD: " << socket_int;
         int client_socket;
         if (!authorized_ip.empty()) {

core/time_proxy/time_proxy.cc

@@ -52,9 +52,12 @@ public:
     setMonotonicTime(microseconds new_time) override
     {
         if (is_monotomic_set) {
-            dbgAssert((new_time+monotonic_delta) >= monotonic_now)
+            if ((new_time+monotonic_delta) < monotonic_now) {
+                dbgAssertOpt((new_time+monotonic_delta) >= monotonic_now)
                     << AlertInfo(AlertTeam::CORE, "time proxy")
                     << "Monotonic time must not go back!";
+                return;
+            }
         } else {
             // The first time that the monotonic time is been set, we take the current value to be the base line.
             // This is in order to avoid the clock going backwards.

deployment/docker-compose/apisix/.env

@@ -0,0 +1,55 @@
+## .env file for docker-compose deployments of open-appsec integrated with APISIX
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+## Make sure to have a valid apisix configuration for APISIX in standalone mode in the following file:
+## For deployment of a simple lab testing environment, you can deploy the example configuration provided
+## for the vulnerable juice-shop container, see instructions further below.
+APISIX_CONFIG=./apisix-config/apisix.yaml
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also adjust the apisix.yaml file in APISIX_CONFIG folder
+## to include route and node configuration for forwarding external traffic to the juiceshop-backend container
+## (apisix listens by default for HTTP/HTTPS on port 9080/9443) 
+## you can use the example file available here:
+## https://raw.githubusercontent.com/openappsec/openappsec/refs/heads/main/examples/juiceshop/apisix.yaml
+## in the appsec-apisix service definition
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/apisix/apisix-config/apisix.yaml

@@ -0,0 +1,9 @@
+routes:
+  -
+    uri: /
+    upstream:
+        nodes:
+            "juiceshop-backend:3000": 1
+        type: roundrobin
+
+#END

deployment/docker-compose/apisix/docker-compose.yaml

@@ -0,0 +1,131 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with APISIX
+##
+
+version: "3.9"
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - registered_server=APISIX Server
+    ipc: shareable
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-apisix:
+    image: ghcr.io/openappsec/apisix-attachment:${APPSEC_VERSION}
+    container_name: appsec-apisix
+    ipc: service:appsec-agent
+    restart: always
+    environment:
+      - APISIX_STAND_ALONE=true
+    volumes:
+      - ${APISIX_CONFIG}:/usr/local/apisix/conf/apisix.yaml:ro
+    ports:
+      - "9080:9080/tcp" # HTTP API port
+      - "9443:9443/tcp" # HTTPS API port
+      - "9180:9180/tcp" # Admin API HTTP port
+      - "9091:9091/tcp" # Admin API HTTPS port
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: always
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: always
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: always
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: always
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/envoy/docker-compose.yaml

@@ -0,0 +1,135 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with Envoy
+##
+
+version: "3.9"
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - registered_server="Envoy Server"
+    ipc: shareable
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-envoy:
+    image: openappsec-envoy:${APPSEC_VERSION}
+# for docs:    image: ghcr.io/openappsec/envoy-attachment:${APPSEC_VERSION}
+    container_name: appsec-envoy
+    ipc: service:appsec-agent
+    restart: unless-stopped
+    environment:
+      - ENVOY_UID=0
+      - CONCURRENCY_CALC=${ENVOY_CONCURRENCY_CALC}
+      - CONCURRENCY_NUMBER=${ENVOY_CONCURRENCY_NUMBER}
+    volumes:
+      - ${ENVOY_CONFIG}:/envoy.yaml
+    command: -c /envoy.yaml
+##  adjustment of threads is possible as follows:
+#   command: -c /envoy.yaml --concurrency ${ENVOY_CONCURRENCY}
+
+    ports:
+      - "80:80"
+      - "443:443"
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/envoy/envoy-config/envoy.yaml

@@ -0,0 +1,56 @@
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 80
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          http_filters:
+          ## The following 10 lines are required to load the envoy attachment filter for open-appsec
+          - name: envoy.filters.http.golang
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config
+              library_id: cp_nano_filter
+              library_path: "/usr/lib/libenvoy_attachment.so"
+              plugin_name: cp_nano_filter
+              plugin_config:
+                "@type": type.googleapis.com/xds.type.v3.TypedStruct
+                value:
+                  prefix_localreply_body: "Configured local reply from go"
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+##
+## The following lines allow you to deploy routing of ingress traffic to the optional juice-shop example container available in the open-appsec docker-compose.yaml file.
+##
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: juiceshop
+
+  clusters:
+  - name: juiceshop
+    type: STRICT_DNS
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: juiceshop
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: juiceshop-backend
+                port_value: 3000

deployment/docker-compose/kong/.env

@@ -0,0 +1,57 @@
+## .env file for docker-compose deployments of open-appsec integrated with Kong
+## For more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+## Make sure to have a valid Kong declarative configuration file kong.yaml in the folder specified for KONG_CONFIG.
+## For deployment of a simple lab testing environment, you can deploy the example configuration provided
+## for the vulnerable juice-shop container, see instructions further below.
+KONG_CONFIG=./kong-config
+
+## For Kong Gateway Enterprise Edition set KONG_IMAGE to kong-gateway-attachment instead of kong-attachment
+KONG_IMAGE=kong-attachment
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also adjust the kong.yaml file in KONG_CONFIG folder
+## to include service and route configuration for forwarding external traffic to the juiceshop-backend container
+## (kong listens by default for HTTP/HTTPS on port 8000/8443) 
+## you can use the example file available here:
+## https://raw.githubusercontent.com/openappsec/openappsec/examples/juiceshop/kong/kong.yaml
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/kong/docker-compose.yaml

@@ -0,0 +1,135 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with Kong
+##
+
+version: "3.9"
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - registered_server=Kong Server
+    ipc: shareable
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-kong:
+    image: ghcr.io/openappsec/${KONG_IMAGE}:${APPSEC_VERSION}
+    container_name: appsec-kong
+    ipc: service:appsec-agent
+## This docker compose deploys Kong in DB-less mode with declarative Kong configuration
+## please make sure to have a valid config present in {KONG_CONFIG}:
+    environment:
+      - KONG_DATABASE=off
+      - KONG_DECLARATIVE_CONFIG=/opt/kong/kong.yaml
+    volumes:
+      - ${KONG_CONFIG}:/opt/kong
+    restart: unless-stopped
+    ports:
+      - "8000:8000"
+      - "8443:8443"
+      - "127.0.0.1:8001:8001"
+      - "127.0.0.1:8444:8444"
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+      
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+    profiles:
+      - juiceshop
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/kong/kong-config/kong.yaml

@@ -0,0 +1,9 @@
+_format_version: "3.0"
+
+services:
+  - name: juiceshop-service
+    url: http://juiceshop-backend:3000
+    routes:
+      - name: juiceshop-route
+        paths:
+          - /

deployment/docker-compose/nginx-proxy-manager-centrally-managed/.env

@@ -0,0 +1,50 @@
+## .env file for docker-compose deployments of open-appsec integrated with NGINX Proxy Manager
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+# Volume mounts for NGINX Proxy Manager have been moved here as well allowing configuration via .env file 
+NPM_DATA=./data
+NPM_LETSENCRYPT=./letsencrypt
+
+## To connect your deployment to central open-appsec Web UI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also create a new proxy host in the NGINX Proxy Manager WebUI 
+## which accepts traffic on http port 80 and proxies traffic to juiceshop-backend on port 3000.
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/nginx-proxy-manager-centrally-managed/docker-compose.yaml

@@ -0,0 +1,132 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with NGINX Proxy Manager
+## with open-appsec management via central open-appsec WebUI (SaaS)
+##
+
+version: '3.9'
+
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    ipc: shareable
+    restart: unless-stopped
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - nginxproxymanager=true
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-nginx-proxy-manager:
+    container_name: appsec-nginx-proxy-manager
+    image: ghcr.io/openappsec/nginx-proxy-manager-centrally-managed-attachment:${APPSEC_VERSION}
+    ipc: service:appsec-agent
+    restart: unless-stopped
+    ports:
+      - 80:80 # Public HTTP Port
+      - 443:443 # Public HTTPS Port
+      - 81:81 # Admin Web Port
+    volumes:
+      - ${NPM_DATA}:/data
+      - ${NPM_LETSENCRYPT}:/etc/letsencrypt
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+      
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+    profiles:
+      - juiceshop
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/nginx-proxy-manager/.env

@@ -0,0 +1,48 @@
+## .env file for docker-compose deployments of open-appsec integrated with NGINX Proxy Manager
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to true for changes
+## of open-appsec configuration in the NGINX Proxy Manager WebUI to be applied automatically
+APPSEC_AUTO_POLICY_LOAD=true
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+# Volume mounts for NGINX Proxy Manager have been moved here as well allowing configuration via .env file 
+NPM_DATA=./data
+NPM_LETSENCRYPT=./letsencrypt
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also create a new proxy host in the NGINX Proxy Manager WebUI 
+## which accepts traffic on http port 80 and proxies traffic to juiceshop-backend on port 3000.
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/nginx-proxy-manager/docker-compose.yaml

@@ -0,0 +1,134 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with NGINX Proxy Manager
+## with open-appsec management via NGINX Proxy Manager WebUI
+##
+
+version: '3.9'
+
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    ipc: shareable
+    restart: unless-stopped
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - nginxproxymanager=true
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-nginx-proxy-manager:
+    container_name: appsec-nginx-proxy-manager
+    image: ghcr.io/openappsec/nginx-proxy-manager-attachment:${APPSEC_VERSION}
+    ipc: service:appsec-agent
+    restart: unless-stopped
+    ports:
+      - 80:80 # Public HTTP Port
+      - 443:443 # Public HTTPS Port
+      - 81:81 # Admin Web Port
+    volumes:
+      - ${NPM_DATA}:/data
+      - ${NPM_LETSENCRYPT}:/etc/letsencrypt
+      - ${APPSEC_LOGS}:/ext/appsec-logs
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+      
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+    profiles:
+      - juiceshop
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/nginx-unified/.env

@@ -0,0 +1,54 @@
+## .env file for docker-compose deployments of open-appsec integrated with NGINX
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+## Make sure to have a valid NGINX configuration file default.conf in the folder specified for NGINX_CONFIG.
+## For deployment of a simple lab testing environment, you can deploy the example configuration provided
+## for the vulnerable juice-shop container, see instructions further below.
+NGINX_CONFIG=./nginx-config
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also adjust the nginx.conf file in NGINX_CONFIG folder
+## to include a proxy_pass directive forwarding external traffic on e.g. port 80 to the juiceshop-backend container
+## you can use the example file available here:
+## https://raw.githubusercontent.com/openappsec/openappsec/examples/juiceshop/nginx/default.conf
+## place the file above in NGINX_CONFIG folder
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/nginx-unified/docker-compose.yaml

@@ -0,0 +1,126 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec deployments of NGINX unified container
+##
+
+version: "3.9"
+services:
+  appsec-agent-nginx-unified:
+    image: ghcr.io/openappsec/agent-unified:${APPSEC_VERSION}
+    container_name: appsec-agent-nginx-unified
+    restart: unless-stopped
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+    ipc: shareable
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+      - ${NGINX_CONFIG}:/etc/nginx/conf.d
+## advanced configuration - volume mount for nginx.conf file:
+## to change global instructions it's possible to also mount your own nginx.conf file by uncommenting the two lines below
+## make sure to include the line starting with "load_module" which loads the appsec attachment
+## and is included in /etc/nginx/conf.d/nginx.conf file as part of the nginx-attachment container
+#     - ${NGINX_CONF_FILE}:/etc/nginx/nginx.conf
+    ports:
+      - "80:80"
+      - "443:443"
+    command: /cp-nano-agent
+  
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent-nginx-unified
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+    profiles:
+      - juiceshop
+
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/nginx-unified/nginx-config/default.conf

@@ -0,0 +1,47 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        proxy_pass http://juiceshop-backend:3000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }    
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+}

deployment/docker-compose/nginx/.env

@@ -0,0 +1,54 @@
+## .env file for docker-compose deployments of open-appsec integrated with NGINX
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+## Make sure to have a valid NGINX configuration file default.conf in the folder specified for NGINX_CONFIG.
+## For deployment of a simple lab testing environment, you can deploy the example configuration provided
+## for the vulnerable juice-shop container, see instructions further below.
+NGINX_CONFIG=./nginx-config
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to also adjust the nginx.conf file in NGINX_CONFIG folder
+## to include a proxy_pass directive forwarding external traffic on e.g. port 80 to the juiceshop-backend container
+## you can use the example file available here:
+## https://raw.githubusercontent.com/openappsec/openappsec/examples/juiceshop/nginx/default.conf
+## place the file above in NGINX_CONFIG folder
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/nginx/docker-compose.yaml

@@ -0,0 +1,136 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with NGINX
+##
+
+version: "3.9"
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - registered_server="NGINX Server"
+    ipc: shareable
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+    
+  appsec-nginx:
+    image: ghcr.io/openappsec/nginx-attachment:${APPSEC_VERSION}
+    container_name: appsec-nginx
+    ipc: service:appsec-agent
+    restart: unless-stopped
+    volumes:
+      - ${NGINX_CONFIG}:/etc/nginx/conf.d
+
+## advanced configuration - volume mount for nginx.conf file:
+## To change global instructions it's possible to also mount your own nginx.conf file by uncommenting the line below
+## then specify a desired local folder for NGINX_CONF_FILE in the .env file.
+## In the nginx.conf file make sure to include the line starting with "load_module" which loads the appsec attachment
+## and is included in /etc/nginx/conf.d/nginx.conf file as part of the nginx-attachment container.
+#     - ${NGINX_CONF_FILE}:/etc/nginx/nginx.conf
+
+    ports:
+      - "80:80"
+      - "443:443"
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+    profiles:
+      - juiceshop
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/nginx/nginx-config/default.conf

@@ -0,0 +1,47 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        proxy_pass http://juiceshop-backend:3000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }    
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+}

deployment/docker-compose/swag/.env

@@ -0,0 +1,76 @@
+## .env file for docker-compose deployments of open-appsec integrated with SWAG
+## for more info see https://docs.openappsec.io
+
+APPSEC_VERSION=latest
+APPSEC_CONFIG=./appsec-config
+APPSEC_DATA=./appsec-data
+APPSEC_LOGS=./appsec-logs
+APPSEC_LOCALCONFIG=./appsec-localconfig
+
+## Make sure the parameter APPSEC_AUTO_POLICY_LOAD is set to false when centrally managing
+## open-appsec configuration via open-appsec Web UI.
+## You can optionally set it to true when using local, declarative management for open-appsec,
+## declarative configuration will then get applied automatically when changed.
+APPSEC_AUTO_POLICY_LOAD=false
+
+## Example for configuring HTTPS Proxy:
+## APPSEC_HTTPS_PROXY=user:password@proxy_address:port
+APPSEC_HTTPS_PROXY=
+
+APPSEC_SMART_SYNC_STORAGE=./appsec-smartsync-storage
+APPSEC_USER_EMAIL=user@email.com
+APPSEC_DB_PASSWORD=pass
+APPSEC_DB_USER=postgres
+APPSEC_DB_HOST=appsec-db
+APPSEC_POSTGRES_STORAGE=./appsec-postgres-data
+
+## Most relevant SWAG parameters have been moved here as well allowing configuration via .env file 
+SWAG_CONFIG=./swag-config
+## Make sure to have a valid nginx config default.conf in SWAG_NGINX_SITE_CONFS folder
+SWAG_NGINX_SITE_CONFS=./swag-nginx-site-confs
+## Make sure to have valid *.conf proxy configuration in SWAG_NGINX_PROXY_CONFS folder
+SWAG_PROXY_CONFS=./swag-proxy-confs
+## For deployment of a simple lab testing environment, you can deploy the example configuration provided
+## for the vulnerable juice-shop container, see instructions further below.
+
+SWAG_TZ=Etc/UTC
+SWAG_VALIDATION=http # configure "http" or "dns" as validation modes 
+SWAG_DNSPLUGIN="" # configure e.g. "route53" or some other DNS Plugin supported by SWAG if you set "dns" above
+
+## Examples parameters for "route53" DNS plugin (AWS DNS service), you can add others here as required,
+## when you do make sure to also add them to the docker compose file
+SWAG_AWS_ACCESS_KEY_ID=""
+SWAG_AWS_SECRET_ACCESS_KEY=""
+##
+
+SWAG_STAGING=true ## switch to 'false' after successful testing
+SWAG_URL=yourdomain.url
+SWAG_SUBDOMAINS=""
+SWAG_ONLY_SUBDOMAINS=""
+## replace yourdomain.url with your own domain
+## make sure your domain's public IP resolves to
+## the docker host for Let's Encrypt cert generation to succeed
+
+## To connect your deployment to central open-appsec WebUI provide the token for a profile
+## which you created in open-appsec WebUI at https://my.openappsec.io
+## Example: APPSEC_AGENT_TOKEN=111-22222-111
+APPSEC_AGENT_TOKEN=
+
+## Important: When not providing token for connection to central WebUI:
+## Make sure to add the value "standalone" to the COMPOSE_PROFILES value, this will enable
+## sharing of learning between processes and allow you to perform tuning locally on CLI
+COMPOSE_PROFILES=
+
+## JUICE SHOP DEMO CONTAINER:
+## In order to deploy the optional, additional, vulnerable juiceshop container (for demo and testing purposes only!):
+## Add the value "juiceshop" to the COMPOSE_PROFILES value above.
+
+## Make sure to put a juiceshop.subfolder.conf file in SWAG_PROXY_CONFS folder
+## for proxying external traffic to the juiceshop-backend container and also adjust the NGINX default.conf file in SWAG_NGINX_SITE_CONFS folder
+## you can use the example files available here:
+## https://raw.githubusercontent.com/openappsec/openappsec/examples/juiceshop/swag/juiceshop.subfolder.conf
+## https://raw.githubusercontent.com/openappsec/openappsec/examples/juiceshop/swag/default.conf
+## note that juiceshop container listens on HTTP port 3000 by default
+
+## Note that COMPOSE_PROFILES can also receive multiple values, e.g. as shown here:
+## COMPOSE_PROFILES=standalone,juiceshop

deployment/docker-compose/swag/docker-compose.yaml

@@ -0,0 +1,145 @@
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+##
+## Docker compose file for open-appsec integrated with SWAG
+##
+
+version: "3.9"
+services:
+  appsec-agent:
+    image: ghcr.io/openappsec/agent:${APPSEC_VERSION}
+    container_name: appsec-agent
+    restart: unless-stopped
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - LEARNING_HOST=appsec-smartsync
+      - TUNING_HOST=appsec-tuning-svc
+      - https_proxy=${APPSEC_HTTPS_PROXY}
+      - user_email=${APPSEC_USER_EMAIL}
+      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+      - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
+      - registered_server=SWAG Server
+    ipc: shareable
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+      - ${APPSEC_DATA}:/etc/cp/data
+      - ${APPSEC_LOGS}:/var/log/nano_agent
+      - ${APPSEC_LOCALCONFIG}:/ext/appsec
+    command: /cp-nano-agent
+
+  appsec-swag:
+    image: ghcr.io/openappsec/swag-attachment:latest
+    container_name: appsec-swag
+    ipc: service:appsec-agent
+    restart: unless-stopped
+    cap_add:
+      - NET_ADMIN
+    environment:
+      - PUID=1000
+      - PGID=1000
+      - TZ=${SWAG_TZ}
+      - URL=${SWAG_URL}
+      - VALIDATION=${SWAG_VALIDATION}
+      - DNSPLUGIN=${SWAG_DNSPLUGIN}
+      - AWS_ACCESS_KEY_ID=${SWAG_AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${SWAG_AWS_SECRET_ACCESS_KEY}
+      - SUBDOMAINS=${SWAG_SUBDOMAINS}
+      - ONLY_SUBDOMAINS=${SWAG_ONLY_SUBDOMAINS}
+## see https://docs.linuxserver.io/images/docker-swag/ for
+## more cert generation/validation options
+      - STAGING=${SWAG_STAGING}
+    volumes:
+      - ${SWAG_CONFIG}:/config
+      - ${SWAG_NGINX_SITE_CONFS}:/config/nginx/site-confs
+      - ${SWAG_PROXY_CONFS}:/config/nginx/proxy-confs
+    ports:
+      - 443:443
+      - 80:80 ## optional
+
+  appsec-smartsync:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync:${APPSEC_VERSION}
+    container_name: appsec-smartsync
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+    restart: unless-stopped
+    depends_on:
+      - appsec-shared-storage
+
+  appsec-shared-storage:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-shared-files:${APPSEC_VERSION}
+    container_name: appsec-shared-storage
+    ipc: service:appsec-agent
+    restart: unless-stopped
+## if you do not want to run this container as "root" user you can comment it out and instead run the below command after the deployment
+## docker exec -u root appsec-shared-storage chown -R appuser:appuser /db
+    user: root
+    volumes:
+      - ${APPSEC_SMART_SYNC_STORAGE}:/db:z
+## instead of using local storage for local learning (see line above)
+## you can also configure central nfs storage by configuring nfs volume (uncomment the relevant section at end of this file)
+## use a shared nfs storage which is recommended in redundant deployments (uncomment line below, comment out the line above) 
+#     - learning_nfs:/db:z
+
+  appsec-tuning-svc:
+    profiles:
+      - standalone
+    image: ghcr.io/openappsec/smartsync-tuning:${APPSEC_VERSION}
+    container_name: appsec-tuning-svc
+    environment:
+      - SHARED_STORAGE_HOST=appsec-shared-storage
+      - QUERY_DB_PASSWORD=${APPSEC_DB_PASSWORD}
+      - QUERY_DB_HOST=${APPSEC_DB_HOST}
+      - QUERY_DB_USER=${APPSEC_DB_USER}
+## only relevant when deploying own DB 
+#     - SSLMODE:
+    restart: unless-stopped
+    volumes:
+      - ${APPSEC_CONFIG}:/etc/cp/conf
+    depends_on:
+      - appsec-shared-storage
+      - appsec-db
+      
+  appsec-db:
+    profiles:
+      - standalone
+    image: postgres
+    container_name: appsec-db
+    restart: unless-stopped
+    environment:
+      - POSTGRES_PASSWORD=${APPSEC_DB_PASSWORD}
+      - POSTGRES_USER=${APPSEC_DB_USER} 
+    volumes:
+      - ${APPSEC_POSTGRES_STORAGE}:/var/lib/postgresql/data
+
+## example juice-shop backend container (vulnerable webserver, USE ONLY FOR TESTING AND IN LAB ENV)
+  juiceshop-backend:
+    image: bkimminich/juice-shop:latest
+    container_name: juiceshop-backend
+
+
+## advanced configuration: learning_nfs volume for nfs storage in shared_storage container
+##
+## when configuring nfs storage in shared_storage container configuration above, make sure to also specify learning_nfs volume (see example below for using AWS EFS storage)
+##
+#volumes:
+# learning_nfs:
+#   driver: local
+#   driver_opts:
+#     type: nfs
+#     o: addr=fs-abcdef.efs.eu-west-1.amazonaws.com,rw,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
+#     device: ":/"

deployment/docker-compose/swag/swag-nginx-site-confs/default.conf

@@ -0,0 +1,84 @@
+## Version 2024/07/16 - Changelog: https://github.com/linuxserver/docker-swag/commits/master/root/defaults/nginx/site-confs/default.conf.sample
+
+# redirect all traffic to https
+server {
+    listen 80 default_server;
+    listen [::]:80 default_server;
+
+    location / {
+        return 301 https://$host$request_uri;
+    }
+}
+
+# main server block
+server {
+    listen 443 ssl default_server;
+    listen [::]:443 ssl default_server;
+
+    server_name _;
+
+    include /config/nginx/ssl.conf;
+
+#    root /config/www;
+#    index index.html index.htm index.php;
+
+    # enable subfolder method reverse proxy confs
+    include /config/nginx/proxy-confs/*.subfolder.conf;
+
+    # enable for ldap auth (requires ldap-location.conf in the location block)
+    #include /config/nginx/ldap-server.conf;
+
+    # enable for Authelia (requires authelia-location.conf in the location block)
+    #include /config/nginx/authelia-server.conf;
+
+    # enable for Authentik (requires authentik-location.conf in the location block)
+    #include /config/nginx/authentik-server.conf;
+
+    #location / {
+        # enable for basic auth
+        #auth_basic "Restricted";
+        #auth_basic_user_file /config/nginx/.htpasswd;
+
+        # enable for ldap auth (requires ldap-server.conf in the server block)
+        #include /config/nginx/ldap-location.conf;
+
+        # enable for Authelia (requires authelia-server.conf in the server block)
+        #include /config/nginx/authelia-location.conf;
+
+        # enable for Authentik (requires authentik-server.conf in the server block)
+        #include /config/nginx/authentik-location.conf;
+
+    #    try_files $uri $uri/ /index.html /index.htm /index.php$is_args$args;
+    #}
+
+    location ~ ^(.+\.php)(.*)$ {
+        # enable the next two lines for http auth
+        #auth_basic "Restricted";
+        #auth_basic_user_file /config/nginx/.htpasswd;
+
+        # enable for ldap auth (requires ldap-server.conf in the server block)
+        #include /config/nginx/ldap-location.conf;
+
+        # enable for Authelia (requires authelia-server.conf in the server block)
+        #include /config/nginx/authelia-location.conf;
+
+        # enable for Authentik (requires authentik-server.conf in the server block)
+        #include /config/nginx/authentik-location.conf;
+
+        fastcgi_split_path_info ^(.+\.php)(.*)$;
+        if (!-f $document_root$fastcgi_script_name) { return 404; }
+        fastcgi_pass 127.0.0.1:9000;
+        fastcgi_index index.php;
+        include /etc/nginx/fastcgi_params;
+    }
+
+    # deny access to .htaccess/.htpasswd files
+    location ~ /\.ht {
+        deny all;
+    }
+}
+
+# enable subdomain method reverse proxy confs
+include /config/nginx/proxy-confs/*.subdomain.conf;
+# enable proxy cache for auth
+proxy_cache_path cache/ keys_zone=auth_cache:10m;

deployment/docker-compose/swag/swag-proxy-confs/juiceshop.subfolder.conf

@@ -0,0 +1,22 @@
+location / {
+    # enable the next two lines for http auth
+    #auth_basic "Restricted";
+    #auth_basic_user_file /config/nginx/.htpasswd;
+
+    # enable for ldap auth (requires ldap-server.conf in the server block)
+    #include /config/nginx/ldap-location.conf;
+
+    # enable for Authelia (requires authelia-server.conf in the server block)
+    #include /config/nginx/authelia-location.conf;
+
+    # enable for Authentik (requires authentik-server.conf in the server block)
+    #include /config/nginx/authentik-location.conf;
+
+    include /config/nginx/proxy.conf;
+    include /config/nginx/resolver.conf;
+    set $upstream_app juiceshop-backend;
+    set $upstream_port 3000;
+    set $upstream_proto http;
+    proxy_pass $upstream_proto://$upstream_app:$upstream_port;
+
+}

deployment/nginx/docker-compose.yaml

@@ -9,7 +9,7 @@ services:
       - TUNING_HOST=appsec-tuning-svc
       - https_proxy=${APPSEC_HTTPS_PROXY}
       - user_email=${USER_EMAIL}
-      - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
+    # - AGENT_TOKEN=${APPSEC_AGENT_TOKEN}
       - autoPolicyLoad=${APPSEC_AUTO_POLICY_LOAD}
       - registered_server="NGINX Server"
     ipc: shareable
@@ -96,7 +96,6 @@ services:
 ## to include a proxy_pass directive forwarding external traffic on e.g. port 80 to the juiceshop-backend container
 ## you can use the example file available here:
 ## https://raw.githubusercontent.com/openappsec/openappsec/refs/heads/main/examples/juiceshop/default.conf
-## place the file above in {NGINX_CONF_DIR} and uncomment the two lines for creating a volume mount
 ## in the appsec-nginx service definition
 ## note that juiceshop container listens on HTTP port 3000 by default
 #

deployment/nginx/nginx-config/default.conf

@@ -0,0 +1,47 @@
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  _;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        proxy_pass http://juiceshop-backend:3000;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+    }    
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+}

nodes/CMakeLists.txt

@@ -14,3 +14,4 @@ add_subdirectory(orchestration)
 add_subdirectory(agent_cache)
 add_subdirectory(http_transaction_handler)
 add_subdirectory(attachment_registration_manager)
+add_subdirectory(central_nginx_manager)

nodes/central_nginx_manager/CMakeLists.txt

@@ -0,0 +1,34 @@
+add_subdirectory(package)
+
+add_executable(cp-nano-central-nginx-manager main.cc)
+
+target_link_libraries(cp-nano-central-nginx-manager
+    -Wl,--start-group
+    ${COMMON_LIBRARIES}
+
+    generic_rulebase
+    generic_rulebase_evaluators
+    ip_utilities
+    version
+    signal_handler
+
+	central_nginx_manager
+    nginx_message_reader
+    rate_limit_comp
+    rate_limit_config
+    nginx_utils
+	http_transaction_data
+    -Wl,--end-group
+)
+
+add_dependencies(cp-nano-central-nginx-manager ngen_core)
+
+install(TARGETS cp-nano-central-nginx-manager DESTINATION bin)
+install(TARGETS cp-nano-central-nginx-manager DESTINATION central_nginx_manager/bin)
+
+gen_package(
+	install-cp-nano-central-nginx-manager.sh
+	central_nginx_manager
+	./install-cp-nano-central-nginx-manager.sh
+	Check Point Central NGINX Manager Nano Service Version ${PACKAGE_VERSION} Install Package
+)

nodes/central_nginx_manager/main.cc

@@ -0,0 +1,31 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "central_nginx_manager.h"
+
+#include "components_list.h"
+#include "nginx_message_reader.h"
+
+using namespace std;
+
+int
+main(int argc, char **argv)
+{
+    NodeComponents<CentralNginxManager, NginxMessageReader> comps;
+
+    comps.registerGlobalValue<bool>("Is Rest primary routine", true);
+    comps.registerGlobalValue<uint>("Nano service API Port Primary", 7555);
+    comps.registerGlobalValue<uint>("Nano service API Port Alternative", 7556);
+
+    return comps.run("Central NGINX Manager", argc, argv);
+}

nodes/central_nginx_manager/package/CMakeLists.txt

@@ -0,0 +1,4 @@
+install(FILES install-cp-nano-central-nginx-manager.sh DESTINATION central_nginx_manager PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ)
+install(FILES cp-nano-central-nginx-manager.cfg  DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ)
+install(FILES cp-nano-central-nginx-manager-conf.json  DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ)
+install(FILES cp-nano-central-nginx-manager-debug-conf.json  DESTINATION central_nginx_manager/conf PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ)

nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-conf.json

@@ -0,0 +1 @@
+{}

nodes/central_nginx_manager/package/cp-nano-central-nginx-manager-debug-conf.json

@@ -0,0 +1,11 @@
+{
+  "Debug": [
+    {
+      "Streams": [
+        {
+          "Output": "/var/log/nano_agent/cp-nano-central-nginx-manager.dbg"
+        }
+      ]
+    }
+  ]
+}

nodes/central_nginx_manager/package/cp-nano-central-nginx-manager.cfg

@@ -0,0 +1,2 @@
+srv_debug_file=/var/log/nano_agent/cp-nano-central-nginx-manager.dbg
+srv_log_file=/var/log/nano_agent/cp-nano-central-nginx-manager.log

nodes/central_nginx_manager/package/install-cp-nano-central-nginx-manager.sh

@@ -0,0 +1,171 @@
+#!/bin/sh
+
+FORCE_STDOUT=true
+INSTALLATION_LOG_FILE="/var/log/nano_agent/install-cp-nano-central-nginx-manager.log"
+INSTALLATION_TIME=$(date)
+CONF_PATH=/etc/cp/conf
+SERVICE_PATH=/etc/cp/centralNginxManager
+WATCHDOG_PATH=/etc/cp/watchdog/cp-nano-watchdog
+NGINX_CONF_PATH="/etc/nginx/nginx.conf"
+CENTRAL_NGINX_CONF_PATH="/tmp/central_nginx.conf"
+
+export INSTALL_COMMAND
+is_install="$(command -v install)"
+if [ -z ${is_install} ]; then
+    INSTALL_COMMAND="cp -f"
+    cp_print "[WARNING]: install command not found - using cp instead" ${FORCE_STDOUT}
+else
+    INSTALL_COMMAND=install
+fi
+
+mkdir -p /var/log/nano_agent
+mkdir -p /tmp/
+
+cp_print()
+{
+    var_text=${1}
+    var_std_out=${2}
+    touch ${INSTALLATION_LOG_FILE}
+    if [ -n "${var_std_out}" ]; then
+        if [ "${var_std_out}" = "true" ]; then
+            printf "%b\n" "${var_text}"
+        fi
+    fi
+    printf "%b\n" "${var_text}" >> ${INSTALLATION_LOG_FILE}
+}
+
+cp_exec()
+{
+    var_cmd=${1}
+    var_std_out=${2}
+    # Send exec output to RES
+    RES=$(${var_cmd} 2>&1)
+    if [ -n "${RES}" ]; then
+        cp_print "${RES}" "${var_std_out}"
+    fi
+}
+
+is_nginx_installed()
+{
+    if [ -x "$(command -v nginx)" ]; then
+        return 0
+    fi
+
+    return 1
+}
+
+get_nginx_conf_path()
+{
+    if ! is_nginx_installed; then
+        return
+    fi
+
+    NGINX_CONF_PATH=$(nginx -V 2>&1 | grep -o '\--conf-path=[^ ]*' | cut -d= -f2)
+    if [ -z "${NGINX_CONF_PATH}" ]; then
+        NGINX_CONF_PATH="/etc/nginx/nginx.conf"
+    fi
+}
+
+run_installation()
+{
+    cp_print "Starting installation of Check Point Central NGINX Manager [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT}
+    cp_exec "${WATCHDOG_PATH} --un-register ${SERVICE_PATH}/cp-nano-central-nginx-manager"
+    cp_exec "mkdir -p ${SERVICE_PATH}"
+    cp_exec "mkdir -p ${CONF_PATH}/centralNginxManager/shared"
+
+    cp_exec "touch ${CONF_PATH}/centralNginxManager/shared/central_nginx_shared.conf"
+    cp_exec "${INSTALL_COMMAND} bin/cp-nano-central-nginx-manager ${SERVICE_PATH}/cp-nano-central-nginx-manager"
+    cp_exec "${INSTALL_COMMAND} bin/cp-nano-nginx-conf-collector ${SERVICE_PATH}/cp-nano-nginx-conf-collector"
+    cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager.cfg ${CONF_PATH}/cp-nano-central-nginx-manager.cfg"
+    cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager-conf.json ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json"
+    cp_exec "${INSTALL_COMMAND} conf/cp-nano-central-nginx-manager-debug-conf.json ${CONF_PATH}/cp-nano-central-nginx-manager-debug-conf.json"
+    cp_exec "chmod +x ${SERVICE_PATH}/cp-nano-central-nginx-manager"
+    cp_exec "chmod +x ${SERVICE_PATH}/cp-nano-nginx-conf-collector"
+    cp_exec "chmod 600 ${CONF_PATH}/cp-nano-central-nginx-manager.cfg"
+    cp_exec "chmod 600 ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json"
+
+    cp_exec "${WATCHDOG_PATH} --register ${SERVICE_PATH}/cp-nano-central-nginx-manager"
+    cp_print "Installation completed successfully." ${FORCE_STDOUT}
+}
+
+usage()
+{
+    echo "Check Point: available flags are"
+    echo "--install           : install central nginx manager"
+    echo "--uninstall         : remove central nginx manager"
+    echo "--pre_install_test  : run Pre-installation test for central nginx manager install package"
+    echo "--post_install_test : run Post-installation test for central nginx manager install package"
+    exit 255
+}
+
+run_uninstall()
+{
+    cp_print "Starting uninstall of Check Point Central NGINX Manager service [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT}
+
+    cp_exec "${WATCHDOG_PATH} --un-register ${SERVICE_PATH}/cp-nano-central-nginx-manager"
+    cp_exec "rm -rf ${SERVICE_PATH}/"
+    cp_exec "rm -f ${CONF_PATH}/cp-nano-central-nginx-manager.cfg"
+    cp_exec "rm -f ${CONF_PATH}/cp-nano-central-nginx-manager-conf.json"
+
+    if [ -f "${CENTRAL_NGINX_CONF_PATH}.base" ]; then
+        cp_print "Restoring central NGINX configuration file" ${FORCE_STDOUT}
+        cp_exec "${INSTALL_COMMAND} ${CENTRAL_NGINX_CONF_PATH}.base ${NGINX_CONF_PATH}"
+        if is_nginx_installed; then
+            if nginx -t > /dev/null 2>&1; then
+                cp_exec "nginx -s reload"
+            else
+                cp_print "Could not reload central NGINX configuration, run 'nginx -t' for more details." ${FORCE_STDOUT}
+            fi
+        fi
+    fi
+
+    if [ -f "${NGINX_CONF_PATH}.orig" ]; then
+        cp_print "Original (pre Check Point Nano Agent deployment) NGINX configuration file can be found at: ${NGINX_CONF_PATH}.orig" ${FORCE_STDOUT}
+    fi
+    cp_print "Check Point Central NGINX Manager service was removed successfully\n" ${FORCE_STDOUT}
+}
+
+run_pre_install_test()
+{
+    cp_print "Successfully finished pre-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT}
+    exit 0
+}
+
+run_post_install_test()
+{
+    if [ ! -d ${SERVICE_PATH} ]; then
+        cp_print "Failed post-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT}
+        exit 1
+    fi
+
+    cp_print "Successfully finished post-installation test for Check Point Central NGINX Manager service installation package [${INSTALLATION_TIME}]\n" ${FORCE_STDOUT}
+    exit 0
+}
+
+
+run()
+{
+    get_nginx_conf_path
+    if [ '--install' = "${1}" ]; then
+        run_installation "${@}"
+    elif [ '--uninstall' = "${1}" ]; then
+        run_uninstall
+    elif [ '--pre_install_test' = "${1}" ]; then
+        run_pre_install_test
+    elif [ '--post_install_test' = "${1}" ]; then
+        run_post_install_test
+    else
+        usage
+        exit 1
+    fi
+}
+
+if [ "$(id -u)" != "0" ]; then
+    echo "Administrative privileges required for this Package (use su or sudo)"
+    exit 1
+fi
+
+shift
+run "${@}"
+
+exit 0

nodes/http_transaction_handler/package/k8s-log-file-handler.sh

@@ -7,5 +7,5 @@ while true; do
         sleep 5
         continue
     fi
-    tail -q -f /var/log/nano_agent/cp-nano-http-transaction-handler.log? >> /proc/1/fd/1
+    tail -q -f /var/log/nano_agent/cp-nano-http-transaction-handler.log* >> /proc/1/fd/1
 done

nodes/orchestration/package/cp-nano-package-list

@@ -28,4 +28,5 @@ sdwan_logger="sdwanLogger 2204 SD-WAN_Logger"
 cpview_metric_provider="cpviewMetricProvider 8282"
 hello_world="hello_world"
 crowdsec_aux="crowdsecAux 8081"
+central_nginx_manager="centralNginxManager 7555"
 # ## Please do not remove this comment - newline at end of file required.

unit_test.cmake

@@ -2,7 +2,7 @@ enable_testing()
 
 function(add_unit_test ut_name ut_sources use_libs)
     add_executable(${ut_name} ${ut_sources})
-    target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)
+    target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} version debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group)
 
     add_test(NAME ${ut_name}
         COMMAND ${ut_name}