First release of open-appsec source code

core/logging/CMakeLists.txt

@@ -0,0 +1,3 @@
+add_library(logging logging.cc log_generator.cc debug_stream.cc file_stream.cc fog_stream.cc syslog_stream.cc cef_stream.cc)
+
+add_subdirectory(logging_ut)

core/logging/cef_stream.cc

@@ -0,0 +1,81 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "logging_comp.h"
+#include "log_streams.h"
+
+using namespace std;
+using namespace cereal;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+CefStream::CefStream(const string &_ip_address, int _port)
+        :
+    i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
+    ip_address(_ip_address),
+    port(_port)
+{
+    connect();
+    if (!socket.ok()) {
+        dbgWarning(D_REPORT) << "Failed to connect to the CEF server";
+    }
+}
+
+CefStream::~CefStream()
+{
+    if (socket.ok()) {
+        i_socket->closeSocket(const_cast<int &>(*socket));
+        socket = genError("Closed socket");
+    }
+}
+
+void
+CefStream::sendLog(const Report &log)
+{
+    if (!socket.ok()) {
+        connect();
+        if (!socket.ok()) {
+            dbgWarning(D_REPORT) << "Failed to connect to the CEF server, log will not be sent.";
+            return;
+        }
+    }
+    dbgTrace(D_REPORT) << "Connected to socket.";
+    string cef_report = log.getCef();
+    vector<char> data(cef_report.begin(), cef_report.end());
+    for (size_t tries = 0; tries < 3; tries++) {
+        if (i_socket->writeData(socket.unpack(), data)) {
+            dbgTrace(D_REPORT) << "log was sent to CEF server";
+            return;
+        } else {
+            dbgWarning(D_REPORT) << "Failed to send log to CEF server";
+        }
+    }
+}
+
+void
+CefStream::connect()
+{
+    auto cef_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.cefServer.IP");
+    auto cef_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.cefServer.port");
+
+    if (cef_ip_address.empty()) {
+        dbgWarning(D_REPORT) << "Cannot connect to CEF server, IP is not configured.";
+        return;
+    }
+    socket = i_socket->genSocket(
+        I_Socket::SocketType::UDP,
+        false,
+        false,
+        cef_ip_address + ":" + to_string(cef_port)
+    );
+}

core/logging/debug_stream.cc

@@ -0,0 +1,34 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log_streams.h"
+
+#include <sstream>
+
+#include "debug.h"
+
+using namespace std;
+using namespace cereal;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+void
+DebugStream::sendLog(const Report &log)
+{
+    stringstream ss;
+    {
+        JSONOutputArchive ar(ss);
+        log.serialize(ar);
+    }
+    dbgInfo(D_REPORT) << ss.str();
+}

core/logging/file_stream.cc

@@ -0,0 +1,139 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log_streams.h"
+#include "debug.h"
+#include "config.h"
+#include "singleton.h"
+#include "logging_comp.h"
+#include "agent_core_utilities.h"
+
+using namespace std;
+using namespace cereal;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+string
+getLogFileName()
+{
+    string file_path = getConfigurationWithDefault<string>("", "Logging", "Log file name");
+    if (file_path != "" && file_path.front() != '/') {
+        file_path = getLogFilesPathConfig() + "/" + file_path;
+    }
+    
+    if (Singleton::exists<I_InstanceAwareness>()) {
+        file_path += Singleton::Consume<I_InstanceAwareness>::by<LoggingComp>()->getUniqueID("");
+    }
+
+    return file_path;
+}
+
+LogFileStream::LogFileStream() : log_file_name(getLogFileName())
+{
+    openLogFile();
+}
+
+LogFileStream::~LogFileStream()
+{
+    closeLogFile();
+}
+
+void
+LogFileStream::sendLog(const Report &log)
+{
+    string maybe_new_log_file_name = getLogFileName();
+    if(maybe_new_log_file_name == "") {
+        closeLogFile();
+        return;
+    }
+    if(maybe_new_log_file_name != log_file_name) {
+        closeLogFile();
+        openLogFile();
+    }
+
+    bool should_format_log = log.isEnreachmentActive(ReportIS::Enreachments::BEAUTIFY_OUTPUT);
+    string logs_separator = getProfileAgentSettingWithDefault<string>("", "agent.config.logFileLineSeparator");
+    logs_separator = getConfigurationWithDefault<string>(logs_separator, "Logging", "Log file line separator");
+
+    stringstream ss;
+    if (should_format_log) {
+        {
+            JSONOutputArchive ar(ss);
+            log.serialize(ar);
+        }
+        log_stream << ss.str() << logs_separator << endl;
+    } else {
+        {
+            JSONOutputArchive ar(ss, JSONOutputArchive::Options::NoIndent());
+            log.serialize(ar);
+        }
+        static const boost::regex reg("\\n");
+        log_stream << NGEN::Regex::regexReplace(__FILE__, __LINE__, ss.str(), reg, "") << logs_separator << endl;
+    }
+
+    if (!log_stream.good()) {
+        dbgWarning(D_REPORT) << "Failed to write log to file, will retry. File path: " << log_file_name;
+
+        if (!retryWritingLog(ss.str())) {
+            dbgWarning(D_REPORT) << "Failed to write log to file";
+            return;
+        }
+    }
+
+    dbgDebug(D_REPORT) << "Successfully wrote log to file";
+}
+
+void
+LogFileStream::openLogFile()
+{
+    log_file_name = getLogFileName();
+    if (log_file_name == "") {
+        dbgInfo(D_REPORT) << "Empty log file name, no log file will be written";
+        return;
+    }
+
+    log_stream.open(log_file_name, ofstream::app);
+    if (!log_stream.is_open()) {
+        dbgWarning(D_REPORT) << "Failed in opening log file. File path: " << log_file_name;
+        return;
+    }
+
+    dbgDebug(D_REPORT) << "Successfully opened log file at path: " << log_file_name;
+}
+
+void
+LogFileStream::closeLogFile()
+{
+    log_stream.close();
+    if (log_stream.is_open() || log_stream.failbit) {
+        dbgWarning(D_REPORT) << "Failed in closing log file. File path: " << log_file_name;
+        return;
+    }
+
+    dbgDebug(D_REPORT) << "Successfully closed log file at path: " << log_file_name;
+}
+
+bool
+LogFileStream::retryWritingLog(const string &log)
+{
+    uint32_t max_num_retries = getConfigurationWithDefault<uint>(3, "Logging", "Maximum number of write retries");
+    for (uint32_t num_retries = 0; num_retries < max_num_retries; num_retries++) {
+        closeLogFile();
+        openLogFile();
+
+        log_stream << log << endl;
+        if (log_stream.good()) return true;
+    }
+
+    return false;
+}

core/logging/fog_stream.cc

@@ -0,0 +1,64 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log_streams.h"
+#include "logging_comp.h"
+
+using namespace std;
+using namespace cereal;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+FogStream::FogStream()
+        :
+    i_msg(Singleton::Consume<I_Messaging>::by<LoggingComp>())
+{
+}
+
+FogStream::~FogStream()
+{
+}
+
+void
+FogStream::sendLog(const Report &log)
+{
+    auto fog_log_uri = getConfigurationWithDefault<string>("/api/v1/agents/events", "Logging", "Fog Log URI");
+
+    ScopedContext ctx;
+    ctx.registerValue<bool>("Obfuscate log field", true);
+
+    LogRest rest(log);
+    i_msg->sendObjectWithPersistence(rest, I_Messaging::Method::POST, fog_log_uri, "", true, MessageTypeTag::LOG);
+}
+
+void
+FogStream::sendLog(const LogBulkRest &logs, bool persistence_only)
+{
+    ScopedContext ctx;
+    ctx.registerValue<bool>("Obfuscate log field", true);
+
+    auto fog_log_uri = getConfigurationWithDefault<string>("/api/v1/agents/events/bulk", "Logging", "Fog Log URI");
+    if (!persistence_only) {
+        i_msg->sendObjectWithPersistence(logs, I_Messaging::Method::POST, fog_log_uri, "", true, MessageTypeTag::LOG);
+    } else {
+        i_msg->sendObjectWithPersistence(
+            logs,
+            I_Messaging::Method::POST,
+            fog_log_uri,
+            "",
+            false,
+            MessageTypeTag::LOG,
+            true
+        );
+    }
+}

core/logging/log_generator.cc

@@ -0,0 +1,110 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log_generator.h"
+
+using namespace std;
+
+extern const string unnamed_service;
+
+LogGen::~LogGen()
+{
+    Singleton::Consume<I_Logging>::by<LogGen>()->sendLog(log);
+}
+
+LogGen &
+LogGen::operator<<(const LogField &field)
+{
+    log << field;
+    return *this;
+}
+
+void
+LogGen::addToOrigin(const LogField &field)
+{
+    log.addToOrigin(field);
+}
+
+void
+LogGen::serialize(cereal::JSONOutputArchive &ar) const
+{
+    log.serialize(ar);
+}
+
+chrono::microseconds
+LogGen::getCurrentTime() const
+{
+    return Singleton::Consume<I_TimeGet>::by<LogGen>()->getWalltime();
+}
+
+ReportIS::AudienceTeam
+LogGen::getAudienceTeam() const
+{
+    if (Singleton::exists<I_Environment>()) {
+        auto team = Singleton::Consume<I_Environment>::by<LogGen>()->get<ReportIS::AudienceTeam>("Audience Team");
+        if (team.ok()) return *team;
+    }
+    return ReportIS::AudienceTeam::NONE;
+}
+
+void
+LogGen::loadBaseLogFields()
+{
+    size_t curr_index = Singleton::Consume<I_Logging>::by<LogGen>()->getCurrentLogId();
+    log.setIndex(curr_index);
+    log << LogField("logIndex", curr_index);
+
+    if (!Singleton::exists<I_Environment>()) return;
+    auto env = Singleton::Consume<I_Environment>::by<LogGen>();
+
+    for (auto &string_by_key : env->getAllStrings(EnvKeyAttr::LogSection::SOURCE)) {
+        log.addToOrigin(LogField(string_by_key.first, string_by_key.second));
+    }
+
+    for (auto &uint64_by_key : env->getAllUints(EnvKeyAttr::LogSection::SOURCE)) {
+        log.addToOrigin(LogField(uint64_by_key.first, uint64_by_key.second));
+    }
+
+    for (auto &bool_by_key : env->getAllBools(EnvKeyAttr::LogSection::SOURCE)) {
+        log.addToOrigin(LogField(bool_by_key.first, bool_by_key.second));
+    }
+
+    for (auto &string_by_key : env->getAllStrings(EnvKeyAttr::LogSection::DATA)) {
+        log << LogField(string_by_key.first, string_by_key.second);
+    }
+
+    for (auto &uint64_by_key : env->getAllUints(EnvKeyAttr::LogSection::DATA)) {
+        log << LogField(uint64_by_key.first, uint64_by_key.second);
+    }
+
+    for (auto &bool_by_key : env->getAllBools(EnvKeyAttr::LogSection::DATA)) {
+        log << LogField(bool_by_key.first, bool_by_key.second);
+    }
+
+    for (auto &string_by_key : env->getAllStrings(EnvKeyAttr::LogSection::SOURCEANDDATA)) {
+        log.addToOrigin(LogField(string_by_key.first, string_by_key.second));
+        log << LogField(string_by_key.first, string_by_key.second);
+    }
+
+    for (auto &uint64_by_key : env->getAllUints(EnvKeyAttr::LogSection::SOURCEANDDATA)) {
+        log.addToOrigin(LogField(uint64_by_key.first, uint64_by_key.second));
+        log << LogField(uint64_by_key.first, uint64_by_key.second);
+    }
+
+    for (auto &bool_by_key : env->getAllBools(EnvKeyAttr::LogSection::SOURCEANDDATA)) {
+        log.addToOrigin(LogField(bool_by_key.first, bool_by_key.second));
+        log << LogField(bool_by_key.first, bool_by_key.second);
+    }
+
+    log.getMarkers() = env->getAllStrings(EnvKeyAttr::LogSection::MARKER);
+}

core/logging/log_streams.h

@@ -0,0 +1,119 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __LOG_STREAMS_H__
+#define __LOG_STREAMS_H__
+
+#include <fstream>
+#include <set>
+
+#include "i_mainloop.h"
+#include "report/report_bulks.h"
+#include "report/log_rest.h"
+#include "logging_metric.h"
+#include "i_logging.h"
+#include "i_socket_is.h"
+
+USE_DEBUG_FLAG(D_REPORT);
+
+class Stream
+{
+public:
+    virtual ~Stream() {}
+    virtual void sendLog(const Report &log) = 0;
+    virtual void
+    sendLog(const LogBulkRest &logs, bool persistance_only)
+    {
+        if (persistance_only) {
+            dbgWarning(D_REPORT) << "Skipping logs due to persistance only setting";
+            return;
+        }
+        for (auto &log : logs) {
+            sendLog(log);
+        }
+    }
+};
+
+class DebugStream : public Stream
+{
+public:
+    void sendLog(const Report &log) override;
+};
+
+class LogFileStream : public Stream
+{
+public:
+    LogFileStream();
+    ~LogFileStream();
+
+    void sendLog(const Report &log) override;
+
+private:
+    void openLogFile();
+    void closeLogFile();
+    bool retryWritingLog(const std::string &log);
+
+    std::string     log_file_name;
+    std::ofstream   log_stream;
+};
+
+class FogStream : public Stream
+{
+public:
+    FogStream();
+    ~FogStream();
+
+    void sendLog(const Report &log) override;
+    void sendLog(const LogBulkRest &logs, bool persistance_only) override;
+
+private:
+    I_Messaging *i_msg = nullptr;
+};
+
+class SyslogStream : public Stream
+{
+public:
+    SyslogStream(const std::string &_ip_address, int _port);
+    ~SyslogStream();
+
+    void sendLog(const Report &log) override;
+
+private:
+    void connect();
+
+    I_Socket *i_socket = nullptr;
+    I_MainLoop *mainloop = nullptr;
+    std::string ip_address;
+    int port;
+    I_MainLoop::RoutineID log_send_routine = -1;
+    Maybe<I_Socket::socketFd> socket = genError("Not set yet");
+};
+
+class CefStream : public Stream
+{
+public:
+    CefStream(const std::string &_ip_address, int _port);
+    ~CefStream();
+
+    void sendLog(const Report &log) override;
+
+private:
+    void connect();
+
+    I_Socket *i_socket = nullptr;
+    std::string ip_address;
+    int port;
+    Maybe<I_Socket::socketFd> socket = genError("Not set yet");
+};
+
+#endif // __LOG_STREAMS_H__

core/logging/logging.cc

@@ -0,0 +1,339 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "logging_comp.h"
+
+#include <map>
+#include <sstream>
+#include <fstream>
+
+#include "log_streams.h"
+#include "common.h"
+#include "singleton.h"
+#include "debug.h"
+#include "rest.h"
+#include "config.h"
+#include "i_mainloop.h"
+#include "report/report_bulks.h"
+#include "report/log_rest.h"
+#include "instance_awareness.h"
+#include "logging_metric.h"
+#include "tag_and_enum_management.h"
+
+using namespace std;
+using namespace cereal;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+class LoggingComp::Impl
+        :
+    Singleton::Provide<I_Logging>::From<LoggingComp>
+{
+public:
+    using StreamType = ReportIS::StreamType;
+
+    void
+    init()
+    {
+        streams = streams_preperation;
+        i_mainloop = Singleton::Consume<I_MainLoop>::by<LoggingComp>();
+
+        auto bulk_msec_interval = getConfigurationWithDefault<uint>(
+            2000,
+            "Logging",
+            "Log bulk sending interval in msec"
+        );
+        log_send_routine = i_mainloop->addRecurringRoutine(
+            I_MainLoop::RoutineType::Offline,
+            chrono::milliseconds(bulk_msec_interval),
+            [this] () { sendBufferedLogs(); },
+            "Logging Fog stream messaging"
+        );
+
+        auto metrics_interval = getConfigurationWithDefault<uint64_t>(600, "Logging", "Metrics Routine Interval");
+        log_metric.init(
+            "Logging data",
+            ReportIS::AudienceTeam::AGENT_CORE,
+            ReportIS::IssuingEngine::AGENT_CORE,
+            chrono::seconds(metrics_interval),
+            false
+        );
+        log_metric.registerListener();
+    }
+
+    void
+    fini()
+    {
+        streams.clear();
+        if (i_mainloop != nullptr && i_mainloop->doesRoutineExist(log_send_routine)) {
+            i_mainloop->stop(log_send_routine);
+        }
+    }
+
+    void
+    preload()
+    {
+        registerConfigPrepareCb([&] () { streams_preperation.clear(); });
+        registerConfigLoadCb([&] () {
+            streams.clear();
+            selectStreams();
+            streams = streams_preperation;
+        });
+        registerConfigAbortCb([&] () {
+            streams_preperation.clear();
+        });
+    }
+
+    bool
+    addStream(StreamType type) override
+    {
+        if (streams_preperation.find(type) != streams_preperation.end()) {
+            dbgWarning(D_REPORT)
+                << "Cannot add second instance of the same stream. Stream type: "
+                << TagAndEnumManagement::convertToString(type);
+            return false;
+        }
+        streams_preperation[type] = makeStream(type);
+        dbgInfo(D_REPORT)
+            << "Successfully added log stream. Stream type: "
+            << TagAndEnumManagement::convertToString(type);
+        return true;
+    }
+
+    bool
+    addStream(ReportIS::StreamType type, const string &log_server_url) override
+    {
+        string log_type = TagAndEnumManagement::convertToString(type);
+        if (streams_preperation.find(type) != streams_preperation.end()) {
+            dbgWarning(D_REPORT)
+                << "Cannot add second instance of the same stream. Stream type: "
+                << log_type;
+            return false;
+        }
+        try {
+            string ip = log_server_url.substr(0, log_server_url.find(':'));
+            string port = log_server_url.substr(log_server_url.find(':') + 1, log_server_url.length());
+            int port_num = stoi(port);
+
+            streams_preperation[type] = makeStream(type, ip, port_num);
+            dbgInfo(D_REPORT)
+                << "Successfully added log stream. Stream type: "
+                << log_type
+                << " url: "
+                << ip
+                << ":"
+                << port;
+        } catch (const exception &e) {
+            dbgWarning(D_REPORT) << "Error in stream configure: " << e.what();
+            return false;
+        }
+        return true;
+    }
+
+    bool
+    delStream(StreamType type) override
+    {
+        if (streams.find(type) == streams.end()) {
+            dbgWarning(D_REPORT)
+                << "Cannot delete stream. Error: Stream does not exist, Stream type: "
+                << TagAndEnumManagement::convertToString(type);
+            return false;
+        }
+        streams.erase(type);
+        return true;
+    }
+
+    void
+    sendLog(const Report &log) override
+    {
+        if (getConf("agent.config.log.useBulkMode", "Enable bulk of logs", true)) {
+            reports.setBulkSize(getConfigurationWithDefault<uint>(100, "Logging", "Sent log bulk size"));
+            reports.push(log);
+            if (reports.sizeQueue() >= 4) {
+                auto persistence_only = getConf("agent.config.log.skip.enable", "Enable Log skipping", true);
+                sendBufferedLogsImpl(false, persistence_only);
+            }
+        } else {
+            LogEventLogsSent(true).notify();
+            for (auto &iter : streams) {
+                if (log.isStreamActive(iter.first)) iter.second->sendLog(log);
+            }
+        }
+    }
+
+    uint64_t
+    getCurrentLogId() override
+    {
+        ++log_id;
+        return log_id;
+    }
+
+    void addGeneralModifier(const GeneralModifier &modifier) override { modifiers.push_back(modifier); }
+
+    pair<bool, string>
+    getLoggingModeConfig()
+    {
+        bool is_bulk_enabled = getConfigurationWithDefault<bool>(
+            true,
+            "Logging",
+            "Enable bulk of logs"
+        );
+        is_bulk_enabled = getProfileAgentSettingWithDefault<bool>(
+            is_bulk_enabled,
+            "agent.config.log.useBulkMode"
+        );
+        static const string default_fog_uri = "/api/v1/agents/events";
+        string default_fog_uri_to_use = default_fog_uri;
+        if (is_bulk_enabled) default_fog_uri_to_use.append("/bulk");
+        string fog_to_use = getConfigurationWithDefault<string>(default_fog_uri_to_use, "Logging", "Fog Log URI");
+        return {is_bulk_enabled, fog_to_use};
+    }
+
+private:
+    void
+    sendBufferedLogs()
+    {
+        while (!reports.empty()) {
+            sendBufferedLogsImpl(true, false);
+        }
+    }
+
+    void
+    sendBufferedLogsImpl(bool is_async, bool persistence_only)
+    {
+        LogEventQueueSize(reports.size()).notify();
+        auto batch = reports.pop();
+        LogEventLogsSent(false, batch.size()).notify();
+
+        for (auto &modifier : modifiers) {
+            modifier(batch);
+        }
+
+        // Copy in order to avoid invalidation during sending of logs
+        auto local_streams = streams;
+        for (auto &iter : local_streams) {
+            LogBulkRest sub_batch;
+            for (const auto &log : batch) {
+                if (log.isStreamActive(iter.first)) sub_batch.push(log);
+            }
+
+            if (sub_batch.size()) {
+                iter.second->sendLog(sub_batch, persistence_only);
+                if (is_async) i_mainloop->yield();
+            }
+        }
+    }
+
+    bool
+    getConf(const string &general_setings, const string &configuration, bool default_value)
+    {
+        bool setting_value = getProfileAgentSettingWithDefault<bool>(default_value, general_setings);
+        return getConfigurationWithDefault<bool>(setting_value, "Logging", configuration);
+    }
+
+    void
+    selectStreams()
+    {
+        if (getConfiguration<string>("Logging", "Log file name").ok()) {
+            addStream(StreamType::JSON_LOG_FILE);
+        } else {
+            addStream(StreamType::JSON_DEBUG);
+        }
+
+        auto agent_mode = Singleton::Consume<I_AgentDetails>::by<LoggingComp>()->getOrchestrationMode();
+        if (agent_mode == OrchestrationMode::OFFLINE || agent_mode == OrchestrationMode::HYBRID) {
+            dbgInfo(D_REPORT) << "Agent not in online mode, fog stream is no supported";
+        } else {
+            addStream(StreamType::JSON_FOG);
+        }
+    }
+
+    shared_ptr<Stream>
+    makeStream(StreamType type)
+    {
+        switch (type) {
+            case StreamType::JSON_DEBUG: return make_shared<DebugStream>();
+            case StreamType::JSON_FOG: return make_shared<FogStream>();
+            case StreamType::JSON_LOG_FILE: return make_shared<LogFileStream>();
+            case StreamType::SYSLOG: return nullptr;
+            case StreamType::CEF: return nullptr;
+            case StreamType::NONE: return nullptr;
+            case StreamType::COUNT: return nullptr;
+        }
+        dbgError(D_REPORT) << "Unknown log stream type";
+        return nullptr;
+    }
+
+    shared_ptr<Stream>
+    makeStream(StreamType type, const string &ip, int port)
+    {
+        switch (type) {
+            case StreamType::SYSLOG:
+                return make_shared<SyslogStream>(ip, port);
+            case StreamType::CEF: return make_shared<CefStream>(ip, port);
+            default:
+                dbgWarning(D_REPORT) << "Invalid stream type with url";
+                return NULL;
+        }
+        dbgError(D_REPORT) << "Unknown log stream type";
+        return nullptr;
+    }
+
+    uint64_t log_id = 0;
+    map<StreamType, shared_ptr<Stream>> streams;
+    map<StreamType, shared_ptr<Stream>> streams_preperation;
+    I_MainLoop *i_mainloop;
+    ReportsBulk reports;
+    I_MainLoop::RoutineID log_send_routine = 0;
+    LogMetric log_metric;
+    vector<GeneralModifier> modifiers;
+};
+
+LoggingComp::LoggingComp() : Component("LoggingComp"), pimpl(make_unique<Impl>()) {}
+
+LoggingComp::~LoggingComp() {}
+
+void
+LoggingComp::preload()
+{
+    registerExpectedConfiguration<bool>("Logging", "Enable event buffer");
+    registerExpectedConfiguration<bool>("Logging", "Enable bulk of logs");
+    registerExpectedConfiguration<bool>("Logging", "Enable Syslog");
+    registerExpectedConfiguration<bool>("Logging", "Enable CEF");
+    registerExpectedConfiguration<bool>("Logging", "Enable Log skipping");
+    registerExpectedConfiguration<string>("Logging", "Log file name");
+    registerExpectedConfiguration<string>("Logging", "Log file line seperator");
+    registerExpectedConfiguration<string>("Logging", "Fog Log URI");
+    registerExpectedConfiguration<string>("Logging", "Syslog IP");
+    registerExpectedConfiguration<uint>("Logging", "Syslog port");
+    registerExpectedConfiguration<string>("Logging", "CEF IP");
+    registerExpectedConfiguration<uint>("Logging", "CEF port");
+    registerExpectedConfiguration<uint>("Logging", "Log bulk sending interval in msec");
+    registerExpectedConfiguration<uint>("Logging", "Sent log bulk size");
+    registerExpectedConfiguration<uint>("Logging", "Maximum number of write retries");
+    registerExpectedConfiguration<uint>("Logging", "Metrics Routine Interval");
+
+    pimpl->preload();
+}
+
+void
+LoggingComp::init()
+{
+    pimpl->init();
+}
+
+void
+LoggingComp::fini()
+{
+    pimpl->fini();
+}

core/logging/logging_metric.h

@@ -0,0 +1,73 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __LOGGING_METRIC_H__
+#define __LOGGING_METRIC_H__
+
+#include "generic_metric.h"
+
+class LogEventQueueSize : public Event<LogEventQueueSize>
+{
+public:
+    LogEventQueueSize(uint64_t _size) : size(_size) {}
+
+    uint64_t getSize() const { return size; }
+
+private:
+    uint64_t size;
+};
+
+class LogEventLogsSent : public Event<LogEventLogsSent>
+{
+public:
+    LogEventLogsSent(bool is_single, uint64_t no_logs = 1) : logs(no_logs),  bulks(is_single ? 0 : 1)  {}
+
+    uint64_t getLogsNumber() const { return logs; }
+    uint64_t getBulksNumber() const { return bulks; }
+
+private:
+    uint64_t logs;
+    uint64_t bulks;
+};
+
+class LogMetric
+        :
+    public GenericMetric,
+    public Listener<LogEventQueueSize>,
+    public Listener<LogEventLogsSent>
+{
+public:
+    void
+    upon(const LogEventQueueSize &event) override
+    {
+        max_queue_size.report(event.getSize());
+        avg_queue_size.report(double(event.getSize()));
+        current_queue_size.report(event.getSize());
+    }
+
+    void
+    upon(const LogEventLogsSent &event) override
+    {
+        sent_logs.report(event.getLogsNumber());
+        sent_logs_bulks.report(event.getBulksNumber());
+    }
+
+private:
+    MetricCalculations::Max<uint64_t> max_queue_size{this, "logQueueMaxSizeSample", 0};
+    MetricCalculations::Average<double> avg_queue_size{this, "logQueueAvgSizeSample"};
+    MetricCalculations::LastReportedValue<uint64_t> current_queue_size{this, "logQueueCurrentSizeSample"};
+    MetricCalculations::Counter sent_logs{this, "sentLogsSum"};
+    MetricCalculations::Counter sent_logs_bulks{this, "sentLogsBulksSum"};
+};
+
+#endif // __LOGGING_METRIC_H__

core/logging/logging_ut/CMakeLists.txt

@@ -0,0 +1,7 @@
+link_directories(${BOOST_ROOT}/lib)
+
+add_unit_test(
+    logging_ut
+    "logging_ut.cc"
+    "logging;singleton;connkey;rest;report;agent_details;event_is;metric;version;-lboost_regex;"
+)

core/logging/syslog_stream.cc

@@ -0,0 +1,92 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "log_streams.h"
+#include "logging_comp.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_REPORT);
+
+SyslogStream::SyslogStream(const string &_ip_address, int _port)
+        :
+    i_socket(Singleton::Consume<I_Socket>::by<LoggingComp>()),
+    mainloop(Singleton::Consume<I_MainLoop>::by<LoggingComp>()),
+    ip_address(_ip_address),
+    port(_port)
+{
+    connect();
+    if (!socket.ok()) {
+        dbgWarning(D_REPORT) << "Failed to connect to the syslog server";
+    }
+}
+
+SyslogStream::~SyslogStream()
+{
+    if (mainloop != nullptr && mainloop->doesRoutineExist(log_send_routine)) mainloop->stop(log_send_routine);
+    if (socket.ok()) {
+        i_socket->closeSocket(const_cast<int &>(*socket));
+        socket = genError("Closed socket");
+    }
+}
+
+void
+SyslogStream::sendLog(const Report &log)
+{
+    if (!socket.ok()) {
+        connect();
+        if (!socket.ok()) {
+            dbgWarning(D_REPORT) << "Failed to connect to the syslog server, Log will not be sent.";
+            return;
+        }
+        dbgTrace(D_REPORT) << "Successfully connect to the syslog server";
+    }
+
+    string syslog_report = log.getSyslog();
+    vector<char> data(syslog_report.begin(), syslog_report.end());
+    mainloop->addOneTimeRoutine(
+        I_MainLoop::RoutineType::Offline,
+        [this, data] ()
+        {
+            int tries = 1;
+            for (; tries <=3; tries++) {
+                if (i_socket->writeData(socket.unpack(), data)) {
+                    dbgTrace(D_REPORT) << "log was sent to syslog server";
+                    return;
+                } else {
+                    dbgWarning(D_REPORT) << "Failed to send log to syslog server";
+                }
+            }
+        },
+        "Logging Syslog stream messaging"
+    );
+}
+
+void
+SyslogStream::connect()
+{
+    auto syslog_ip_address = getProfileAgentSettingWithDefault<string>(ip_address, "agent.config.log.syslogServer.IP");
+    auto syslog_port = getProfileAgentSettingWithDefault<uint>(port, "agent.config.log.syslogServer.port");
+
+    if (syslog_ip_address.empty()) {
+        dbgWarning(D_REPORT) << "Cannot connect to Syslog server, IP is not configured.";
+        return;
+    }
+
+    socket = i_socket->genSocket(
+        I_Socket::SocketType::UDP,
+        false,
+        false,
+        syslog_ip_address + ":" + to_string(syslog_port)
+    );
+}