Merge pull request #153 from openappsec/Jun_26_2024-Dev

June 27th update

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 2.8.4)
 project (ngen)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -Wno-terminate")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC -Wall -Wno-terminate")
 
 execute_process(COMMAND grep -c "Alpine Linux" /etc/os-release OUTPUT_VARIABLE IS_ALPINE)
 if(NOT IS_ALPINE EQUAL "0")

build_system/apisix/docker-compose.yaml

@@ -1,15 +1,15 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
 
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 version: "3"
 

build_system/docker/entry.sh

@@ -11,6 +11,7 @@ var_fog_address=
 var_proxy=
 var_mode=
 var_token=
+var_ignore=
 init=
 
 if [ ! -f /nano-service-installers/$ORCHESTRATION_INSTALLATION_SCRIPT ]; then
@@ -33,6 +34,8 @@ while true; do
         var_proxy="$1"
     elif [ "$1" == "--hybrid-mode" ] || [ "$1" == "--standalone" ]; then
         var_mode="--hybrid_mode"
+    elif [ "$1" == "--no-upgrade" ]; then
+        var_ignore="--ignore all"
     elif [ "$1" == "--token" ]; then
         shift
         var_token="$1"
@@ -47,6 +50,7 @@ fi
 
 orchestration_service_installation_flags="--container_mode --skip_registration"
 if [ ! -z $var_token ]; then
+    export AGENT_TOKEN="$var_token"
     orchestration_service_installation_flags="$orchestration_service_installation_flags --token $var_token"
 fi
 if [ ! -z $var_fog_address ]; then
@@ -59,6 +63,9 @@ fi
 if [ ! -z $var_mode ]; then
     orchestration_service_installation_flags="$orchestration_service_installation_flags $var_mode"
 fi
+if [ ! -z "$var_ignore" ]; then
+    orchestration_service_installation_flags="$orchestration_service_installation_flags $var_ignore"
+fi
 
 
 /nano-service-installers/$ORCHESTRATION_INSTALLATION_SCRIPT --install $orchestration_service_installation_flags

components/CMakeLists.txt

@@ -4,7 +4,6 @@ add_subdirectory(signal_handler)
 add_subdirectory(gradual_deployment)
 add_subdirectory(packet)
 add_subdirectory(pending_key)
-add_subdirectory(health_check_manager)
 
 add_subdirectory(utils)
 add_subdirectory(attachment-intakers)

components/attachment-intakers/nginx_attachment/nginx_attachment.cc

@@ -1136,8 +1136,7 @@ private:
             NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
             uuid = opaque.getSessionUUID();
         }
-        web_response_data.uuid_size =
-            string("Incident Id: ").length() + uuid.size();
+        web_response_data.uuid_size = uuid.size();
 
         if (web_trigger_conf.getDetailsLevel() == "Redirect") {
             web_response_data.response_data.redirect_data.redirect_location_size =
@@ -1699,7 +1698,7 @@ private:
                 }
             };
         mainloop->addFileRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             server_sock,
             accept_attachment_routine,
             "Nginx Attachment registration listener",

components/attachment-intakers/nginx_attachment/user_identifiers_config.cc

@@ -306,17 +306,20 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
 void
 UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, ExtractType type) const
 {
+    auto i_transaction_table = Singleton::Consume<I_TableSpecific<SessionID>>::by<NginxAttachment>();
+    if (!i_transaction_table || !i_transaction_table->hasState<NginxAttachmentOpaque>()) {
+        dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Can't get the transaction table";
+        return;
+    }
+    NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
+    opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
+    dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "xff found, value from header: " << static_cast<string>(header.getValue());
     auto value = parseXForwardedFor(header.getValue());
     if (!value.ok()) {
         dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
         return;
     };
-    auto i_transaction_table = Singleton::Consume<I_TableSpecific<SessionID>>::by<NginxAttachment>();
-    if (!i_transaction_table || !i_transaction_table->hasState<NginxAttachmentOpaque>()) {
-        dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Can't get the transaction table";
-        return;
-    }
-    NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
+
     if (type == ExtractType::SOURCEIDENTIFIER) {
         opaque.setSourceIdentifier(header.getKey(), value.unpack());
         dbgDebug(D_NGINX_ATTACHMENT_PARSER)

components/health_check_manager/health_check_manager_ut/CMakeLists.txt

@@ -1,8 +0,0 @@
-include_directories(${CMAKE_SOURCE_DIR}/components/include)
-link_directories(${BOOST_ROOT}/lib)
-
-add_unit_test(
-    health_check_manager_ut
-    "health_check_manager_ut.cc"
-    "singleton;messaging;mainloop;health_check_manager;event_is;metric;-lboost_regex"
-)

components/include/http_geo_filter.h

@@ -15,7 +15,8 @@ class HttpGeoFilter
     public Component,
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_GeoLocation>,
-    Singleton::Consume<I_GenericRulebase>
+    Singleton::Consume<I_GenericRulebase>,
+    Singleton::Consume<I_Environment>
 {
 public:
     HttpGeoFilter();

components/include/http_transaction_data.h

@@ -136,6 +136,7 @@ public:
     static const std::string req_body;
     static const std::string source_identifier;
     static const std::string proxy_ip_ctx;
+    static const std::string xff_vals_ctx;
 
     static const CompressionType default_response_content_encoding;
 

components/include/i_details_resolver.h

@@ -29,7 +29,9 @@ public:
     virtual bool isGwNotVsx() = 0;
     virtual bool isVersionAboveR8110() = 0;
     virtual bool isReverseProxy() = 0;
+    virtual bool isCloudStorageEnabled() = 0;
     virtual Maybe<std::tuple<std::string, std::string, std::string>> parseNginxMetadata() = 0;
+    virtual Maybe<std::tuple<std::string, std::string, std::string, std::string, std::string>> readCloudMetadata() = 0;
     virtual std::map<std::string, std::string> getResolvedDetails() = 0;
 #if defined(gaia) || defined(smb)
     virtual bool compareCheckpointVersion(int cp_version, std::function<bool(int, int)> compare_operator) const = 0;

components/include/i_downloader.h

@@ -22,7 +22,7 @@
 class I_Downloader
 {
 public:
-    virtual Maybe<std::string> downloadFileFromFog(
+    virtual Maybe<std::string> downloadFile(
         const std::string &checksum,
         Package::ChecksumTypes,
         const GetResourceFile &resourse_file

components/include/i_service_controller.h

@@ -64,7 +64,7 @@ public:
         const std::string &service_id
     ) = 0;
 
-    virtual std::map<std::string, PortNumber> getServiceToPortMap() = 0;
+    virtual std::map<std::string, std::vector<PortNumber>> getServiceToPortMap() = 0;
 
 protected:
     virtual ~I_ServiceController() {}

components/include/i_update_communication.h

@@ -32,6 +32,7 @@ public:
         const std::string &policy_versions
     ) const = 0;
     virtual Maybe<void> authenticateAgent() = 0;
+    virtual void registerLocalAgentToFog() = 0;
     virtual Maybe<void> getUpdate(CheckUpdateRequest &request) = 0;
     virtual Maybe<std::string> downloadAttributeFile(
         const GetResourceFile &resourse_file,

components/include/orchestration_comp.h

@@ -31,6 +31,7 @@
 #include "i_environment.h"
 #include "i_tenant_manager.h"
 #include "i_package_handler.h"
+#include "i_proxy_configuration.h"
 #include "i_env_details.h"
 #include "component.h"
 
@@ -54,7 +55,8 @@ class OrchestrationComp
     Singleton::Consume<I_UpdateCommunication>,
     Singleton::Consume<I_Downloader>,
     Singleton::Consume<I_ManifestController>,
-    Singleton::Consume<I_EnvDetails>
+    Singleton::Consume<I_EnvDetails>,
+    Singleton::Consume<I_ProxyConfiguration>
 {
 public:
     OrchestrationComp();

components/include/report_messaging.h

@@ -36,7 +36,6 @@ public:
             title,
             audience_team,
             obj,
-            false,
             MessageCategory::GENERIC,
             std::forward<Args>(args)...
         )
@@ -48,26 +47,6 @@ public:
         const std::string &title,
         const ReportIS::AudienceTeam &audience_team,
         const T &obj,
-        bool is_async_message,
-        Args ...args)
-            :
-        ReportMessaging(
-            title,
-            audience_team,
-            obj,
-            is_async_message,
-            MessageCategory::GENERIC,
-            std::forward<Args>(args)...
-        )
-    {
-    }
-
-    template <typename ...Args, typename T>
-    ReportMessaging(
-        const std::string &title,
-        const ReportIS::AudienceTeam &audience_team,
-        const T &obj,
-        bool is_async_message,
         const MessageCategory &message_type,
         Args ...args)
             :
@@ -77,7 +56,6 @@ public:
             ReportIS::Severity::INFO,
             ReportIS::Priority::LOW,
             obj,
-            is_async_message,
             message_type,
             std::forward<Args>(args)...
         )
@@ -99,7 +77,6 @@ public:
             severity,
             priority,
             obj,
-            false,
             MessageCategory::GENERIC,
             std::forward<Args>(args)...
         )
@@ -114,7 +91,6 @@ public:
         const ReportIS::Severity &severity,
         const ReportIS::Priority &priority,
         const T &obj,
-        bool _is_async_message,
         const MessageCategory &message_type,
         Args ...args)
             :
@@ -131,7 +107,6 @@ public:
             std::chrono::seconds(0),
             std::forward<Args>(args)...
         ),
-        is_async_message(_is_async_message),
         message_type_tag(message_type)
     {
         report << LogField("eventObject", obj);
@@ -141,11 +116,13 @@ public:
 
     ReportMessaging & operator<<(const LogField &field);
 
+    Maybe<void, HTTPResponse> sendReportSynchronously();
+
     void setForceBuffering(bool _force_buffering);
 
 private:
     Report report;
-    bool is_async_message;
+    bool is_async_message = true;
     bool force_buffering = false;
     MessageCategory message_type_tag;
 };

components/report_messaging/report_messaging.cc

@@ -24,6 +24,7 @@ static const string url = "/api/v1/agents/events";
 ReportMessaging::~ReportMessaging()
 {
     if (!Singleton::exists<I_Messaging>()) return;
+    if (!is_async_message) return;
 
     LogRest log_rest(report);
 
@@ -47,6 +48,25 @@ ReportMessaging::operator<<(const LogField &field)
     return *this;
 }
 
+class LogRestWithReply : public LogRest
+{
+public:
+    LogRestWithReply(const Report &report) : LogRest(report) {}
+
+    bool loadJson(const string &) const { return true; }
+};
+
+Maybe<void, HTTPResponse>
+ReportMessaging::sendReportSynchronously()
+{
+    is_async_message = false;
+
+    LogRestWithReply log_rest(report);
+
+    auto messaging = Singleton::Consume<I_Messaging>::by<ReportMessaging>();
+    return messaging->sendSyncMessage(HTTPMethod::POST, url, log_rest, message_type_tag);
+}
+
 void
 ReportMessaging::setForceBuffering(bool _force_buffering)
 {

components/report_messaging/report_messaging_ut/report_messaging_ut.cc

@@ -103,7 +103,48 @@ TEST_F(ReportMessagingTest, title_only)
             _
         )
     ).Times(1);
-    ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL);
+    ReportMessaging("test", ReportIS::AudienceTeam::AGENT_CORE, 1, ReportIS::Tags::ACCESS_CONTROL);
+}
+
+TEST_F(ReportMessagingTest, sync_sending)
+{
+    EXPECT_CALL(
+        mock_messaging,
+        sendSyncMessage(
+            _,
+            _,
+            "{\n"
+            "    \"log\": {\n"
+            "        \"eventTime\": \"Best Time ever\",\n"
+            "        \"eventName\": \"test\",\n"
+            "        \"eventSeverity\": \"Info\",\n"
+            "        \"eventPriority\": \"Low\",\n"
+            "        \"eventType\": \"Event Driven\",\n"
+            "        \"eventLevel\": \"Log\",\n"
+            "        \"eventLogLevel\": \"info\",\n"
+            "        \"eventAudience\": \"Internal\",\n"
+            "        \"eventAudienceTeam\": \"Agent Core\",\n"
+            "        \"eventFrequency\": 0,\n"
+            "        \"eventTags\": [\n"
+            "            \"Access Control\"\n"
+            "        ],\n"
+            "        \"eventSource\": {\n"
+            "            \"eventTraceId\": \"\",\n"
+            "            \"eventSpanId\": \"\",\n"
+            "            \"issuingEngineVersion\": \"\",\n"
+            "            \"serviceName\": \"Unnamed Nano Service\"\n"
+            "        },\n"
+            "        \"eventData\": {\n"
+            "            \"eventObject\": 1\n"
+            "        }\n"
+            "    }\n"
+            "}",
+            _,
+            _
+        )
+    ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, "response!!")));
+    ReportMessaging report("test", ReportIS::AudienceTeam::AGENT_CORE, 1, ReportIS::Tags::ACCESS_CONTROL);
+    EXPECT_TRUE(report.sendReportSynchronously().ok());
 }
 
 TEST_F(ReportMessagingTest, with_buffering)
@@ -144,7 +185,7 @@ TEST_F(ReportMessagingTest, with_buffering)
             true
         )
     ).Times(1);
-    ReportMessaging report("test", ReportIS::AudienceTeam::AGENT_CORE, 1, true, ReportIS::Tags::ACCESS_CONTROL);
+    ReportMessaging report("test", ReportIS::AudienceTeam::AGENT_CORE, 1, ReportIS::Tags::ACCESS_CONTROL);
     report.setForceBuffering(true);
 }
 

components/security_apps/http_geo_filter/CMakeLists.txt

@@ -1 +1,5 @@
+include_directories(../waap/include)
+include_directories(../waap/waap_clib)
+include_directories(../../attachment-intakers/nginx_attachment)
+
 add_library(http_geo_filter http_geo_filter.cc)

components/security_apps/http_geo_filter/http_geo_filter.cc

@@ -4,10 +4,16 @@
 #include <unistd.h>
 #include <stddef.h>
 #include <algorithm>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <boost/algorithm/string.hpp>
 
+#include "cidrs_data.h"
 #include "generic_rulebase/generic_rulebase.h"
 #include "generic_rulebase/parameters_config.h"
 #include "generic_rulebase/triggers_config.h"
+#include "user_identifiers_config.h"
 #include "debug.h"
 #include "config.h"
 #include "rest.h"
@@ -21,9 +27,10 @@ USE_DEBUG_FLAG(D_GEO_FILTER);
 
 static const LogTriggerConf default_triger;
 
-class HttpGeoFilter::Impl : public Listener<NewHttpTransactionEvent>
+class HttpGeoFilter::Impl : public Listener<HttpRequestHeaderEvent>
 {
 public:
+
     void
     init()
     {
@@ -55,32 +62,42 @@ public:
     }
 
     EventVerdict
-    respond(const NewHttpTransactionEvent &event) override
+    respond(const HttpRequestHeaderEvent &event) override
     {
         dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event";
 
-        if (!ParameterException::isGeoLocationExceptionExists() &&
-            !getConfiguration<GeoConfig>("rulebase", "httpGeoFilter").ok()
-        ) {
-            dbgTrace(D_GEO_FILTER) << "No geo location practice nor exception was found. Returning default verdict";
+        if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
+        std::set<std::string> xff_set;
+        auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
+        auto maybe_xff = env->get<std::string>(HttpTransactionData::xff_vals_ctx);
+        if (!maybe_xff.ok()) {
+            dbgTrace(D_GEO_FILTER) << "failed to get xff vals from env";
+        } else {
+            xff_set = split(maybe_xff.unpack(), ',');
+        }
+        dbgDebug(D_GEO_FILTER) << getListenerName() << " last header, start lookup";
+
+        if (xff_set.size() > 0) {
+            removeTrustedIpsFromXff(xff_set);
+        } else {
+            dbgDebug(D_GEO_FILTER) << "xff not found in headers";
+        }
+
+        auto maybe_source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+        if (!maybe_source_ip.ok()) {
+            dbgWarning(D_GEO_FILTER) << "failed to get source ip from env";
             return EventVerdict(default_action);
         }
 
-        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
-        auto asset_location = i_geo_location->lookupLocation(event.getSourceIP());
-        if (!asset_location.ok()) {
-            dbgTrace(D_GEO_FILTER) << "Lookup location failed, Error: " << asset_location.getErr();
-            return EventVerdict(default_action);
-        }
+        auto source_ip = convertIpAddrToString(maybe_source_ip.unpack());
+        xff_set.insert(source_ip);
 
-        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data = asset_location.unpack();
-
-        ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(event, geo_location_data);
+        ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(xff_set);
         if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
             return EventVerdict(exception_verdict);
         }
 
-        ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(event, geo_location_data);
+        ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(xff_set);
         if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
             return EventVerdict(geo_lookup_verdict);
         }
@@ -88,6 +105,73 @@ public:
     }
 
 private:
+    std::set<std::string>
+    split(const std::string& s, char delim) {
+        std::set<std::string> elems;
+        std::stringstream ss(s);
+        std::string value;
+        while (std::getline(ss, value, delim)) {
+            elems.insert(trim(value));
+        }
+        return elems;
+    }
+
+    static inline std::string &ltrim(std::string &s) {
+        s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+            [] (char c) { return !std::isspace(c); }));
+        return s;
+    }
+
+    // trim from end
+    static inline std::string &rtrim(std::string &s) {
+        s.erase(std::find_if(s.rbegin(), s.rend(),
+            [] (char c) { return !std::isspace(c); }).base(), s.end());
+        return s;
+    }
+
+    // trim from both ends
+    static inline std::string &trim(std::string &s) {
+        return ltrim(rtrim(s));
+    }
+
+    void
+    removeTrustedIpsFromXff(std::set<std::string> &xff_set)
+    {
+        auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
+            "rulebase",
+            "usersIdentifiers"
+        );
+        if (!identify_config.ok()) {
+            dbgDebug(D_GEO_FILTER) << "did not find users identifiers definition in policy";
+        } else {
+            auto trusted_ips = (*identify_config).getHeaderValuesFromConfig("x-forwarded-for");
+            for (auto it = xff_set.begin(); it != xff_set.end();) {
+                if (isIpTrusted(*it, trusted_ips)) {
+                    dbgTrace(D_GEO_FILTER) << "xff value is in trusted ips: " << *it;
+                    it = xff_set.erase(it);
+                } else {
+                    dbgTrace(D_GEO_FILTER) << "xff value is not in trusted ips: " << *it;
+                    ++it;
+                }
+            }
+        }
+    }
+
+    bool
+    isIpTrusted(const string &ip, const vector<string> &trusted_ips)
+    {
+        for (const auto &trusted_ip : trusted_ips) {
+            CIDRSData cidr_data(trusted_ip);
+            if (
+                ip == trusted_ip ||
+                (cidr_data.contains(ip))
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     string
     convertIpAddrToString(const IPAddr &ip_to_convert)
     {
@@ -117,54 +201,75 @@ private:
     }
 
     ngx_http_cp_verdict_e
-    getGeoLookupVerdict(
-        const NewHttpTransactionEvent &event,
-        const EnumArray<I_GeoLocation::GeoLocationField, std::string> &geo_location_data)
+    getGeoLookupVerdict(const std::set<std::string> &sources)
     {
         auto maybe_geo_config = getConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
         if (!maybe_geo_config.ok()) {
-            dbgWarning(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
+            dbgTrace(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
             return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
         }
         GeoConfig geo_config = maybe_geo_config.unpack();
-        string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
+        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
 
-        if (geo_config.isAllowedCountry(country_code)) {
-            dbgTrace(D_GEO_FILTER)
-                << "geo verdict ACCEPT, practice id: "
-                << geo_config.getId()
-                << ", country code: "
-                << country_code;
-            generateVerdictLog(
-                ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
-                event,
-                geo_config.getId(),
-                true,
-                geo_location_data
-            );
-            return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
-        }
-        if (geo_config.isBlockedCountry(country_code)) {
-            dbgTrace(D_GEO_FILTER)
-                << "geo verdict DROP, practice id: "
-                << geo_config.getId()
-                << ", country code: "
-                << country_code;
-            generateVerdictLog(
-                ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
-                event,
-                geo_config.getId(),
-                true,
-                geo_location_data
-            );
-            return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+        for (const std::string& source : sources) {
+            Maybe<IPAddr> maybe_source_ip = IPAddr::createIPAddr(source);
+            if (!maybe_source_ip.ok()){
+                dbgWarning(D_GEO_FILTER) <<
+                "create ip address failed for source: " <<
+                source <<
+                ", Error: " <<
+                maybe_source_ip.getErr();
+                continue;
+            }
+            auto asset_location = i_geo_location->lookupLocation(maybe_source_ip.unpack());
+            if (!asset_location.ok()) {
+                dbgWarning(D_GEO_FILTER) <<
+                "Lookup location failed for source: " <<
+                source <<
+                ", Error: " <<
+                asset_location.getErr();
+                continue;
+            }
+
+            geo_location_data = asset_location.unpack();
+
+            string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+
+            if (geo_config.isAllowedCountry(country_code)) {
+                dbgTrace(D_GEO_FILTER)
+                    << "geo verdict ACCEPT, practice id: "
+                    << geo_config.getId()
+                    << ", country code: "
+                    << country_code;
+                generateVerdictLog(
+                    ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
+                    geo_config.getId(),
+                    true,
+                    geo_location_data
+                );
+                return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+            }
+            if (geo_config.isBlockedCountry(country_code)) {
+                dbgTrace(D_GEO_FILTER)
+                    << "geo verdict DROP, practice id: "
+                    << geo_config.getId()
+                    << ", country code: "
+                    << country_code;
+                generateVerdictLog(
+                    ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
+                    geo_config.getId(),
+                    true,
+                    geo_location_data
+                );
+                return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+            }
         }
         dbgTrace(D_GEO_FILTER)
             << "No matched practice. Returned default action: "
             << geo_config.getDefaultAction();
         generateVerdictLog(
             convertActionToVerdict(geo_config.getDefaultAction()),
-            event,
             geo_config.getId(),
             true,
             geo_location_data,
@@ -176,7 +281,6 @@ private:
     Maybe<pair<ngx_http_cp_verdict_e, string>>
     getBehaviorsVerdict(
         const unordered_map<string, set<string>> &behaviors_map_to_search,
-        const NewHttpTransactionEvent &event,
         EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data)
     {
         bool is_matched = false;
@@ -193,7 +297,6 @@ private:
                 dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId();
                 generateVerdictLog(
                     matched_verdict,
-                    event,
                     behavior.getId(),
                     false,
                     geo_location_data
@@ -218,63 +321,74 @@ private:
     }
 
     ngx_http_cp_verdict_e
-    getExceptionVerdict(
-        const NewHttpTransactionEvent &event,
-        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data
-    ){
-        string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
-        string country_name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME];
-        string source_ip = convertIpAddrToString(event.getSourceIP());
+    getExceptionVerdict(const std::set<std::string> &sources) {
 
         pair<ngx_http_cp_verdict_e, string> curr_matched_behavior;
         ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
+        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
+        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
 
-        dbgTrace(D_GEO_FILTER)
+        for (const std::string& source : sources) {
+
+            Maybe<IPAddr> maybe_source_ip = IPAddr::createIPAddr(source);
+            if (!maybe_source_ip.ok()){
+                dbgWarning(D_GEO_FILTER) <<
+                "create ip address failed for source: " <<
+                source <<
+                ", Error: " <<
+                maybe_source_ip.getErr();
+                continue;
+            }
+
+
+            auto asset_location = i_geo_location->lookupLocation(maybe_source_ip.unpack());
+            if (!asset_location.ok()) {
+                dbgWarning(D_GEO_FILTER) << "Lookup location failed for source: " <<
+                source <<
+                ", Error: " <<
+                asset_location.getErr();
+                continue;
+            }
+            geo_location_data = asset_location.unpack();
+            string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+            string country_name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME];
+            dbgTrace(D_GEO_FILTER)
             << "Get exception verdict. "
             << "country code: "
             << country_code
             << ", country name: "
             << country_name
             << ", source ip address: "
-            << source_ip;
+            << source;
 
-        unordered_map<string, set<string>> exception_value_source_ip = {{"sourceIP", {source_ip}}};
-        auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_source_ip, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
+            unordered_map<string, set<string>> exception_value_country_code = {
+                {"countryCode", {country_code}}
+            };
+            auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, geo_location_data);
+            if (matched_behavior_maybe.ok()) {
+                curr_matched_behavior = matched_behavior_maybe.unpack();
+                verdict = curr_matched_behavior.first;
+                if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+                    return verdict;
+                }
+            }
+
+            unordered_map<string, set<string>> exception_value_country_name = {
+                {"countryName", {country_name}}
+            };
+            matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, geo_location_data);
+            if (matched_behavior_maybe.ok()) {
+                curr_matched_behavior = matched_behavior_maybe.unpack();
+                verdict = curr_matched_behavior.first;
+                if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+                    return verdict;
+                }
             }
         }
 
-        unordered_map<string, set<string>> exception_value_country_code = {
-            {"countryCode", {country_code}}
-        };
-        matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
-            }
-        }
-
-        unordered_map<string, set<string>> exception_value_country_name = {
-            {"countryName", {country_name}}
-        };
-        matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
-            }
-        }
         if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) {
             generateVerdictLog(
                 verdict,
-                event,
                 curr_matched_behavior.second,
                 false,
                 geo_location_data
@@ -286,7 +400,6 @@ private:
     void
     generateVerdictLog(
         const ngx_http_cp_verdict_e &verdict,
-        const NewHttpTransactionEvent &event,
         const string &matched_id,
         bool is_geo_filter,
         const EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data,
@@ -307,14 +420,27 @@ private:
             LogField(matched_on, matched_id),
             ReportIS::Tags::HTTP_GEO_FILTER
         );
-        log
-            << LogField("sourceIP", convertIpAddrToString(event.getSourceIP()))
-            << LogField("sourcePort", event.getSourcePort())
-            << LogField("hostName", event.getDestinationHost())
-            << LogField("httpMethod", event.getHttpMethod())
-            << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
+        auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
+        auto source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+        if (source_ip.ok()) log << LogField("sourceIP", convertIpAddrToString(source_ip.unpack()));
+
+        auto source_identifier = env->get<string>(HttpTransactionData::source_identifier);
+        if (source_identifier.ok()) log << LogField("httpSourceId", source_identifier.unpack());
+
+        auto source_port = env->get<string>(HttpTransactionData::client_port_ctx);
+        if (source_port.ok()) log << LogField("sourcePort", source_port.unpack());
+
+        auto host_name = env->get<string>(HttpTransactionData::host_name_ctx);
+        if (host_name.ok()) log << LogField("hostName", host_name.unpack());
+
+        auto method = env->get<string>(HttpTransactionData::method_ctx);
+        if (method.ok()) log << LogField("httpMethod", method.unpack());
+
+        log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
 
         if (is_default_action) log << LogField("isDefaultSecurityAction", true);
+        auto xff = env->get<string>(HttpTransactionData::xff_vals_ctx);
+        if (xff.ok()) log << LogField("proxyIP", xff.unpack());
 
         log
             << LogField("sourceCountryCode", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE])

components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc

@@ -142,6 +142,13 @@ string disabled_settings =
         "}"
     "],\n";
 
+
+string local_intelligence =
+    "\"intelligence\":{"
+    "  \"local intelligence server ip\":\"127.0.0.1\","
+    "  \"local intelligence server primary port\":9090"
+    "}\n,";
+
 string policy =
     "\"rulebase\": {"
         "\"usersIdentifiers\": ["
@@ -259,7 +266,7 @@ Layer7AccessControlTest::verifyReport(
 
 TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string intelligence_response_ok = loadIntelligenceResponse("data/ok_intelligence_response.json");
@@ -305,7 +312,7 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
 
 TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
     
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -351,7 +358,7 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
 
 TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -403,7 +410,7 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
 
 TEST_F(Layer7AccessControlTest, AcceptOnDetect)
 {
-    stringstream ss_conf(detect_settings + policy);
+    stringstream ss_conf(detect_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
     
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -449,7 +456,7 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect)
 
 TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");

components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc

@@ -27,7 +27,7 @@ NewAppsecTriggerAccessControlLogging::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Access Control Logging";
     parseAppsecJSONKey<bool>("allowEvents", ac_allow_events, archive_in, false);
-    parseAppsecJSONKey<bool>("dropEvents", ac_drop_events, archive_in, false);
+    parseAppsecJSONKey<bool>("dropEvents", ac_drop_events, archive_in, true);
 }
 
 void
@@ -36,8 +36,7 @@ NewAppsecTriggerAdditionalSuspiciousEventsLogging::load(cereal::JSONInputArchive
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Additional Suspicious Events Logging";
     parseAppsecJSONKey<bool>("enabled", enabled, archive_in, true);
     parseAppsecJSONKey<bool>("responseBody", response_body, archive_in, false);
-    //the old code didn't parse the responsecode so ask Noam what is the currenct default value for it
-    parseAppsecJSONKey<bool>("responseCode", response_code, archive_in, false);
+    parseAppsecJSONKey<bool>("responseCode", response_code, archive_in, true);
     parseAppsecJSONKey<string>("minSeverity", minimum_severity, archive_in, "high");
     if (valid_severities.count(minimum_severity) == 0) {
         dbgWarning(D_LOCAL_POLICY)
@@ -133,7 +132,7 @@ void
 NewLoggingService::load(cereal::JSONInputArchive &archive_in)
 {
     parseAppsecJSONKey<string>("address", address, archive_in);
-    parseAppsecJSONKey<string>("proto", proto, archive_in);
+    parseAppsecJSONKey<string>("proto", proto, archive_in, "tcp");
     if (valid_protocols.count(proto) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Logging Service - proto invalid: " << proto;
         throw PolicyGenException("AppSec Logging Service - proto invalid: " + proto);
@@ -175,8 +174,12 @@ void
 NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination";
-    // TBD: support "file"
-    parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") {
+        // TBD: support "file"
+        parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    } else {
+        cloud = false;
+    }
     auto mode = Singleton::Consume<I_AgentDetails>::by<NewAppsecTriggerLogDestination>()->getOrchestrationMode();
     auto env_type = Singleton::Consume<I_EnvDetails>::by<NewAppsecTriggerLogDestination>()->getEnvType();
     bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S);
@@ -184,7 +187,7 @@ NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in)
 
     NewStdoutLogging stdout_log;
     parseAppsecJSONKey<NewStdoutLogging>("stdout", stdout_log, archive_in);
-    agent_local = !(stdout_log.getFormat().empty());
+    parseAppsecJSONKey<bool>("logToAgent", agent_local, archive_in, true);
     beautify_logs = stdout_log.getFormat() == "json-formatted";
     parseAppsecJSONKey<NewLoggingService>("syslogService", syslog_service, archive_in);
     parseAppsecJSONKey<NewLoggingService>("cefService", cef_service, archive_in);

components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc

@@ -147,7 +147,7 @@ string
 PolicyMakerUtils::dumpPolicyToFile(
     const PolicyWrapper &policy,
     const string &policy_path,
-    const string &settings_path)
+    const string &)
 {
     clearElementsMaps();
 
@@ -170,6 +170,7 @@ PolicyMakerUtils::dumpPolicyToFile(
         cereal::JSONOutputArchive ar(settings_ss);
         policy.getSettings().save(ar);
     }
+#if 0
     string settings_str = settings_ss.str();
     try {
         ofstream settings_file(settings_path);
@@ -179,6 +180,7 @@ PolicyMakerUtils::dumpPolicyToFile(
         dbgDebug(D_NGINX_POLICY) << "Error while writing settings to " << settings_path << ", Error: " << e.what();
     }
     dbgDebug(D_LOCAL_POLICY) << settings_path << " content: " << settings_str;
+#endif
 
     return policy_str;
 }

components/security_apps/local_policy_mgmt_gen/triggers_section.cc

@@ -387,8 +387,12 @@ void
 AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination";
-    // TBD: support "file"
-    parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") {
+        // TBD: support "file"
+        parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    } else {
+        cloud = false;
+    }
     auto mode = Singleton::Consume<I_AgentDetails>::by<AppsecTriggerLogDestination>()->getOrchestrationMode();
     auto env_type = Singleton::Consume<I_EnvDetails>::by<AppsecTriggerLogDestination>()->getEnvType();
     bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S);

components/security_apps/orchestration/CMakeLists.txt

@@ -12,6 +12,8 @@ add_subdirectory(manifest_controller)
 add_subdirectory(update_communication)
 add_subdirectory(details_resolver)
 add_subdirectory(health_check)
+add_subdirectory(health_check_manager)
+add_subdirectory(updates_process_reporter)
 add_subdirectory(env_details)
 
 #add_subdirectory(orchestration_ut)

components/security_apps/orchestration/details_resolver/details_resolver.cc

@@ -44,6 +44,8 @@ public:
     bool isGwNotVsx() override;
     bool isVersionAboveR8110() override;
     bool isReverseProxy() override;
+    bool isCloudStorageEnabled() override;
+    Maybe<tuple<string, string, string, string, string>> readCloudMetadata() override;
     Maybe<tuple<string, string, string>> parseNginxMetadata() override;
 #if defined(gaia) || defined(smb)
     bool compareCheckpointVersion(int cp_version, std::function<bool(int, int)> compare_operator) const override;
@@ -135,6 +137,18 @@ DetailsResolver::Impl::isReverseProxy()
     return getenv("DOCKER_RPM_ENABLED") && getenv("DOCKER_RPM_ENABLED") == string("true");
 }
 
+bool
+DetailsResolver::Impl::isCloudStorageEnabled()
+{
+    auto cloud_storage_mode_override = getProfileAgentSetting<bool>("agent.cloudStorage.enabled");
+    if (cloud_storage_mode_override.ok()) {
+        dbgInfo(D_ORCHESTRATOR) << "Received cloud-storage mode override: " << *cloud_storage_mode_override;
+        return *cloud_storage_mode_override;
+    }
+
+    return getenv("CLOUD_STORAGE_ENABLED") && getenv("CLOUD_STORAGE_ENABLED") == string("true");
+}
+
 bool
 DetailsResolver::Impl::isKernelVersion3OrHigher()
 {
@@ -175,17 +189,16 @@ DetailsResolver::Impl::getCheckpointVersion() const
 {
 #ifdef gaia
     static const string cmd =
-        "echo $CPDIR | awk -F'-' '{print $NF}' | cut -c 2- |"
-        " awk -F'.' '{ if( NF == 1 ) {print $1\"00\"} else {print $1$2} }'";
+        "echo $CPDIR | awk '{sub(/.*-R/,\"\"); sub(/\\/.*/,\"\")}/^[0-9]*$/{$0=$0\".00\"}{sub(/\\./, \"\"); print}'";
 #else // smb
     static const string cmd = "sqlcmd 'select major,minor from cpver' |"
         "awk '{if ($1 == \"major\") v += (substr($3,2) * 100);"
         " if ($1 == \"minor\") v += $3; } END { print v}'";
-
 #endif // gaia
     auto version_out = DetailsResolvingHanlder::getCommandOutput(cmd);
     int cp_version = 0;
     if (version_out.ok()) {
+        dbgTrace(D_ORCHESTRATOR) << "Identified version " << version_out.unpack();
         stringstream version_stream(version_out.unpack());
         version_stream >> cp_version;
     }
@@ -287,6 +300,67 @@ DetailsResolver::Impl::parseNginxMetadata()
     return make_tuple(config_opt, cc_opt, nginx_version);
 }
 
+Maybe<tuple<string, string, string, string, string>>
+DetailsResolver::Impl::readCloudMetadata()
+{
+    auto env_read_cloud_metadata = []() -> Maybe<tuple<string, string, string, string, string>> {
+            string account_id = getenv("CLOUD_ACCOUNT_ID") ? getenv("CLOUD_ACCOUNT_ID") : "";
+            string vpc_id = getenv("CLOUD_VPC_ID") ? getenv("CLOUD_VPC_ID") : "";
+            string instance_id = getenv("CLOUD_INSTANCE_ID") ? getenv("CLOUD_INSTANCE_ID") : "";
+            string instance_local_ip = getenv("CLOUD_INSTANCE_LOCAL_IP") ? getenv("CLOUD_INSTANCE_LOCAL_IP") : "";
+            string region = getenv("CLOUD_REGION") ? getenv("CLOUD_REGION") : "";
+
+        if (
+            account_id.empty() ||
+            vpc_id.empty() ||
+            instance_id.empty() ||
+            instance_local_ip.empty() ||
+            region.empty()) {
+            return genError("Could not read cloud metadata");
+        }
+
+        return make_tuple(account_id, vpc_id, instance_id, instance_local_ip, region);
+    };
+
+    auto cloud_metadata = env_read_cloud_metadata();
+    if (!cloud_metadata.ok()) {
+        const string cmd = getFilesystemPathConfig() + "/scripts/get-cloud-metadata.sh";
+        dbgTrace(D_ORCHESTRATOR) << cloud_metadata.getErr() << ", trying to fetch it via cmd: " << cmd;
+
+        auto result = DetailsResolvingHanlder::getCommandOutput(cmd);
+        if (result.ok()) {
+            istringstream iss(result.unpack());
+            string line;
+            while (getline(iss, line)) {
+                size_t pos = line.find('=');
+                if (pos != string::npos) {
+                    string key = line.substr(0, pos);
+                    string value = line.substr(pos + 1);
+                    if (!key.empty() && !value.empty()) setenv(key.c_str(), value.c_str(), 1);
+                }
+            }
+            cloud_metadata = env_read_cloud_metadata();
+        } else {
+            dbgWarning(D_ORCHESTRATOR) << "Could not fetch cloud metadata from cmd: " << result.getErr();
+        }
+    }
+
+    if (!cloud_metadata.ok()) {
+        dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr();
+        return genError("Failed to fetch cloud metadata");
+    }
+
+    dbgTrace(D_ORCHESTRATOR)
+        << "Successfully fetched cloud metadata: "
+        << ::get<0>(cloud_metadata.unpack()) << ", "
+        << ::get<1>(cloud_metadata.unpack()) << ", "
+        << ::get<2>(cloud_metadata.unpack()) << ", "
+        << ::get<3>(cloud_metadata.unpack()) << ", "
+        << ::get<4>(cloud_metadata.unpack());
+
+    return cloud_metadata;
+}
+
 DetailsResolver::DetailsResolver() : Component("DetailsResolver"), pimpl(make_unique<Impl>()) {}
 
 DetailsResolver::~DetailsResolver() {}

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -15,7 +15,9 @@
 #define __CHECKPOINT_PRODUCT_HANDLERS_H__
 
 #include <algorithm>
+#include <regex>
 #include <boost/regex.hpp>
+#include <boost/algorithm/string.hpp>
 
 #if defined(gaia)
 
@@ -32,6 +34,17 @@ checkSAMLSupportedBlade(const string &command_output)
     return genError("Current host does not have SAML capability");
 }
 
+Maybe<string>
+checkIDABlade(const string &command_output)
+{
+    string idaBlade = "identityServer";
+    if (command_output.find(idaBlade) != string::npos) {
+        return string("true");
+    }
+
+    return genError("Current host does not have IDA installed");
+}
+
 Maybe<string>
 checkSAMLPortal(const string &command_output)
 {
@@ -43,9 +56,22 @@ checkSAMLPortal(const string &command_output)
 }
 
 Maybe<string>
-getIDASSamlGaia(const string &command_output)
+checkPepIdaIdnStatus(const string &command_output)
 {
-    return string("idaSaml_gaia");
+    if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) {
+        return string("true");
+    }
+    return genError("Current host does not have PEP control scaled_sharing enabled");
+}
+
+Maybe<string>
+getIDAGaiaPackages(const string &command_output)
+{
+    string result = "idaSaml_gaia;idaIdn_gaia;idaIdnBg_gaia;";
+    if (command_output.find("nac_pep_scaled_sharing_enabled = 1") != string::npos) {
+        result += "agentIntelligenceService_gaia;";
+    }
+    return result;
 }
 
 Maybe<string>
@@ -76,6 +102,18 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
     return command_output;
 }
 
+Maybe<string>
+getQUID(const string &command_output)
+{
+    if (command_output == "" ) return string("false");
+    //  validate valid QUID - contains exactly 4 '-'
+    if (std::count(command_output.begin(), command_output.end(), '-') != 4) {
+        return genError("not valid QUID");
+    }
+    return command_output;
+}
+
+
 Maybe<string>
 checkHasSDWan(const string &command_output)
 {
@@ -92,6 +130,26 @@ checkCanUpdateSDWanData(const string &command_output)
     return string("true");
 }
 
+Maybe<string>
+checkLsmProfileName(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    return genError("LSM profile name was not found");
+}
+
+Maybe<string>
+checkLsmProfileUuid(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    return genError("LSM profile uuid was not found");
+}
+
 Maybe<string>
 getMgmtObjType(const string &command_output)
 {
@@ -271,6 +329,34 @@ getSmbGWIPSecVPNBlade(const string &command_output)
 {
     return getSmbBlade(command_output, "IPSec VPN Blade was not found");
 }
+
+Maybe<string>
+extractManagements(const string &command_output)
+{
+    size_t start_pos = command_output.find(":masters(");
+    if (start_pos == string::npos) {
+        return genError("Starting pattern \":masters(\" not found.");
+    }
+    size_t end_pos = command_output.find("))):", start_pos);
+    if (end_pos == string::npos) {
+        return genError("Ending pattern \"))):\" not found.");
+    }
+    string input_string = command_output.substr(start_pos, end_pos - start_pos + 3);
+    string json_output = "[";
+    regex pattern("\\(ReferenceObject\\:Uid\\(\"\\{([\\w-]+)\\}\"\\)\\:Name\\(([^\\)]+)\\)\\:Table\\(([^\\)]+)\\)\\)");
+    smatch matches;
+    auto words_begin = sregex_iterator(input_string.begin(), input_string.end(), pattern);
+    auto words_end = sregex_iterator();
+    for (sregex_iterator i = words_begin; i != words_end; ++i) {
+        const smatch& match = *i;
+        string uid = boost::algorithm::to_lower_copy(match[1].str());
+        string name = match[2].str();
+        if (json_output.back() != '[') json_output += ",";
+        json_output += "{\"Uid\":\"" + uid + "\",\"Name\":\"" + name + "\"}";
+    }
+    json_output += "]";
+    return json_output;
+}
 #endif // gaia || smb
 
 #if defined(gaia)

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -43,9 +43,15 @@ SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz
 #if defined(gaia) || defined(smb)
 SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
 SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
-    "[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] "
-    "&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
+    "FS_PATH=<FILESYSTEM-PREFIX>; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] "
+    "&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
     checkIsInstallHorizonTelemetrySucceeded)
+SHELL_CMD_HANDLER("QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message || echo ''",
+    getQUID)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(
     "canUpdateSDWanData",
@@ -56,6 +62,16 @@ SHELL_CMD_HANDLER(
     "isSdwanRunning",
     "[ -v $(pidof cp-nano-sdwan) ] && echo 'false' || echo 'true'",
     checkIfSdwanRunning)
+SHELL_CMD_HANDLER(
+    "lsmProfileName",
+    "jq -r .lsm_profile_name /tmp/cpsdwan_getdata_orch.json",
+    checkLsmProfileName
+)
+SHELL_CMD_HANDLER(
+    "lsmProfileUuid",
+    "jq -r .lsm_profile_uuid /tmp/cpsdwan_getdata_orch.json",
+    checkLsmProfileUuid
+)
 SHELL_CMD_HANDLER(
     "IP Address",
     "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" "
@@ -84,8 +100,10 @@ SHELL_CMD_HANDLER(
 
 #if defined(gaia)
 SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade)
-SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status saml-vpn", checkSAMLPortal)
-SHELL_CMD_HANDLER("requiredNanoServices", "ida_saml_gaia", getIDASSamlGaia)
+SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade)
+SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal)
+SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_scaled_sharing_enabled", checkPepIdaIdnStatus)
+SHELL_CMD_HANDLER("requiredNanoServices", "fw ctl get int nac_pep_scaled_sharing_enabled", getIDAGaiaPackages)
 SHELL_CMD_HANDLER(
     "cpProductIntegrationMgmtParentObjectName",
     "cat $FWDIR/database/myself_objects.C "
@@ -134,6 +152,12 @@ SHELL_CMD_HANDLER(
     "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
     getSMCBasedMgmtName
 )
+SHELL_CMD_HANDLER(
+    "managements",
+    "sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |"
+    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    extractManagements
+)
 #endif //gaia
 
 #if defined(smb)
@@ -184,6 +208,13 @@ SHELL_CMD_HANDLER(
     "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
     getSMCBasedMgmtName
 )
+
+SHELL_CMD_HANDLER(
+    "managements",
+    "sed -n '/:masters (/,$p' /tmp/local.cfg |"
+    " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'",
+    extractManagements
+)
 #endif//smb
 
 SHELL_CMD_OUTPUT("kernel_version", "uname -r")

components/security_apps/orchestration/details_resolver/details_resolving_handler.cc

@@ -77,14 +77,22 @@ void
 DetailsResolvingHanlder::Impl::init()
 {
     string actual_filesystem_prefix = getFilesystemPathConfig();
+    size_t place_holder_size = filesystem_place_holder.size();
 
     for (auto &file_handler : file_content_handlers) {
         string &path = file_handler.second.first;
-        size_t place_holder_size = filesystem_place_holder.size();
         if (path.substr(0, place_holder_size) == filesystem_place_holder) {
             path = actual_filesystem_prefix + path.substr(place_holder_size);
         }
     }
+
+    for (auto &cmd_handler_pair : shell_command_handlers) {
+        string &cmd_str = cmd_handler_pair.second.first;
+        size_t fs_pos = cmd_str.find(filesystem_place_holder);
+        if (fs_pos != string::npos) {
+            cmd_str.replace(fs_pos, place_holder_size, actual_filesystem_prefix);
+        }
+    }
 }
 
 map<string, string>

components/security_apps/orchestration/downloader/CMakeLists.txt

@@ -1,5 +1,5 @@
 ADD_DEFINITIONS(-Wno-deprecated-declarations -Dalpine)
 
-add_library(orchestration_downloader curl_client.cc downloader.cc http_client.cc https_client.cc https_client_helper.cc)
+add_library(orchestration_downloader curl_client.cc downloader.cc https_client.cc https_client_helper.cc)
 
 #add_subdirectory(downloader_ut)

components/security_apps/orchestration/downloader/downloader.cc

@@ -15,7 +15,7 @@
 
 #include "i_orchestration_tools.h"
 #include "singleton.h"
-#include "http_client.h"
+#include "https_client.h"
 #include "debug.h"
 #include "config.h"
 #include "rest.h"
@@ -63,7 +63,7 @@ class Downloader::Impl : Singleton::Provide<I_Downloader>::From<Downloader>
 public:
     void init();
 
-    Maybe<string> downloadFileFromFog(
+    Maybe<string> downloadFile(
         const string &checksum,
         Package::ChecksumTypes checksum_type,
         const GetResourceFile &resourse_file
@@ -87,7 +87,7 @@ public:
     string getProfileFromMap(const string &tenant_id) const override;
 
 private:
-    Maybe<string> downloadFileFromFogByHTTP(
+    Maybe<string> downloadAttributeFile(
         const GetResourceFile &resourse_file,
         const string &file_name
     ) const;
@@ -130,12 +130,12 @@ Downloader::Impl::init()
 }
 
 Maybe<string>
-Downloader::Impl::downloadFileFromFog(
+Downloader::Impl::downloadFile(
     const string &checksum,
     Package::ChecksumTypes checksum_type,
     const GetResourceFile &resourse_file) const
 {
-    auto file_path = downloadFileFromFogByHTTP(resourse_file, resourse_file.getFileName() + ".download");
+    auto file_path = downloadAttributeFile(resourse_file, resourse_file.getFileName() + ".download");
 
     if (!file_path.ok()) {
         return file_path;
@@ -378,11 +378,11 @@ Downloader::Impl::validateChecksum(
 }
 
 Maybe<string>
-Downloader::Impl::downloadFileFromFogByHTTP(const GetResourceFile &resourse_file, const string &file_name) const
+Downloader::Impl::downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_name) const
 {
     string file_path = dir_path + "/" + file_name;
 
-    dbgInfo(D_ORCHESTRATOR) << "Downloading file from fog. File: " << resourse_file.getFileName();
+    dbgInfo(D_ORCHESTRATOR) << "Downloading file. File: " << resourse_file.getFileName();
 
     I_UpdateCommunication *update_communication = Singleton::Consume<I_UpdateCommunication>::by<Downloader>();
     auto downloaded_file = update_communication->downloadAttributeFile(resourse_file, file_path);
@@ -408,17 +408,14 @@ Downloader::Impl::getFileFromLocal(const string &local_file_path, const string &
 Maybe<string>
 Downloader::Impl::getFileFromURL(const URLParser &url, const string &file_path, bool auth_required) const
 {
-    ofstream outFile(file_path, ofstream::out | ofstream::binary);
-    HTTPClient http_client;
     dbgInfo(D_ORCHESTRATOR) << "Downloading file. URL: " << url;
-    auto get_file_response = http_client.getFile(url, outFile, auth_required);
+    auto get_file_response = HTTPSClient().getFile(url, file_path, auth_required);
     if (!get_file_response.ok()) {
         Maybe<string> error = genError("Failed to download file from " + url.getBaseURL().unpack() +
                     ". Error: " + get_file_response.getErr());
         dbgWarning(D_ORCHESTRATOR) << "Download failed";
         return error;
     }
-    outFile.close();
     dbgInfo(D_ORCHESTRATOR) << "Download completed. URL: " << url;
     return file_path;
 }

components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc

@@ -38,7 +38,7 @@ TEST_F(DownloaderTest, doNothing)
 {
 }
 
-TEST_F(DownloaderTest, downloadFileFromFog)
+TEST_F(DownloaderTest, downloadFile)
 {
     string fog_response = "bla bla";
     string checksum = "123";
@@ -55,7 +55,7 @@ TEST_F(DownloaderTest, downloadFileFromFog)
 
     EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/virtualSettings.download")).WillOnce(Return(true));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file
@@ -76,7 +76,7 @@ TEST_F(DownloaderTest, downloadFileFromFogFailure)
         downloadAttributeFile(resourse_file, "/tmp/settings.download")
     ).WillOnce(Return(fog_response));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file
@@ -241,7 +241,7 @@ TEST_F(DownloaderTest, downloadEmptyFileFromFog)
         calculateChecksum(Package::ChecksumTypes::SHA256, "/tmp/manifest.download")
     ).WillOnce(Return(checksum));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file

components/security_apps/orchestration/downloader/http_client.cc

@@ -1,268 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "http_client.h"
-
-#include "curl_client.h"
-#include "downloader.h"
-#include "debug.h"
-#include "i_encryptor.h"
-#include "url_parser.h"
-#include "config.h"
-#include "i_environment.h"
-#include "orchestration_comp.h"
-
-#include <fstream>
-#include <string>
-#include <iostream>
-#include <chrono>
-#include <boost/asio/ip/tcp.hpp>
-
-using boost::asio::ip::tcp;
-using namespace std;
-
-USE_DEBUG_FLAG(D_ORCHESTRATOR);
-USE_DEBUG_FLAG(D_HTTP_REQUEST);
-
-// LCOV_EXCL_START Reason: Depends on real download server.
-class ClientConnection
-{
-public:
-    ClientConnection(
-        const URLParser &_url,
-        const Maybe<string> &_proxy_url,
-        const Maybe<uint16_t> &_proxy_port,
-        const Maybe<string> &_proxy_auth,
-        const string &_token)
-            :
-        url(_url),
-        proxy_url(_proxy_url),
-        proxy_port(_proxy_port),
-        proxy_auth(_proxy_auth),
-        token(_token)
-    {
-    }
-
-    Maybe<void>
-    handleConnect()
-    {
-        if (!url.getBaseURL().ok()) {
-            return genError("Failed to handle connection. Error: " + url.getBaseURL().getErr());
-        }
-        string server_name = url.getBaseURL().unpack();
-        string port = url.getPort();
-        string query = url.getQuery();
-        string host = server_name;
-        try {
-            if (stoi(port) != 80) {
-                host = host + ":" + port;
-            }
-        } catch (const exception &err) {
-            return genError("Failed to parse port to a number. Port: " + port );
-        }
-
-        chrono::duration<unsigned int, ratio<1>> sleep_time(60);
-        io_stream.expires_from_now(sleep_time);
-
-        if (proxy_url.ok()) {
-            if (!proxy_port.ok()) {
-                return genError(
-                    "Failed to handle connection to server. proxy domain is defined with invalid port, Error: " +
-                    proxy_port.getErr()
-                );
-            }
-            io_stream.connect(proxy_url.unpack(), to_string(proxy_port.unpack()));
-        } else {
-            io_stream.connect(server_name, port);
-        }
-
-        if (!io_stream) {
-            return genError("Failed to handle connection to server. Error: " + io_stream.error().message());
-        }
-
-        string request_url = query;
-        if (proxy_url.ok()) {
-            request_url = host + query;
-        }
-
-        stringstream http_request;
-        http_request << "GET http://" << request_url << " HTTP/1.1\r\n";
-        http_request << "Host: " << host << "\r\n";
-        if (!token.empty()) {
-            http_request << "Authorization: " << "Bearer " << token << "\r\n";
-        }
-        http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n";
-
-        auto i_trace_env = Singleton::Consume<I_Environment>::by<OrchestrationComp>();
-        http_request << i_trace_env->getCurrentHeaders();
-        http_request << "Accept: */*\r\n";
-
-        if (proxy_url.ok()) {
-            http_request << "Accept-Encoding: identity";
-            http_request << "Connection: close\r\n";
-            http_request << "Proxy-Connection: Keep-Alive\r\n";
-
-            if (proxy_auth.ok()) {
-                I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<Downloader>();
-                http_request << "Proxy-Authorization: Basic " + encryptor->base64Encode(proxy_auth.unpack()) + "\r\n";
-            }
-            http_request << "\r\n";
-        } else {
-            http_request << "Connection: close\r\n\r\n";
-        }
-
-        dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str();
-        io_stream << http_request.str();
-        return Maybe<void>();
-    }
-
-    Maybe<void>
-    handleResponse(ofstream &out_file)
-    {
-        string response_http_version;
-        io_stream >> response_http_version;
-        unsigned int status_code;
-        io_stream >> status_code;
-        string status_message;
-        getline(io_stream, status_message);
-
-        if (!io_stream || response_http_version.substr(0, 5) != "HTTP/")  {
-            return genError("Invalid response");
-        }
-
-        if (status_code != 200) {
-            return genError("HTTP response returned with status code " + status_code);
-        }
-
-        string header;
-        vector<string> headers;
-        while (getline(io_stream, header) && header != "\r") {
-            headers.push_back(header);
-        }
-
-        out_file << io_stream.rdbuf();
-
-        dbgTrace(D_HTTP_REQUEST)
-            << "Received HTTP Response with the following data (downloaded file will not be printed):"
-            << endl
-            << response_http_version
-            << " "
-            << status_code
-            << " "
-            << status_message
-            << endl
-            << makeSeparatedStr(headers, "\n");
-
-
-        return Maybe<void>();
-    }
-
-private:
-    const URLParser url;
-    const Maybe<string> proxy_url;
-    const Maybe<uint16_t> proxy_port;
-    const Maybe<string> proxy_auth;
-    const string &token;
-    boost::asio::ip::tcp::iostream io_stream;
-};
-
-Maybe<void>
-HTTPClient::getFile(const URLParser &url, ofstream &out_file, bool auth_required)
-{
-    auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-    auto load_env_proxy = proxy_config->loadProxy();
-    if (!load_env_proxy.ok()) return load_env_proxy;
-
-    string token = "";
-    if (auth_required) {
-        token = Singleton::Consume<I_AgentDetails>::by<HTTPClient>()->getAccessToken();
-    }
-
-    if (url.isOverSSL()) {
-        if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
-        if (getFileSSL(url, out_file, token).ok()) return Maybe<void>();
-        //CURL fallback
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL).";
-        return curlGetFileOverSSL(url, out_file, token);
-    }
-    auto get_file_http_res = getFileHttp(url, out_file, token);
-    if (!get_file_http_res.ok())
-    {
-        //CURL fallback
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over HTTP. Trying via CURL (HTTP).";
-        return curlGetFileOverHttp(url, out_file, token);
-    }
-
-    return get_file_http_res;
-}
-
-Maybe<void>
-HTTPClient::curlGetFileOverHttp(const URLParser &url, ofstream &out_file, const string &token)
-{
-    try {
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-
-        HttpCurl http_curl_client(
-            url,
-            out_file,
-            token,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
-            proxy_config->getProxyPort(ProxyProtocol::HTTPS),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS));
-
-        http_curl_client.setCurlOpts();
-        bool connection_ok = http_curl_client.connect();
-        if (!connection_ok)
-        {
-            stringstream url_s;
-            url_s << url;
-            string err_msg = string("Failed to get file over HTTP. URL: ") + url_s.str();
-            return genError(err_msg);
-        }
-
-        // As this class is a temporal solution catch all exception types is enabled.
-    } catch (const exception &e) {
-        string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what());
-        return genError(err_msg);
-    }
-
-    return Maybe<void>();
-}
-
-Maybe<void>
-HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string &token)
-{
-    try {
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-        ClientConnection client_connection(
-            url,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTP),
-            proxy_config->getProxyPort(ProxyProtocol::HTTP),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTP),
-            token
-        );
-        auto handle_connect_res = client_connection.handleConnect();
-        if (!handle_connect_res.ok()) return handle_connect_res;
-
-        return client_connection.handleResponse(out_file);
-
-    // As this class is a temporal solution catch all exception types is enabled.
-    } catch (const exception &e) {
-        string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what());
-        return genError(err_msg);
-    }
-
-    return Maybe<void>();
-}
-// LCOV_EXCL_STOP

components/security_apps/orchestration/downloader/http_client.h

@@ -1,44 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __HTTP_CLIENT_H__
-#define __HTTP_CLIENT_H__
-
-#include <string>
-#include "maybe_res.h"
-#include "url_parser.h"
-#include "i_agent_details.h"
-#include "i_proxy_configuration.h"
-
-// LCOV_EXCL_START Reason: Depends on real download server.
-class HTTPClient
-        :
-    public Singleton::Consume<I_AgentDetails>,
-    public Singleton::Consume<I_ProxyConfiguration>
-{
-public:
-    HTTPClient() = default;
-
-    Maybe<void> getFile(const URLParser &url, std::ofstream &out_file, bool auth_required);
-
-private:
-    std::string loadCAChainDir();
-    Maybe<void> getFileSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> getFileSSLDirect(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> getFileHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> curlGetFileOverHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> curlGetFileOverSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-};
-// LCOV_EXCL_STOP
-
-#endif // __HTTP_CLIENT_H__

components/security_apps/orchestration/downloader/https_client.cc

@@ -11,498 +11,51 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "http_client.h"
-#include "curl_client.h"
-
-#include "debug.h"
-#include "i_agent_details.h"
-#include "i_encryptor.h"
-#include "downloader.h"
-#include "config.h"
-#include "boost/uuid/uuid.hpp"
-#include "boost/uuid/uuid_generators.hpp"
-#include <boost/asio/deadline_timer.hpp>
-#include "boost/uuid/uuid_io.hpp"
+#include "https_client.h"
 
+#include <fstream>
 #include <string>
 #include <iostream>
-#include <istream>
-#include <ostream>
-#include <fstream>
-#include <boost/asio.hpp>
-#include <boost/bind/bind.hpp>
-#include <boost/asio/ssl.hpp>
-#include <exception>
+#include <chrono>
+
+#include "config.h"
+#include "curl_client.h"
 
-using namespace boost::placeholders;
-using boost::asio::ip::tcp;
 using namespace std;
 
-USE_DEBUG_FLAG(D_COMMUNICATION);
-USE_DEBUG_FLAG(D_HTTP_REQUEST);
 USE_DEBUG_FLAG(D_ORCHESTRATOR);
+USE_DEBUG_FLAG(D_HTTP_REQUEST);
 
 // LCOV_EXCL_START Reason: Depends on real download server.
-class BadResponseFromServer : public exception
+Maybe<void>
+HTTPSClient::getFile(const URLParser &url, const string &out_file, bool auth_required)
 {
-public:
-    BadResponseFromServer() : message("Bad response returned from server") {}
-    BadResponseFromServer(const string &msg) : message(msg) {}
-    const char *
-    what() const throw()
-    {
-        return message.c_str();
+    auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<OrchestrationComp>();
+    auto load_env_proxy = proxy_config->loadProxy();
+    if (!load_env_proxy.ok()) return load_env_proxy;
+
+    string token = "";
+    if (auth_required) {
+        token = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAccessToken();
     }
 
-private:
-    string message;
-};
+    if (!url.isOverSSL()) return genError("URL is not over SSL.");
 
-class Client
-{
-public:
-    Client(
-        ofstream &out_file,
-        boost::asio::io_service &io_service,
-        boost::asio::ssl::context &context,
-        const URLParser &_url,
-        const Maybe<string> &_proxy_url,
-        const Maybe<uint16_t> &_proxy_port,
-        const Maybe<string> &_proxy_auth,
-        const string &_token)
-            :
-        out_file(out_file),
-        url(_url),
-        proxy_url(_proxy_url),
-        proxy_port(_proxy_port),
-        proxy_auth(_proxy_auth),
-        resolver_(io_service),
-        deadline(io_service),
-        socket_(io_service),
-        ssl_socket(socket_, context),
-        token(_token)
-    {
-    }
+    if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
+    dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
 
-    Maybe<void>
-    handleConnection()
-    {
-        ostream request_stream(&request_);
-        stringstream http_request;
-        http_request << "GET " << url.getQuery() << " HTTP/1.1\r\n";
-        string host = url.getHost();
-        string port = url.getPort();
-        int port_int;
-        try {
-            port_int = stoi(port);
-        } catch (const exception &err) {
-            dbgWarning(D_COMMUNICATION)
-                << "Failed to convert port number from string. Port: "
-                << port
-                << ", Error: "
-                << err.what();
-            return genError("Failed to parse port to a number. Port: " + port);
-        }
-        if (port_int != 443) {
-            host = host + ":" + port;
-        }
+    if (getFileSSL(url, out_file, token).ok()) return Maybe<void>();
+    dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL).";
 
-        http_request << "Host: " << host << "\r\n";
-
-        if (!token.empty()) {
-            http_request << "Authorization: " << "Bearer " << token << "\r\n";
-        }
-        http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n";
-        boost::uuids::uuid correlation_id;
-        try {
-            correlation_id = uuid_random_gen();
-        } catch (const boost::uuids::entropy_error &) {
-            dbgWarning(D_COMMUNICATION) << "Failed to generate random correlation id - entropy exception";
-        }
-        http_request << "X-Trace-Id: " + boost::uuids::to_string(correlation_id) + "\r\n";
-        http_request << "Accept: */*\r\n";
-        http_request << "Connection: close\r\n\r\n";
-
-        request_stream << http_request.str();
-
-        deadline.expires_from_now(boost::posix_time::minutes(5));
-        deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1));
-
-        if (proxy_url.ok()) {
-            if (!proxy_port.ok()) {
-                dbgWarning(D_COMMUNICATION)
-                    << "Failed to connect to proxy due to invalid port value, Error: "
-                    << proxy_port.getErr();
-
-                return genError(
-                    "Failed to handle connection to server. proxy port is invalid, Error: " +
-                    proxy_port.getErr()
-                );
-            }
-            if (port_int == 443) host = host + ":" + port;
-            ostream connect_request_stream(&connect_request);
-            stringstream proxy_request;
-            proxy_request << "CONNECT " << host << " HTTP/1.1\r\n";
-            proxy_request << "Host: " << host << "\r\n";
-            if (proxy_auth.ok()) {
-                I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<Downloader>();
-                proxy_request
-                    << "Proxy-Authorization: Basic "
-                    << encryptor->base64Encode(proxy_auth.unpack())
-                    << "\r\n";
-            }
-            proxy_request << "\r\n";
-
-            dbgTrace(D_HTTP_REQUEST) << "Connecting to proxy: " << endl << proxy_request.str();
-            connect_request_stream << proxy_request.str();
-
-            tcp::resolver::query query(proxy_url.unpack(), to_string(proxy_port.unpack()));
-            resolver_.async_resolve(
-                query,
-                boost::bind(
-                    &Client::overProxyResolver,
-                    this,
-                    boost::asio::placeholders::error,
-                    boost::asio::placeholders::iterator
-                )
-            );
-        } else {
-            tcp::resolver::query query(url.getBaseURL().unpack(), port);
-            resolver_.async_resolve(
-                query,
-                boost::bind(
-                    &Client::handleResolve,
-                    this,
-                    boost::asio::placeholders::error,
-                    boost::asio::placeholders::iterator
-                )
-            );
-        }
-
-        dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str();
-        return Maybe<void>();
-    }
-
-private:
-    void
-    checkDeadline(const boost::system::error_code &err)
-    {
-        if (err) return;
-        if (deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) {
-            boost::system::error_code ignored_ec = boost::asio::error::operation_aborted;
-            socket_.close(ignored_ec);
-            deadline.expires_at(boost::posix_time::pos_infin);
-            return;
-        }
-        deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1));
-    }
-
-    void
-    overProxyResolver(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator)
-    {
-        if (!err) {
-            boost::asio::async_connect(socket_, endpoint_iterator,
-                boost::bind(&Client::overProxyHandleConnect, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_msg = "Failed to connect to proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleConnect(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_write(socket_, connect_request,
-                boost::bind(&Client::overProxyHandleWriteRequest, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_msg = "Failed to connect to proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleWriteRequest(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_read_until(
-                socket_,
-                response_,
-                "\r\n",
-                boost::bind(&Client::overProxyHandleReadStatusLine, this, boost::asio::placeholders::error)
-            );
-        } else {
-            string err_msg = "Failed to write over proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleReadStatusLine(const boost::system::error_code &err)
-    {
-        if (err) {
-            string err_msg = "Failed to read status line over proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-        // Check that response is OK.
-        istream response_stream(&response_);
-        string response_http_version;
-        response_stream >> response_http_version;
-        unsigned int status_code;
-        response_stream >> status_code;
-        string status_message;
-        getline(response_stream, status_message);
-        if (!response_stream || response_http_version.substr(0, 5) != "HTTP/") {
-            throw BadResponseFromServer("Invalid response");
-            return;
-        }
-
-        if (status_code != 200) {
-            string err_msg = "Response returned with status code " + status_code;
-            throw BadResponseFromServer(err_msg);
-        }
-
-        dbgTrace(D_HTTP_REQUEST)
-            << "Received HTTP Response over proxied connection with the following data:"
-            << endl
-            << response_http_version
-            << " "
-            << status_code
-            << " "
-            << status_message;
-
-        if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.message.ignoreSslValidation") == false) {
-            ssl_socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);
-            ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2));
-        } else {
-            dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation";
-        }
-
-        ssl_socket.async_handshake(
-            boost::asio::ssl::stream_base::client,
-            boost::bind(&Client::handleHandshake, this, boost::asio::placeholders::error)
-        );
-    }
-
-    void
-    handleResolve(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator)
-    {
-        if (!err) {
-            boost::asio::async_connect(ssl_socket.lowest_layer(), endpoint_iterator,
-                boost::bind(&Client::handleConnect, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string message = "Failed to connect. Error: " + err.message();
-            throw BadResponseFromServer(message);
-        }
-    }
-
-    bool
-    verifyCertificate(bool preverified, boost::asio::ssl::verify_context &ctx)
-    {
-        if (!token.empty()) {
-            X509_STORE_CTX *cts = ctx.native_handle();
-
-            switch (X509_STORE_CTX_get_error(cts))
-            {
-                case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
-                    dbgWarning(D_ORCHESTRATOR) << "SSL verification error: X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT";
-                    break;
-                case X509_V_ERR_CERT_NOT_YET_VALID:
-                case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD:
-                    dbgWarning(D_ORCHESTRATOR) << "SSL verification error: Certificate not yet valid";
-                    break;
-                case X509_V_ERR_CERT_HAS_EXPIRED:
-                case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD:
-                    dbgWarning(D_ORCHESTRATOR) << "Certificate expired";
-                    break;
-                case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
-                case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
-                    dbgDebug(D_ORCHESTRATOR) << "Self signed certificate in chain";
-                    if (getConfigurationWithDefault(false, "orchestration", "Self signed certificates acceptable")) {
-                        preverified = true;
-                    }
-                    break;
-                default:
-                    if (!preverified) {
-                        dbgWarning(D_ORCHESTRATOR)
-                            << "Certificate verification error number: "
-                            << X509_STORE_CTX_get_error(cts);
-                    }
-                    break;
-            }
-            return preverified;
-        }
-        return true;
-    }
-
-    void
-    handleConnect(const boost::system::error_code &err)
-    {
-        if (!err) {
-            if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.message.ignoreSslValidation") == false) {
-                ssl_socket.set_verify_mode(
-                    boost::asio::ssl::verify_peer |
-                    boost::asio::ssl::verify_fail_if_no_peer_cert
-                );
-                ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2));
-            } else {
-                dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation";
-            }
-
-            ssl_socket.async_handshake(boost::asio::ssl::stream_base::client,
-                boost::bind(&Client::handleHandshake, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Failed to connect. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleHandshake(const boost::system::error_code &error)
-    {
-        if (!error) {
-            boost::asio::buffer_cast<const char*>(request_.data());
-
-            boost::asio::async_write(ssl_socket, request_,
-                boost::bind(&Client::handleWriteRequest, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Handshake failed. Error: " + error.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleWriteRequest(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_read_until(ssl_socket, resp, "\r\n",
-                boost::bind(&Client::handleReadStatusLine, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Failed to handle write request. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadStatusLine(const boost::system::error_code &err)
-    {
-        if (!err) {
-            istream response_stream(&resp);
-            string http_version;
-            response_stream >> http_version;
-            unsigned int status_code;
-            response_stream >> status_code;
-            string status_message;
-            getline(response_stream, status_message);
-            dbgTrace(D_HTTP_REQUEST)
-                << "Received HTTP Response with the following data:"
-                << endl
-                << http_version
-                << " "
-                << status_code;
-
-            if (!response_stream || http_version.substr(0, 5) != "HTTP/") {
-                string err_message = "Invalid response";
-                throw BadResponseFromServer(err_message);
-            }
-            if (status_code != 200) {
-                string err_message = "HTTPS response returned with status code " + to_string(status_code)
-                    + ". URL: " + url.toString();
-                throw BadResponseFromServer(err_message);
-            }
-
-            boost::asio::async_read_until(ssl_socket, resp, "\r\n\r\n",
-                boost::bind(&Client::handleReadHeaders, this,
-                    boost::asio::placeholders::error));
-        } else {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response status. Error:" << err.message();
-            string err_message = "Failed to read status. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadHeaders(const boost::system::error_code &err)
-    {
-        if (!err) {
-            // Process the response headers.
-            istream response_stream(&resp);
-            string header;
-            vector<string> headers;
-            while (getline(response_stream, header) && header != "\r") {
-                headers.push_back(header);
-            }
-
-            dbgTrace(D_HTTP_REQUEST) << "Received Response headers:" << endl << makeSeparatedStr(headers, "\n");
-            // Write whatever content we already have to output.
-            if (resp.size() > 0)
-                out_file << &resp;
-
-            // Start reading remaining data until EOF.
-            boost::asio::async_read(ssl_socket, resp,
-                boost::asio::transfer_at_least(1),
-                boost::bind(&Client::handleReadContent, this,
-                    boost::asio::placeholders::error));
-        } else {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response headers. Error:" << err.message();
-            string err_message = "Failed to read headers. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadContent(const boost::system::error_code &err)
-    {
-        if (!err) {
-            // Write all of the data that has been read so far.
-            out_file << &resp;
-            // Continue reading remaining data until EOF.
-            boost::asio::async_read(
-                ssl_socket,
-                resp,
-                boost::asio::transfer_at_least(1),
-                boost::bind(&Client::handleReadContent, this, boost::asio::placeholders::error)
-            );
-        } else if (err != boost::asio::error::eof && err != boost::asio::ssl::error::stream_truncated) {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response body. Error:" << err.message();
-            string err_message = "Failed to read content. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        } else if (err == boost::asio::ssl::error::stream_truncated) {
-            dbgError(D_COMMUNICATION) << "Had SSL warning during reading response body stage. Error:" << err.message();
-            deadline.cancel();
-        } else {
-            deadline.cancel();
-        }
-    }
-
-    ofstream &out_file;
-    const URLParser &url;
-    const Maybe<string> proxy_url;
-    const Maybe<uint16_t> proxy_port;
-    const Maybe<string> proxy_auth;
-    tcp::resolver resolver_;
-    boost::asio::deadline_timer deadline;
-    boost::asio::ip::tcp::socket socket_;
-    boost::asio::ssl::stream<boost::asio::ip::tcp::socket&> ssl_socket;
-    boost::asio::streambuf request_;
-    boost::asio::streambuf connect_request;
-    boost::asio::streambuf response_;
-    boost::asio::streambuf resp;
-    const string &token;
-    boost::uuids::random_generator uuid_random_gen;
-};
+    //CURL fallback
+    return curlGetFileOverSSL(url, out_file, token);
+}
 
 string
-HTTPClient::loadCAChainDir()
+HTTPSClient::loadCAChainDir()
 {
     string ca_chain_dir;
-    auto agent_details = Singleton::Consume<I_AgentDetails>::by<Downloader>();
+    auto agent_details = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>();
     auto load_ca_chain_dir = agent_details->getOpenSSLDir();
     if (load_ca_chain_dir.ok()) {
         ca_chain_dir = load_ca_chain_dir.unpack();
@@ -511,69 +64,26 @@ HTTPClient::loadCAChainDir()
 }
 
 Maybe<void>
-HTTPClient::getFileSSL(const URLParser &url, ofstream &out_file, const string &token)
+HTTPSClient::getFileSSL(const URLParser &url, const string &out_file, const string &)
 {
-    try {
-        boost::asio::ssl::context ctx(boost::asio::ssl::context::sslv23);
-        if (!token.empty()) {
-            string cert_file_path = getConfigurationWithDefault<string>(
-                getFilesystemPathConfig() + "/certs/fog.pem",
-                "message",
-                "Certificate chain file path"
-            );
-            dbgTrace(D_ORCHESTRATOR) << "Http client, cert file path: " << cert_file_path;
-            auto trusted_ca_directory = getConfiguration<string>("message", "Trusted CA directory");
-            if (trusted_ca_directory.ok() && !trusted_ca_directory.unpack().empty()) {
-                ctx.add_verify_path(trusted_ca_directory.unpack());
-            } else {
-                string cert_file_path = getConfigurationWithDefault<string>(
-                    getFilesystemPathConfig() + "/certs/fog.pem",
-                    "message",
-                    "Certificate chain file path"
-                );
-                ctx.load_verify_file(cert_file_path);
-            }
-        }
-        boost::asio::io_service io_service;
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-
-        Client client(
-            out_file,
-            io_service,
-            ctx,
-            url,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
-            proxy_config->getProxyPort(ProxyProtocol::HTTPS),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS),
-            token
-        );
-
-        auto connection_result = client.handleConnection();
-        if (!connection_result.ok()) {
-            return connection_result;
-        };
-
-        auto mainloop = Singleton::Consume<I_MainLoop>::by<Downloader>();
-        while (!io_service.stopped()) {
-            io_service.poll_one();
-            mainloop->yield(true);
-        }
-    } catch (const exception &e) {
-        dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what());
-        string error_str = "Failed to get file over HTTPS, exception: " + string(e.what());
-        return genError(error_str);
+    auto downlaod_file = Singleton::Consume<I_Messaging>::by<OrchestrationComp>()->downloadFile(
+        HTTPMethod::GET,
+        url.getQuery(),
+        out_file
+    );
+    if (!downlaod_file.ok()) {
+        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Error: " << downlaod_file.getErr().toString();
+        return genError(downlaod_file.getErr().toString());
     }
-
     return Maybe<void>();
 }
 
 Maybe<void>
-HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const string &token)
+HTTPSClient::curlGetFileOverSSL(const URLParser &url, const string &out_file, const string &token)
 {
     try {
         string cert_file_path;
-        if (!token.empty())
-        {
+        if (!token.empty()) {
             cert_file_path = getConfigurationWithDefault<string>(
                 getFilesystemPathConfig() + "/certs/fog.pem",
                 "message",
@@ -581,11 +91,12 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
             );
         }
 
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
+        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<OrchestrationComp>();
+        ofstream out_file_stream(out_file, ofstream::out | ofstream::binary);
 
         HttpsCurl ssl_curl_client(
             url,
-            out_file,
+            out_file_stream,
             token,
             proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
             proxy_config->getProxyPort(ProxyProtocol::HTTPS),
@@ -594,8 +105,7 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
 
         ssl_curl_client.setCurlOpts();
         bool connection_ok = ssl_curl_client.connect();
-        if (!connection_ok)
-        {
+        if (!connection_ok) {
             stringstream url_s;
             url_s << url;
             string err_msg = string("Failed to get file over HTTPS. URL: ") + url_s.str();
@@ -603,12 +113,11 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
         }
 
     } catch (const exception &e) {
-        dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what());
+        dbgWarning(D_HTTP_REQUEST) << "Failed to get file over HTTPS. Error:" << string(e.what());
         string error_str = "Failed to get file over HTTPS, exception: " + string(e.what());
         return genError(error_str);
     }
 
     return Maybe<void>();
 }
-
 // LCOV_EXCL_STOP

components/security_apps/orchestration/downloader/https_client.h

@@ -0,0 +1,36 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __HTTPS_CLIENT_H__
+#define __HTTPS_CLIENT_H__
+
+#include <string>
+#include "maybe_res.h"
+#include "url_parser.h"
+#include "orchestration_comp.h"
+
+// LCOV_EXCL_START Reason: Depends on real download server.
+class HTTPSClient
+{
+public:
+    Maybe<void> getFile(const URLParser &url, const std::string &out_file, bool auth_required);
+
+private:
+    std::string loadCAChainDir();
+    Maybe<void> getFileSSL(const URLParser &url, const std::string &out_file, const std::string &_token);
+    Maybe<void> getFileSSLDirect(const URLParser &url, const std::string &out_file, const std::string &_token);
+    Maybe<void> curlGetFileOverSSL(const URLParser &url, const std::string &out_file, const std::string &_token);
+};
+// LCOV_EXCL_STOP
+
+#endif // __HTTPS_CLIENT_H__

components/security_apps/orchestration/downloader/https_client_helper.cc

@@ -11,10 +11,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "http_client.h"
+#include "https_client.h"
 
 Maybe<void>
-HTTPClient::getFileSSLDirect(const URLParser &, std::ofstream &, const std::string &)
+HTTPSClient::getFileSSLDirect(const URLParser &, const std::string &, const std::string &)
 {
     return genError("No direct downloading in open-source");
 }

components/security_apps/orchestration/env_details/env_details.cc

@@ -15,6 +15,7 @@
 
 #include "config.h"
 #include "debug.h"
+#include "orchestration_tools.h"
 
 using namespace std;
 
@@ -22,10 +23,15 @@ USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
 static const string k8s_service_account = "/var/run/secrets/kubernetes.io/serviceaccount";
 // LCOV_EXCL_START Reason: can't use on the pipline environment
-EnvDetails::EnvDetails()
+EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
 {
+    auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
+    if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
     token = retrieveToken();
-    token.empty() ? env_type = EnvType::LINUX : env_type = EnvType::K8S;
+    if (!token.empty()) {
+        auto env_res = getenv("deployment_type");
+        env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
+    }
 }
 
 EnvType

components/security_apps/orchestration/health_check/health_check.cc

@@ -246,7 +246,7 @@ private:
             }
         }
         routine_id = i_mainloop->addFileRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             server_sock,
             [this] () { handleConnection(); },
             "Health check probe server",
@@ -344,7 +344,7 @@ private:
         dbgDebug(D_HEALTH_CHECK) << "Successfully accepted client, client fd: " << new_client_socket;
         open_connections_counter++;
         auto curr_routine = i_mainloop->addOneTimeRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             [this] ()
             {
                 auto curr_routine_id = i_mainloop->getCurrentRoutineId().unpack();

components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt

@@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib)
 add_unit_test(
     health_check_ut
     "health_check_ut.cc"
-    "health_check;messaging;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system"
+    "health_check;updates_process_reporter;messaging;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system"
 )

components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc

@@ -111,12 +111,12 @@ TEST_F(HealthCheckerTest, clientConnection)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -145,7 +145,7 @@ TEST_F(HealthCheckerTest, loadFromDynamicConfiguration)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     health_checker.init();
@@ -183,7 +183,7 @@ TEST_F(HealthCheckerTest, connectionsLimit)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(mock_mainloop, doesRoutineExist(_)).WillRepeatedly(Return(false));
@@ -218,12 +218,12 @@ TEST_F(HealthCheckerTest, disablingAfterEnabled)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -273,12 +273,12 @@ TEST_F(HealthCheckerTest, changePortIpConfig)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -321,12 +321,12 @@ TEST_F(HealthCheckerTest, FailedHealthCheck)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;

components/security_apps/orchestration/health_check_manager/CMakeLists.txt

@@ -1,3 +1 @@
 add_library(health_check_manager health_check_manager.cc)
-
-add_subdirectory(health_check_manager_ut)

components/security_apps/orchestration/health_check_manager/health_check_manager.cc

@@ -21,6 +21,7 @@
 #include "config.h"
 #include "cereal/archives/json.hpp"
 #include "customized_cereal_map.h"
+#include "updates_process_event.h"
 
 using namespace std;
 
@@ -79,19 +80,22 @@ class HealthCheckValue
 public:
     HealthCheckValue() = default;
 
-    HealthCheckValue(HealthCheckStatus raw_status, const map<string, HealthCheckStatusReply> &descriptions)
+    HealthCheckValue(HealthCheckStatus raw_status, const HealthCheckStatusReply &description)
             :
         status(raw_status)
     {
-        for (const auto &single_stat : descriptions) {
-            if (single_stat.second.getStatus() == HealthCheckStatus::HEALTHY) {
-                dbgTrace(D_HEALTH_CHECK_MANAGER) << "Ignoring healthy status reply. Comp name: " << single_stat.first;
-                continue;
-            }
+        if (description.getStatus() == HealthCheckStatus::HEALTHY) {
+            dbgTrace(D_HEALTH_CHECK_MANAGER)
+                << "Ignoring healthy status reply. Comp name: "
+                << description.getCompName();
+            return;
+        }
 
-            for (const auto &status : single_stat.second.getExtendedStatus()) {
-                errors.push_back(HealthCheckError(single_stat.first + " " + status.first, status.second));
-            }
+        for (const auto &extended_status : description.getExtendedStatus()) {
+            errors.push_back(
+                HealthCheckError(description.getCompName() + " " + extended_status.first,
+                extended_status.second
+            ));
         }
     }
 
@@ -113,9 +117,9 @@ private:
 class HealthCheckPatch : public ClientRest
 {
 public:
-    HealthCheckPatch(HealthCheckStatus raw_status, const map<string, HealthCheckStatusReply> &descriptions)
+    HealthCheckPatch(HealthCheckStatus raw_status, const HealthCheckStatusReply &description)
     {
-        health_check = HealthCheckValue(raw_status, descriptions);
+        health_check = HealthCheckValue(raw_status, description);
     }
 
     C2S_LABEL_PARAM(HealthCheckValue, health_check, "healthCheck");
@@ -123,7 +127,8 @@ public:
 
 class HealthCheckManager::Impl
         :
-    Singleton::Provide<I_Health_Check_Manager>::From<HealthCheckManager>
+    Singleton::Provide<I_Health_Check_Manager>::From<HealthCheckManager>,
+    public Listener<UpdatesProcessEvent>
 {
 public:
     void
@@ -132,6 +137,7 @@ public:
         auto rest = Singleton::Consume<I_RestApi>::by<HealthCheckManager>();
         rest->addRestCall<HealthCheckOnDemand>(RestAction::SHOW, "health-check-on-demand");
 
+        registerListener();
         int interval_in_seconds =
             getProfileAgentSettingWithDefault<int>(30, "agent.healthCheck.intervalInSeconds");
 
@@ -157,9 +163,62 @@ public:
     void
     printRepliesHealthStatus(ofstream &oputput_file)
     {
-        getRegisteredComponentsHealthStatus();
         cereal::JSONOutputArchive ar(oputput_file);
-        ar(cereal::make_nvp("allComponentsHealthCheckReplies", all_comps_health_status));
+        ar(cereal::make_nvp(health_check_reply.getCompName(), health_check_reply));
+    }
+
+    void
+    upon(const UpdatesProcessEvent &event)
+    {
+
+        OrchestrationStatusFieldType status_field_type = event.getStatusFieldType();
+        HealthCheckStatus _status = convertResultToHealthCheckStatus(event.getResult());
+        string status_field_type_str = convertOrchestrationStatusFieldTypeToStr(status_field_type);
+
+        extended_status[status_field_type_str] =
+            _status == HealthCheckStatus::HEALTHY ?
+            "Success" :
+            event.parseDescription();
+        field_types_status[status_field_type_str] = _status;
+
+        switch(_status) {
+            case HealthCheckStatus::UNHEALTHY: {
+                general_health_aggregated_status = HealthCheckStatus::UNHEALTHY;
+                break;
+            }
+            case HealthCheckStatus::DEGRADED: {
+                for (const auto &type_status : field_types_status) {
+                    if ((type_status.first != status_field_type_str)
+                            && (type_status.second == HealthCheckStatus::UNHEALTHY))
+                    {
+                        break;
+                    }
+                }
+                general_health_aggregated_status = HealthCheckStatus::DEGRADED;
+                break;
+            }
+            case HealthCheckStatus::HEALTHY: {
+                for (const auto &type_status : field_types_status) {
+                    if ((type_status.first != status_field_type_str)
+                            && (type_status.second == HealthCheckStatus::UNHEALTHY
+                                || type_status.second == HealthCheckStatus::DEGRADED)
+                        )
+                    {
+                        break;
+                    }
+                    general_health_aggregated_status = HealthCheckStatus::HEALTHY;
+                }
+                break;
+            }
+            case HealthCheckStatus::IGNORED: {
+                break;
+            }
+        }
+        health_check_reply = HealthCheckStatusReply(
+            "Orchestration",
+            general_health_aggregated_status,
+            extended_status
+        );
     }
 
 private:
@@ -168,9 +227,10 @@ private:
     {
         dbgFlow(D_HEALTH_CHECK_MANAGER) << "Sending a health check patch";
 
-        HealthCheckPatch patch_to_send(general_health_aggregated_status, all_comps_health_status);
-        auto messaging = Singleton::Consume<I_Messaging>::by<HealthCheckManager>();
-        return messaging->sendSyncMessageWithoutResponse(
+        HealthCheckPatch patch_to_send(general_health_aggregated_status, health_check_reply);
+        extended_status.clear();
+        field_types_status.clear();
+        return Singleton::Consume<I_Messaging>::by<HealthCheckManager>()->sendSyncMessageWithoutResponse(
             HTTPMethod::PATCH,
             "/agents",
             patch_to_send,
@@ -178,59 +238,11 @@ private:
         );
     }
 
-    void
-    getRegisteredComponentsHealthStatus()
-    {
-        vector<HealthCheckStatusReply> health_check_event_reply = HealthCheckStatusEvent().query();
-        all_comps_health_status.clear();
-        for (const auto &reply : health_check_event_reply) {
-            if (reply.getStatus() != HealthCheckStatus::IGNORED) {
-                all_comps_health_status.emplace(reply.getCompName(), reply);
-            }
-        }
-    }
-
-    void
-    calcGeneralHealthAggregatedStatus()
-    {
-        general_health_aggregated_status = HealthCheckStatus::HEALTHY;
-
-        for (const auto &reply : all_comps_health_status) {
-            HealthCheckStatus status = reply.second.getStatus();
-
-            dbgTrace(D_HEALTH_CHECK_MANAGER)
-                << "Current aggregated status is: "
-                << HealthCheckStatusReply::convertHealthCheckStatusToStr(
-                        general_health_aggregated_status
-                    )
-                << ". Got health status: "
-                << HealthCheckStatusReply::convertHealthCheckStatusToStr(status)
-                << "for component: "
-                << reply.first;
-
-            switch (status) {
-                case HealthCheckStatus::UNHEALTHY : {
-                    general_health_aggregated_status = HealthCheckStatus::UNHEALTHY;
-                    return;
-                }
-                case HealthCheckStatus::DEGRADED : {
-                    general_health_aggregated_status = HealthCheckStatus::DEGRADED;
-                    break;
-                }
-                case HealthCheckStatus::IGNORED : break;
-                case HealthCheckStatus::HEALTHY : break;
-            }
-        }
-    }
-
     void
     executeHealthCheck()
     {
         dbgFlow(D_HEALTH_CHECK_MANAGER) << "Collecting health status from all registered components.";
 
-        getRegisteredComponentsHealthStatus();
-        calcGeneralHealthAggregatedStatus();
-
         dbgTrace(D_HEALTH_CHECK_MANAGER)
             << "Aggregated status: "
             << HealthCheckStatusReply::convertHealthCheckStatusToStr(general_health_aggregated_status);
@@ -244,9 +256,43 @@ private:
         };
     }
 
-    HealthCheckStatus general_health_aggregated_status;
-    map<string, HealthCheckStatusReply> all_comps_health_status;
+    string
+    convertOrchestrationStatusFieldTypeToStr(OrchestrationStatusFieldType type)
+    {
+        switch (type) {
+            case OrchestrationStatusFieldType::REGISTRATION : return "Registration";
+            case OrchestrationStatusFieldType::MANIFEST : return "Manifest";
+            case OrchestrationStatusFieldType::LAST_UPDATE : return "Last Update";
+            case OrchestrationStatusFieldType::COUNT : return "Count";
+        }
+
+        dbgAssert(false) << "Trying to convert unknown orchestration status field to string.";
+        return "";
+    }
+
+    HealthCheckStatus
+    convertResultToHealthCheckStatus(UpdatesProcessResult result)
+    {
+        switch (result) {
+            case UpdatesProcessResult::SUCCESS : return HealthCheckStatus::HEALTHY;
+            case UpdatesProcessResult::UNSET : return HealthCheckStatus::IGNORED;
+            case UpdatesProcessResult::FAILED : return HealthCheckStatus::UNHEALTHY;
+            case UpdatesProcessResult::DEGRADED : return HealthCheckStatus::DEGRADED;
+        }
+
+        dbgAssert(false) << "Trying to convert unknown update process result field to health check status.";
+        return HealthCheckStatus::IGNORED;
+    }
+
+    HealthCheckStatus general_health_aggregated_status = HealthCheckStatus::HEALTHY;
+    HealthCheckStatusReply health_check_reply = HealthCheckStatusReply(
+        "Orchestration",
+        HealthCheckStatus::HEALTHY,
+        {}
+    );
     bool should_patch_report;
+    map<string, string> extended_status;
+    map<string, HealthCheckStatus> field_types_status;
 };
 
 HealthCheckManager::HealthCheckManager() : Component("HealthCheckManager"), pimpl(make_unique<Impl>()) {}

components/security_apps/orchestration/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc

@@ -13,42 +13,13 @@
 #include "mock/mock_mainloop.h"
 #include "mock/mock_messaging.h"
 #include "mock/mock_rest_api.h"
+#include "updates_process_event.h"
 
 using namespace std;
 using namespace testing;
 
 USE_DEBUG_FLAG(D_HEALTH_CHECK);
 
-class TestHealthCheckStatusListener : public Listener<HealthCheckStatusEvent>
-{
-public:
-    void upon(const HealthCheckStatusEvent &) override {}
-
-    HealthCheckStatusReply
-    respond(const HealthCheckStatusEvent &) override
-    {
-        map<string, string> extended_status;
-        extended_status["team"] = team;
-        extended_status["city"] = city;
-        HealthCheckStatusReply reply(comp_name, status, extended_status);
-        return reply;
-    }
-
-    void setStatus(HealthCheckStatus new_status) { status = new_status; }
-
-    string getListenerName() const { return "TestHealthCheckStatusListener"; }
-
-private:
-    static const string comp_name;
-    HealthCheckStatus status = HealthCheckStatus::HEALTHY;
-    static const string team;
-    static const string city;
-};
-
-const string TestHealthCheckStatusListener::comp_name = "Test";
-const string TestHealthCheckStatusListener::team = "Hapoel";
-const string TestHealthCheckStatusListener::city = "Tel-Aviv";
-
 class TestEnd {};
 
 class HealthCheckManagerTest : public Test
@@ -56,8 +27,7 @@ class HealthCheckManagerTest : public Test
 public:
     HealthCheckManagerTest()
     {
-        Debug::setNewDefaultStdout(&debug_output);
-        Debug::setUnitTestFlag(D_HEALTH_CHECK, Debug::DebugLevel::INFO);
+        Debug::setUnitTestFlag(D_HEALTH_CHECK, Debug::DebugLevel::NOISE);
 
         EXPECT_CALL(mock_ml, addRecurringRoutine(_, _, _, _, _)).WillRepeatedly(
             DoAll(SaveArg<2>(&health_check_periodic_routine), Return(1))
@@ -70,7 +40,6 @@ public:
         );
 
         env.preload();
-        event_listener.registerListener();
 
         env.init();
 
@@ -98,14 +67,12 @@ public:
     StrictMock<MockMainLoop>        mock_ml;
     StrictMock<MockRestApi>         mock_rest;
     StrictMock<MockMessaging>       mock_message;
-    stringstream                    debug_output;
     ConfigComponent                 config;
     Config::I_Config                *i_config = nullptr;
     ::Environment                   env;
     HealthCheckManager              health_check_manager;
     I_Health_Check_Manager          *i_health_check_manager;
     unique_ptr<ServerRest>          health_check_server;
-    TestHealthCheckStatusListener   event_listener;
 };
 
 TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
@@ -142,7 +109,20 @@ TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
     EXPECT_EQ(actual_body, expected_healthy_body);
     EXPECT_EQ("Healthy", aggregated_status_str);
 
-    event_listener.setStatus(HealthCheckStatus::DEGRADED);
+    UpdatesProcessEvent(
+        UpdatesProcessResult::DEGRADED,
+        UpdatesConfigType::SETTINGS,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "setting.json",
+        "File not found"
+    ).notify();
+    UpdatesProcessEvent(
+        UpdatesProcessResult::DEGRADED,
+        UpdatesConfigType::MANIFEST,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "manifest.json",
+        "File not found"
+    ).notify();
     try {
         health_check_periodic_routine();
     } catch (const TestEnd &t) {}
@@ -156,16 +136,16 @@ TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
         "        \"status\": \"Degraded\",\n"
         "        \"errors\": [\n"
         "            {\n"
-        "                \"code\": \"Test city\",\n"
+        "                \"code\": \"Orchestration Last Update\",\n"
         "                \"message\": [\n"
-        "                    \"Tel-Aviv\"\n"
+        "                    \"Failed to download the file setting.json. Error: File not found\"\n"
         "                ],\n"
         "                \"internal\": true\n"
         "            },\n"
         "            {\n"
-        "                \"code\": \"Test team\",\n"
+        "                \"code\": \"Orchestration Manifest\",\n"
         "                \"message\": [\n"
-        "                    \"Hapoel\"\n"
+        "                    \"Failed to download the file manifest.json. Error: File not found\"\n"
         "                ],\n"
         "                \"internal\": true\n"
         "            }\n"
@@ -196,19 +176,24 @@ TEST_F(HealthCheckManagerTest, runOnDemandHealthCheckTest)
     config.preload();
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
 
+    UpdatesProcessEvent(
+        UpdatesProcessResult::FAILED,
+        UpdatesConfigType::MANIFEST,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "manifest.json",
+        "File not found"
+    ).notify();
+
     stringstream is;
     is << "{}";
     health_check_server->performRestCall(is);
 
     string expected_status =
         "{\n"
-        "    \"allComponentsHealthCheckReplies\": {\n"
-        "        \"Test\": {\n"
-        "            \"status\": \"Healthy\",\n"
-        "            \"extendedStatus\": {\n"
-        "                \"city\": \"Tel-Aviv\",\n"
-        "                \"team\": \"Hapoel\"\n"
-        "            }\n"
+        "    \"Orchestration\": {\n"
+        "        \"status\": \"Unhealthy\",\n"
+        "        \"extendedStatus\": {\n"
+        "            \"Manifest\": \"Failed to download the file manifest.json. Error: File not found\"\n"
         "        }\n"
         "    }\n"
         "}";

components/security_apps/orchestration/include/fog_authenticator.h

@@ -33,6 +33,7 @@
 #include "i_rest_api.h"
 #include "i_time_get.h"
 #include "i_encryptor.h"
+#include "i_shell_cmd.h"
 #include "maybe_res.h"
 
 class FogAuthenticator
@@ -43,10 +44,13 @@ class FogAuthenticator
     Singleton::Consume<I_DetailsResolver>,
     Singleton::Consume<I_OrchestrationStatus>,
     Singleton::Consume<I_OrchestrationTools>,
+    Singleton::Consume<I_EnvDetails>,
     Singleton::Consume<I_Encryptor>,
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_Messaging>,
-    Singleton::Consume<I_TimeGet>
+    Singleton::Consume<I_TimeGet>,
+    Singleton::Consume<I_ShellCmd>,
+    Singleton::Consume<I_Environment>
 {
     class AccessToken
     {
@@ -88,6 +92,8 @@ public:
         void serialize(cereal::JSONOutputArchive &out_ar) const;
         void serialize(cereal::JSONInputArchive &in_ar);
 
+        std::string getData() const;
+
     private:
         AuthenticationType type;
         std::string data;
@@ -103,6 +109,7 @@ public:
     Maybe<void> authenticateAgent() override;
     void setAddressExtenesion(const std::string &extension) override;
     static std::string getUserEdition();
+    void registerLocalAgentToFog() override;
 
 protected:
     class UserCredentials
@@ -138,6 +145,7 @@ protected:
 
     bool saveCredentialsToFile(const UserCredentials &credentials) const;
     Maybe<UserCredentials> getCredentialsFromFile() const;
+    Maybe<RegistrationData> getRegistrationToken();
     Maybe<RegistrationData> getRegistrationData();
 
     std::string base64Encode(const std::string &in) const;

components/security_apps/orchestration/include/hybrid_communication.h

@@ -37,10 +37,7 @@
 #include "maybe_res.h"
 #include "declarative_policy_utils.h"
 
-class HybridCommunication
-        :
-    public FogAuthenticator,
-    Singleton::Consume<I_EnvDetails>
+class HybridCommunication : public FogAuthenticator
 {
 public:
     void init() override;

components/security_apps/orchestration/include/local_communication.h

@@ -29,6 +29,7 @@ public:
     void init();
 
     Maybe<void> authenticateAgent() override;
+    void registerLocalAgentToFog() override;
     Maybe<void> getUpdate(CheckUpdateRequest &request) override;
 
     Maybe<std::string> downloadAttributeFile(

components/security_apps/orchestration/include/mock/mock_details_resolver.h

@@ -26,6 +26,13 @@ operator<<(std::ostream &os, const Maybe<std::tuple<std::string, std::string, st
     return os;
 }
 
+std::ostream &
+operator<<(
+    std::ostream &os, const Maybe<std::tuple<std::string, std::string, std::string, std::string, std::string>> &)
+{
+    return os;
+}
+
 class MockDetailsResolver
         :
     public Singleton::Provide<I_DetailsResolver>::From<MockProvider<I_DetailsResolver>>
@@ -35,12 +42,15 @@ public:
     MOCK_METHOD0(getPlatform,                Maybe<std::string>());
     MOCK_METHOD0(getArch,                    Maybe<std::string>());
     MOCK_METHOD0(getAgentVersion,            std::string());
+    MOCK_METHOD0(isCloudStorageEnabled,        bool());
     MOCK_METHOD0(isReverseProxy,             bool());
     MOCK_METHOD0(isKernelVersion3OrHigher,   bool());
     MOCK_METHOD0(isGwNotVsx,                 bool());
     MOCK_METHOD0(getResolvedDetails,         std::map<std::string, std::string>());
     MOCK_METHOD0(isVersionAboveR8110, bool());
     MOCK_METHOD0(parseNginxMetadata, Maybe<std::tuple<std::string, std::string, std::string>>());
+    MOCK_METHOD0(
+        readCloudMetadata, Maybe<std::tuple<std::string, std::string, std::string, std::string, std::string>>());
 };
 
 #endif // __MOCK_DETAILS_RESOLVER_H__

components/security_apps/orchestration/include/mock/mock_downloader.h

@@ -24,7 +24,7 @@ class MockDownloader :
 {
 public:
     MOCK_CONST_METHOD3(
-        downloadFileFromFog,
+        downloadFile,
         Maybe<std::string>(const std::string &, Package::ChecksumTypes, const GetResourceFile &)
     );
 

components/security_apps/orchestration/include/mock/mock_service_controller.h

@@ -64,7 +64,7 @@ public:
         )
     );
 
-    typedef std::map<std::string, PortNumber> ServicePortMap;
+    typedef std::map<std::string, std::vector<PortNumber>> ServicePortMap;
     MOCK_METHOD0(getServiceToPortMap, ServicePortMap());
     MOCK_METHOD3(updateReconfStatus, void(int id, const std::string &service_name, ReconfStatus status));
     MOCK_METHOD4(

components/security_apps/orchestration/include/mock/mock_update_communication.h

@@ -29,6 +29,7 @@ class MockUpdateCommunication :
 public:
     void init() {}
     MOCK_METHOD0(authenticateAgent, Maybe<void>());
+    MOCK_METHOD0(registerLocalAgentToFog, void());
     MOCK_METHOD1(getUpdate, Maybe<void>(CheckUpdateRequest &));
     MOCK_METHOD2(
         downloadAttributeFile,

components/security_apps/orchestration/include/updates_process_event.h

@@ -0,0 +1,130 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __UPDATES_PROCESS_EVENT_H__
+#define __UPDATES_PROCESS_EVENT_H__
+
+#include "event.h"
+#include "singleton.h"
+#include "config.h"
+#include "debug.h"
+#include "i_orchestration_status.h"
+#include "health_check_status/health_check_status.h"
+#include "customized_cereal_map.h"
+
+USE_DEBUG_FLAG(D_UPDATES_PROCESS_REPORTER);
+
+enum class UpdatesFailureReason {
+    CHECK_UPDATE,
+    REGISTRATION,
+    ORCHESTRATION_SELF_UPDATE,
+    GET_UPDATE_REQUEST,
+    DOWNLOAD_FILE,
+    HANDLE_FILE,
+    INSTALLATION_QUEUE,
+    INSTALL_PACKAGE,
+    CHECKSUM_UNMATCHED,
+    POLICY_CONFIGURATION,
+    POLICY_FOG_CONFIGURATION,
+    NONE
+
+};
+
+enum class UpdatesConfigType { MANIFEST, POLICY, SETTINGS, DATA, GENERAL };
+enum class UpdatesProcessResult { UNSET, SUCCESS, FAILED, DEGRADED };
+
+static inline std::string
+convertUpdatesFailureReasonToStr(UpdatesFailureReason reason)
+{
+    switch (reason) {
+        case UpdatesFailureReason::CHECK_UPDATE : return "CHECK_UPDATE";
+        case UpdatesFailureReason::REGISTRATION : return "REGISTRATION";
+        case UpdatesFailureReason::ORCHESTRATION_SELF_UPDATE : return "ORCHESTRATION_SELF_UPDATE";
+        case UpdatesFailureReason::GET_UPDATE_REQUEST : return "GET_UPDATE_REQUEST";
+        case UpdatesFailureReason::DOWNLOAD_FILE : return "DOWNLOAD_FILE";
+        case UpdatesFailureReason::HANDLE_FILE : return "HANDLE_FILE";
+        case UpdatesFailureReason::INSTALLATION_QUEUE : return "INSTALLATION_QUEUE";
+        case UpdatesFailureReason::INSTALL_PACKAGE : return "INSTALL_PACKAGE";
+        case UpdatesFailureReason::CHECKSUM_UNMATCHED : return "CHECKSUM_UNMATCHED";
+        case UpdatesFailureReason::POLICY_CONFIGURATION : return "POLICY_CONFIGURATION";
+        case UpdatesFailureReason::POLICY_FOG_CONFIGURATION : return "POLICY_FOG_CONFIGURATION";
+        case UpdatesFailureReason::NONE : return "NONE";
+    }
+
+    dbgWarning(D_UPDATES_PROCESS_REPORTER) << "Trying to convert unknown updates failure reason to string.";
+    return "";
+}
+
+static inline std::string
+convertUpdatesConfigTypeToStr(UpdatesConfigType type)
+{
+    switch (type) {
+        case UpdatesConfigType::MANIFEST : return "MANIFEST";
+        case UpdatesConfigType::POLICY : return "POLICY";
+        case UpdatesConfigType::SETTINGS : return "SETTINGS";
+        case UpdatesConfigType::DATA : return "DATA";
+        case UpdatesConfigType::GENERAL : return "GENERAL";
+    }
+
+    dbgWarning(D_UPDATES_PROCESS_REPORTER) << "Trying to convert unknown updates failure reason to string.";
+    return "";
+}
+
+static inline std::string
+convertUpdateProcessResultToStr(UpdatesProcessResult result)
+{
+    switch (result) {
+        case UpdatesProcessResult::SUCCESS : return "SUCCESS";
+        case UpdatesProcessResult::UNSET : return "UNSET";
+        case UpdatesProcessResult::FAILED : return "FAILURE";
+        case UpdatesProcessResult::DEGRADED : return "DEGRADED";
+    }
+
+    dbgWarning(D_UPDATES_PROCESS_REPORTER) << "Trying to convert unknown updates failure reason to string.";
+    return "";
+}
+
+class UpdatesProcessEvent : public Event<UpdatesProcessEvent>
+{
+public:
+    UpdatesProcessEvent() {}
+    UpdatesProcessEvent(
+        UpdatesProcessResult _result,
+        UpdatesConfigType _type,
+        UpdatesFailureReason _reason = UpdatesFailureReason::NONE,
+        const std::string &_detail = "",
+        const std::string &_description = "");
+
+    ~UpdatesProcessEvent() {}
+
+    UpdatesProcessResult getResult() const { return result; }
+    UpdatesConfigType getType() const { return type; }
+    UpdatesFailureReason getReason() const { return reason; }
+    std::string getDetail() const { return detail; }
+    std::string getDescription() const { return description; }
+
+    OrchestrationStatusFieldType getStatusFieldType() const;
+    OrchestrationStatusResult getOrchestrationStatusResult() const;
+
+    std::string parseDescription() const;
+
+private:
+    UpdatesProcessResult result;
+    UpdatesConfigType type;
+    UpdatesFailureReason reason;
+    std::string detail;
+    std::string description;
+
+};
+
+#endif // __UPDATES_PROCESS_EVENT_H__

components/security_apps/orchestration/include/updates_process_report.h

@@ -0,0 +1,61 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __UPDATES_PROCESS_REPORT_H__
+#define __UPDATES_PROCESS_REPORT_H__
+
+#include <sstream>
+#include <string>
+
+#include "singleton.h"
+#include "i_time_get.h"
+#include "updates_process_event.h"
+
+class UpdatesProcessReport : Singleton::Consume<I_TimeGet>
+{
+public:
+    UpdatesProcessReport(
+        UpdatesProcessResult result,
+        UpdatesConfigType type,
+        UpdatesFailureReason reason,
+        const std::string &description)
+            :
+        result(result), type(type), reason(reason), description(description)
+    {
+        time_stamp = Singleton::Consume<I_TimeGet>::by<UpdatesProcessReport>()->getWalltimeStr();
+    }
+
+    std::string
+    toString() const
+    {
+        std::stringstream report;
+        report
+            << "["
+            << time_stamp << "] - "
+            << convertUpdateProcessResultToStr(result) << " | "
+            << convertUpdatesConfigTypeToStr(type) << " | "
+            << convertUpdatesFailureReasonToStr(reason) << " | "
+            << description;
+
+        return report.str();
+    }
+
+private:
+    UpdatesProcessResult result;
+    UpdatesConfigType type;
+    UpdatesFailureReason reason;
+    std::string description;
+    std::string time_stamp;
+};
+
+#endif // __UPDATES_PROCESS_EVENT_H__

components/security_apps/orchestration/include/updates_process_reporter.h

@@ -0,0 +1,39 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __UPDATES_PROCESS_REPORTER_H__
+#define __UPDATES_PROCESS_REPORTER_H__
+
+#include <string>
+
+#include "event.h"
+#include "singleton.h"
+#include "config.h"
+#include "debug.h"
+#include "i_orchestration_status.h"
+#include "health_check_status/health_check_status.h"
+#include "updates_process_event.h"
+#include "updates_process_report.h"
+
+class UpdatesProcessReporter : public Listener<UpdatesProcessEvent>
+{
+public:
+    void upon(const UpdatesProcessEvent &event) override;
+
+private:
+    void sendReoprt();
+
+    static std::vector<UpdatesProcessReport> reports;
+};
+
+#endif // __UPDATES_PROCESS_REPORTER_H__

components/security_apps/orchestration/manifest_controller/manifest_controller.cc

@@ -21,6 +21,7 @@
 #include "version.h"
 #include "log_generator.h"
 #include "orchestration_comp.h"
+#include "updates_process_event.h"
 
 using namespace std;
 using namespace ReportIS;
@@ -219,6 +220,13 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file)
     if (isIgnoreFile(new_manifest_file)) {
         if (!orchestration_tools->copyFile(new_manifest_file, manifest_file_path)) {
             dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new manifest file";
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::MANIFEST,
+                UpdatesFailureReason::HANDLE_FILE,
+                new_manifest_file,
+                "Failed to copy a new manifest file"
+            ).notify();
             return false;
         }
         return true;
@@ -237,6 +245,13 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file)
 
         if (!orchestration_tools->copyFile(new_manifest_file, manifest_file_path)) {
             dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new manifest file";
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::MANIFEST,
+                UpdatesFailureReason::HANDLE_FILE,
+                new_manifest_file,
+                "Failed to copy a new manifest file"
+            ).notify();
             return false;
         }
         return true;
@@ -245,6 +260,13 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file)
     Maybe<map<string, Package>> parsed_manifest = orchestration_tools->loadPackagesFromJson(new_manifest_file);
     if (!parsed_manifest.ok()) {
         dbgWarning(D_ORCHESTRATOR) << "Failed to parse the new manifest file. File: " << new_manifest_file;
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::MANIFEST,
+            UpdatesFailureReason::HANDLE_FILE,
+            new_manifest_file,
+            "Failed to parse the new manifest file"
+        ).notify();
         return false;
     }
 
@@ -332,6 +354,13 @@ ManifestController::Impl::updateManifest(const string &new_manifest_file)
         dbgWarning(D_ORCHESTRATOR)
             << "Failed building installation queue. Error: "
             << installation_queue_res.getErr();
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::MANIFEST,
+            UpdatesFailureReason::INSTALLATION_QUEUE,
+            "",
+            installation_queue_res.getErr()
+        ).notify();
         return false;
     }
     const vector<Package> &installation_queue = installation_queue_res.unpack();
@@ -447,11 +476,25 @@ ManifestController::Impl::changeManifestFile(const string &new_manifest_file)
     dbgDebug(D_ORCHESTRATOR) << "Writing new manifest to file";
     if (!orchestration_tools->copyFile(new_manifest_file, manifest_file_path)) {
         dbgWarning(D_ORCHESTRATOR) << "Failed write new manifest to file";
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::MANIFEST,
+            UpdatesFailureReason::HANDLE_FILE,
+            new_manifest_file,
+            "Failed write new manifest to file"
+        ).notify();
         return false;
     }
 
     if (!orchestration_tools->isNonEmptyFile(manifest_file_path)) {
         dbgWarning(D_ORCHESTRATOR) << "Failed to get manifest file data";
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::MANIFEST,
+            UpdatesFailureReason::HANDLE_FILE,
+            manifest_file_path,
+            "Failed to get manifest file data"
+        ).notify();
         return false;
     }
 

components/security_apps/orchestration/manifest_controller/manifest_controller_ut/manifest_controller_ut.cc

@@ -281,13 +281,7 @@ TEST_F(ManifestControllerTest, badChecksum)
     EXPECT_CALL(mock_orchestration_tools, doesFileExist("/etc/cp/packages/my/my")).WillOnce(Return(false));
 
     string hostname = "hostname";
-    string empty_err;
-    EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(empty_err));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return( Maybe<string>(hostname)));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
     EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
 }
 
@@ -710,10 +704,6 @@ TEST_F(ManifestControllerTest, selfUpdateWithOldCopyWithError)
     string hostname = "hostname";
     string empty_err;
     EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(empty_err));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
     load(manifest, new_services);
 
     EXPECT_CALL(mock_orchestration_tools,
@@ -932,10 +922,6 @@ TEST_F(ManifestControllerTest, badInstall)
     string empty_err;
     EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(empty_err));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return( Maybe<string>(hostname)));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
 
     string corrupted_packages_manifest =
         "{"
@@ -1008,12 +994,6 @@ TEST_F(ManifestControllerTest, failToDownloadWithselfUpdate)
         doesFileExist("/etc/cp/packages/orchestration/orchestration")
     ).WillOnce(Return(false));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname")));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
-    string not_error;
-    EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(not_error));
     EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
 }
 
@@ -1404,12 +1384,6 @@ TEST_F(ManifestControllerTest, failureOnDownloadSharedObject)
     ).WillOnce(Return(false));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname")));
     EXPECT_CALL(mock_orchestration_tools, removeFile("/tmp/temp_file1")).WillOnce(Return(true));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
-    string not_error;
-    EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(not_error));
 
     EXPECT_FALSE(i_manifest_controller->updateManifest(file_name));
 }
@@ -2538,12 +2512,6 @@ TEST_F(ManifestDownloadTest, download_relative_path)
         doesFileExist("/etc/cp/packages/orchestration/orchestration")
     ).WillOnce(Return(false));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname")));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
-    string not_error;
-    EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(not_error));
 
     EXPECT_FALSE(i_manifest_controller->updateManifest(manifest_file.fname));
 }
@@ -2589,8 +2557,6 @@ TEST_F(ManifestDownloadTest, download_relative_path_no_fog_domain)
         mock_orchestration_tools,
         doesFileExist("/etc/cp/packages/orchestration/orchestration")
     ).WillOnce(Return(false));
-    string not_error;
-    EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(not_error));
 
     checkIfFileExistsCall(new_packages.at("orchestration"));
 
@@ -2604,10 +2570,6 @@ TEST_F(ManifestDownloadTest, download_relative_path_no_fog_domain)
         )
     ).WillOnce(Return(downloaded_package));
     EXPECT_CALL(mock_details_resolver, getHostname()).WillOnce(Return(string("hostname")));
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::FAILED, _)
-    );
 
     EXPECT_FALSE(i_manifest_controller->updateManifest(manifest_file.fname));
 }

components/security_apps/orchestration/manifest_controller/manifest_handler.cc

@@ -19,6 +19,7 @@
 #include "config.h"
 #include "agent_details.h"
 #include "orchestration_comp.h"
+#include "updates_process_event.h"
 
 using namespace std;
 
@@ -174,14 +175,13 @@ ManifestHandler::downloadPackages(const map<string, Package> &new_packages_to_do
                     " software update failed. Agent is running previous software. Contact Check Point support.";
             }
 
-            auto orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<ManifestHandler>();
-            if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) {
-                orchestration_status->setFieldStatus(
-                    OrchestrationStatusFieldType::MANIFEST,
-                    OrchestrationStatusResult::FAILED,
-                    install_error
-                );
-            }
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::MANIFEST,
+                UpdatesFailureReason::DOWNLOAD_FILE,
+                package.getName(),
+                install_error
+            ).notify();
             return genError(
                 "Failed to download installation package. Package: " +
                 package.getName() +
@@ -219,11 +219,13 @@ ManifestHandler::installPackage(
                 err_hostname +
                 " software update failed. Agent is running previous software. Contact Check Point support.";
             if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) {
-                orchestration_status->setFieldStatus(
-                    OrchestrationStatusFieldType::MANIFEST,
-                    OrchestrationStatusResult::FAILED,
+                UpdatesProcessEvent(
+                    UpdatesProcessResult::FAILED,
+                    UpdatesConfigType::MANIFEST,
+                    UpdatesFailureReason::INSTALL_PACKAGE,
+                    package_name,
                     install_error
-                );
+                ).notify();
             }
         }
         return self_update_status;
@@ -289,11 +291,13 @@ ManifestHandler::installPackage(
 
         auto orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<ManifestHandler>();
         if (orchestration_status->getManifestError().find("Gateway was not fully deployed") == string::npos) {
-            orchestration_status->setFieldStatus(
-                OrchestrationStatusFieldType::MANIFEST,
-                OrchestrationStatusResult::FAILED,
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::MANIFEST,
+                UpdatesFailureReason::INSTALL_PACKAGE,
+                package_name,
                 install_error
-            );
+            ).notify();
         }
         return false;
     }

components/security_apps/orchestration/modules/modules_ut/orchestration_status_ut.cc

@@ -13,6 +13,7 @@
 #include "mock/mock_agent_details.h"
 #include "mock/mock_mainloop.h"
 #include "mock/mock_rest_api.h"
+#include "updates_process_event.h"
 
 using namespace testing;
 using namespace std;
@@ -200,6 +201,19 @@ TEST_F(OrchestrationStatusTest, checkUpdateStatus)
     auto result = orchestrationStatusFileToString();
     EXPECT_EQ(buildOrchestrationStatusJSON("attempt time", "Succeeded ", "current time"), result);
 }
+TEST_F(OrchestrationStatusTest, checkUpdateStatusByRaiseEvent)
+{
+    init();
+    EXPECT_CALL(time, getLocalTimeStr())
+        .WillOnce(Return(string("attempt time")))
+        .WillOnce(Return(string("current time")));
+
+    i_orchestration_status->setLastUpdateAttempt();
+
+    UpdatesProcessEvent(UpdatesProcessResult::SUCCESS, UpdatesConfigType::GENERAL).notify();
+    auto result = orchestrationStatusFileToString();
+    EXPECT_EQ(buildOrchestrationStatusJSON("attempt time", "Succeeded ", "current time"), result);
+}
 
 TEST_F(OrchestrationStatusTest, recoveryFields)
 {
@@ -482,3 +496,69 @@ TEST_F(OrchestrationStatusTest, setAllFields)
     EXPECT_EQ(i_orchestration_status->getServiceSettings(), service_map_a);
     EXPECT_EQ(i_orchestration_status->getRegistrationDetails(), agent_details);
 }
+
+TEST_F(OrchestrationStatusTest, checkErrorByRaiseEvent)
+{
+    init();
+    string fog_address = "http://fog.address";
+    string registar_error = "Fail to registar";
+    string manifest_error = "Fail to achieve manifest";
+    string last_update_error = "Fail to update";
+
+    EXPECT_CALL(time, getLocalTimeStr()).Times(3).WillRepeatedly(Return(string("Time")));
+
+    UpdatesProcessEvent(UpdatesProcessResult::SUCCESS, UpdatesConfigType::GENERAL).notify();
+    i_orchestration_status->setIsConfigurationUpdated(
+        EnumArray<OrchestrationStatusConfigType, bool>(true, true, true)
+    );
+    UpdatesProcessEvent(
+        UpdatesProcessResult::FAILED,
+        UpdatesConfigType::GENERAL,
+        UpdatesFailureReason::NONE,
+        "",
+        last_update_error
+    ).notify();
+    i_orchestration_status->setIsConfigurationUpdated(
+        EnumArray<OrchestrationStatusConfigType, bool>(false, false, false)
+    );
+
+    i_orchestration_status->setUpgradeMode("Online upgrades");
+    i_orchestration_status->setFogAddress(fog_address);
+
+    i_orchestration_status->setUpgradeMode("Online upgrades");
+    i_orchestration_status->setFogAddress(fog_address);
+
+    UpdatesProcessEvent(
+        UpdatesProcessResult::FAILED,
+        UpdatesConfigType::GENERAL,
+        UpdatesFailureReason::REGISTRATION,
+        "",
+        registar_error
+    ).notify();
+    UpdatesProcessEvent(
+        UpdatesProcessResult::FAILED,
+        UpdatesConfigType::MANIFEST,
+        UpdatesFailureReason::NONE,
+        "",
+        manifest_error
+    ).notify();
+    EXPECT_EQ(i_orchestration_status->getManifestError(), manifest_error);
+
+    auto result = orchestrationStatusFileToString();
+    EXPECT_EQ(
+        buildOrchestrationStatusJSON(
+            "None",
+            "Failed. Reason: " + last_update_error,
+            "Time",
+            "Time",
+            "",
+            "Time",
+            "Time",
+            "Online upgrades",
+            fog_address,
+            "Failed. Reason: Registration failed. Error: " + registar_error,
+            "Failed. Reason: " + manifest_error
+        ),
+        result
+    );
+}

components/security_apps/orchestration/modules/orchestration_status.cc

@@ -19,6 +19,8 @@
 
 #include "debug.h"
 #include "config.h"
+#include "updates_process_event.h"
+#include "health_check_status/health_check_status.h"
 
 using namespace cereal;
 using namespace std;
@@ -383,7 +385,10 @@ private:
     map<string, string> service_settings;
 };
 
-class OrchestrationStatus::Impl : Singleton::Provide<I_OrchestrationStatus>::From<OrchestrationStatus>
+class OrchestrationStatus::Impl
+        :
+    Singleton::Provide<I_OrchestrationStatus>::From<OrchestrationStatus>,
+    public Listener<UpdatesProcessEvent>
 {
 public:
     void
@@ -462,6 +467,13 @@ public:
             },
             "Write Orchestration status file"
         );
+        registerListener();
+    }
+
+    void
+    upon(const UpdatesProcessEvent &event) override
+    {
+        setFieldStatus(event.getStatusFieldType(), event.getOrchestrationStatusResult(), event.parseDescription());
     }
 
 private:

components/security_apps/orchestration/orchestration_comp.cc

@@ -42,6 +42,8 @@
 #include "hybrid_communication.h"
 #include "agent_core_utilities.h"
 #include "fog_communication.h"
+#include "updates_process_event.h"
+#include "updates_process_reporter.h"
 
 using namespace std;
 using namespace chrono;
@@ -53,85 +55,6 @@ USE_DEBUG_FLAG(D_ORCHESTRATOR);
 static string fw_last_update_time = "";
 #endif // gaia || smb
 
-class HealthCheckStatusListener : public Listener<HealthCheckStatusEvent>
-{
-public:
-    void upon(const HealthCheckStatusEvent &) override {}
-
-    HealthCheckStatusReply
-    respond(const HealthCheckStatusEvent &) override
-    {
-        return HealthCheckStatusReply(comp_name, status, extended_status);
-    }
-
-    string getListenerName() const override { return "HealthCheckStatusListener"; }
-
-    void
-    setStatus(
-        HealthCheckStatus _status,
-        OrchestrationStatusFieldType _status_field_type,
-        const string &_status_description = "Success")
-    {
-        string status_field_type_str = convertOrchestrationStatusFieldTypeToStr(_status_field_type);
-        extended_status[status_field_type_str] = _status_description;
-        field_types_status[status_field_type_str] = _status;
-
-        switch(_status) {
-            case HealthCheckStatus::UNHEALTHY: {
-                status = HealthCheckStatus::UNHEALTHY;
-                return;
-            }
-            case HealthCheckStatus::DEGRADED: {
-                for (const auto &type_status : field_types_status) {
-                    if ((type_status.first != status_field_type_str)
-                            && (type_status.second == HealthCheckStatus::UNHEALTHY))
-                    {
-                        return;
-                    }
-                }
-                status = HealthCheckStatus::DEGRADED;
-                return;
-            }
-            case HealthCheckStatus::HEALTHY: {
-                for (const auto &type_status : field_types_status) {
-                    if ((type_status.first != status_field_type_str)
-                            && (type_status.second == HealthCheckStatus::UNHEALTHY
-                                || type_status.second == HealthCheckStatus::DEGRADED)
-                        )
-                    {
-                        return;
-                    }
-                    status = HealthCheckStatus::HEALTHY;
-                }
-                return;
-            }
-            case HealthCheckStatus::IGNORED: {
-                return;
-            }
-        }
-    }
-
-private:
-    string
-    convertOrchestrationStatusFieldTypeToStr(OrchestrationStatusFieldType type)
-    {
-        switch (type) {
-            case OrchestrationStatusFieldType::REGISTRATION : return "Registration";
-            case OrchestrationStatusFieldType::MANIFEST : return "Manifest";
-            case OrchestrationStatusFieldType::LAST_UPDATE : return "Last Update";
-            case OrchestrationStatusFieldType::COUNT : return "Count";
-        }
-
-        dbgError(D_ORCHESTRATOR) << "Trying to convert unknown orchestration status field to string.";
-        return "";
-    }
-
-    string comp_name = "Orchestration";
-    HealthCheckStatus status = HealthCheckStatus::IGNORED;
-    map<string, string> extended_status;
-    map<string, HealthCheckStatus> field_types_status;
-};
-
 class SetAgentUninstall
         :
     public ServerRest,
@@ -203,7 +126,7 @@ public:
         loadFogAddress();
 
         Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->addOneTimeRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             [this] () { run(); },
             "Orchestration runner",
             true
@@ -257,6 +180,13 @@ private:
             << "Failed to load Orchestration Policy. Error: "
             << maybe_policy.getErr()
             <<  "Trying to load from backup.";
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::POLICY,
+            UpdatesFailureReason::POLICY_CONFIGURATION,
+            orchestration_policy_file,
+            maybe_policy.getErr()
+        ).notify();
         return loadOrchestrationPolicyFromBackup();
     }
 
@@ -280,6 +210,13 @@ private:
             return maybe_policy;
         }
 
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::POLICY,
+            UpdatesFailureReason::POLICY_CONFIGURATION,
+            orchestration_policy_file + backup_ext,
+            maybe_policy.getErr()
+        ).notify();
         return genError("Failed to load Orchestration policy from backup.");
     }
 
@@ -310,7 +247,7 @@ private:
         dbgInfo(D_ORCHESTRATOR) << "There is a new manifest file.";
         GetResourceFile resource_file(GetResourceFile::ResourceFileType::MANIFEST);
         Maybe<string> new_manifest_file =
-            Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
+            Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFile(
                 orch_manifest.unpack(),
                 I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
                 resource_file
@@ -337,17 +274,13 @@ private:
                 << new_manifest_file.getErr()
                 << " Presenting the next message to the user: "
                 << install_error;
-            i_orchestration_status->setFieldStatus(
-                OrchestrationStatusFieldType::MANIFEST,
-                OrchestrationStatusResult::FAILED,
-                install_error
-            );
-
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::UNHEALTHY,
-                OrchestrationStatusFieldType::MANIFEST,
-                install_error
-            );
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::MANIFEST,
+                UpdatesFailureReason::DOWNLOAD_FILE,
+                resource_file.getFileName(),
+                new_manifest_file.getErr()
+            ).notify();
 
             return genError(install_error);
         }
@@ -372,29 +305,17 @@ private:
                 << "Manifest failed to be updated. Presenting the next message to the user: "
                 << install_error;
 
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::UNHEALTHY,
-                OrchestrationStatusFieldType::MANIFEST,
-                install_error
-            );
-
             return genError(install_error);
         }
-
-        i_orchestration_status->setFieldStatus(
-                OrchestrationStatusFieldType::MANIFEST,
-                OrchestrationStatusResult::SUCCESS
-        );
-        health_check_status_listener.setStatus(
-            HealthCheckStatus::HEALTHY,
-            OrchestrationStatusFieldType::MANIFEST
-        );
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::MANIFEST
+        ).notify();
 
         ifstream restart_watchdog_orch(filesystem_prefix + "/orchestration/restart_watchdog");
         if (restart_watchdog_orch.good()) {
             ofstream restart_watchdog("/tmp/restart_watchdog", ofstream::out);
             restart_watchdog.close();
-            remove((filesystem_prefix + "/orchestration/restart_watchdog").c_str());
             restart_watchdog_orch.close();
         }
 
@@ -474,6 +395,13 @@ private:
             if (!updateFogAddress(policy.getFogAddress())) {
                 dbgWarning(D_ORCHESTRATOR) << "Failed to restore the old Fog address.";
             }
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::POLICY,
+                UpdatesFailureReason::POLICY_FOG_CONFIGURATION,
+                orchestration_policy.getFogAddress(),
+                "Failed to update the new Fog address."
+            ).notify();
             return "";
         }
 
@@ -500,13 +428,19 @@ private:
         // Handling policy update.
         dbgInfo(D_ORCHESTRATOR) << "There is a new policy file.";
         GetResourceFile resource_file(GetResourceFile::ResourceFileType::POLICY);
-        Maybe<string> new_policy_file =
-        Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
+        Maybe<string> new_policy_file = Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFile(
             new_policy.unpack(),
             I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
             resource_file
         );
         if (!new_policy_file.ok()) {
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::POLICY,
+                UpdatesFailureReason::DOWNLOAD_FILE,
+                resource_file.getFileName(),
+                new_policy_file.getErr()
+            ).notify();
             return genError("Failed to download the new policy file. Error: " + new_policy_file.getErr());
         }
 
@@ -565,6 +499,13 @@ private:
                 << LogField("policyVersion", updated_policy_version)
                 << LogField("previousPolicyVersion", old_policy_version);
 
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::POLICY,
+                UpdatesFailureReason::POLICY_CONFIGURATION,
+                updated_policy_version,
+                res.getErr()
+            ).notify();
             return genError(error_str);
         }
         i_service_controller->moveChangedPolicies();
@@ -649,6 +590,11 @@ private:
             "Send policy update report"
         );
 
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::POLICY
+        ).notify();
+
         dbgInfo(D_ORCHESTRATOR) << "Policy update report was successfully sent to fog";
 
         return Maybe<void>();
@@ -677,17 +623,31 @@ private:
             "Data file path"
         );
         GetResourceFile resource_file(GetResourceFile::ResourceFileType::DATA);
-        Maybe<string> new_data_files = Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
+        Maybe<string> new_data_files = Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFile(
             orch_data.unpack(),
             I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
             resource_file
         );
 
         if (!new_data_files.ok()) {
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::DATA,
+                UpdatesFailureReason::DOWNLOAD_FILE,
+                resource_file.getFileName(),
+                new_data_files.getErr()
+            ).notify();
             return genError("Failed to download new data file, Error: " + new_data_files.getErr());
         }
         auto new_data_file_input = i_orchestration_tools->readFile(new_data_files.unpack());
         if (!new_data_file_input.ok()) {
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::DATA,
+                UpdatesFailureReason::HANDLE_FILE,
+                resource_file.getFileName(),
+                "Failed to read new data file, Error: " + new_data_file_input.getErr()
+            ).notify();
             return genError("Failed to read new data file, Error: " + new_data_file_input.getErr());
         }
 
@@ -703,21 +663,35 @@ private:
                 << e.what()
                 << ". Content: "
                 << new_data_files.unpack();
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::DATA,
+                UpdatesFailureReason::HANDLE_FILE,
+                new_data_files.unpack(),
+                string("Failed to load data from JSON file, Error: ") + e.what()
+            ).notify();
             return genError(e.what());
         }
 
         for (const auto &data_file : parsed_data) {
             const string data_file_save_path = getPolicyConfigPath(data_file.first, Config::ConfigFileType::Data);
             Maybe<string> new_data_file =
-            Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromURL(
-                data_file.second.getDownloadPath(),
-                data_file.second.getChecksum(),
-                I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
-                "data_" + data_file.first
-            );
+                Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromURL(
+                    data_file.second.getDownloadPath(),
+                    data_file.second.getChecksum(),
+                    I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
+                    "data_" + data_file.first
+                );
 
             if (!new_data_file.ok()) {
                 dbgWarning(D_ORCHESTRATOR) << "Failed to download the " << data_file.first << " data file.";
+                UpdatesProcessEvent(
+                    UpdatesProcessResult::FAILED,
+                    UpdatesConfigType::DATA,
+                    UpdatesFailureReason::DOWNLOAD_FILE,
+                    data_file.first,
+                    new_data_file.getErr()
+                ).notify();
                 return new_data_file.passErr();
             }
             auto data_new_checksum = getChecksum(new_data_file.unpack());
@@ -730,6 +704,16 @@ private:
                     << data_new_checksum;
 
                 dbgWarning(D_ORCHESTRATOR) << current_error.str();
+                UpdatesProcessEvent(
+                    UpdatesProcessResult::FAILED,
+                    UpdatesConfigType::DATA,
+                    UpdatesFailureReason::CHECKSUM_UNMATCHED,
+                    data_file.first,
+                    " Expected checksum: " +
+                    data_file.second.getChecksum() +
+                    ". Downloaded checksum: " +
+                    data_new_checksum
+                ).notify();
                 return genError(current_error.str());
             }
             if (!i_orchestration_tools->copyFile(new_data_file.unpack(), data_file_save_path)) {
@@ -742,6 +726,10 @@ private:
             dbgWarning(D_ORCHESTRATOR) << "Failed to copy a new agents' data file to " << data_file_path;
         }
 
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::DATA
+        ).notify();
         return Maybe<void>();
     }
 
@@ -752,8 +740,7 @@ private:
 
         dbgInfo(D_ORCHESTRATOR) << "There is a new settings file.";
         GetResourceFile resource_file(GetResourceFile::ResourceFileType::SETTINGS);
-        Maybe<string> new_settings_file =
-        Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFileFromFog(
+        Maybe<string> new_settings_file = Singleton::Consume<I_Downloader>::by<OrchestrationComp>()->downloadFile(
             orch_settings.unpack(),
             I_OrchestrationTools::SELECTED_CHECKSUM_TYPE,
             resource_file
@@ -763,6 +750,13 @@ private:
             dbgWarning(D_ORCHESTRATOR)
                 << "Failed to download the new settings file. Error: "
                 << new_settings_file.getErr();
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::SETTINGS,
+                UpdatesFailureReason::DOWNLOAD_FILE,
+                resource_file.getFileName(),
+                new_settings_file.getErr()
+            ).notify();
             return genError("Failed to download the new settings file. Error: " + new_settings_file.getErr());
         }
 
@@ -770,6 +764,10 @@ private:
         if (res.ok()) {
             settings_file_path = *res;
             reloadConfiguration();
+            UpdatesProcessEvent(
+                UpdatesProcessResult::SUCCESS,
+                UpdatesConfigType::SETTINGS
+            ).notify();
             return Maybe<void>();
         }
 
@@ -878,11 +876,13 @@ private:
 
         if (!response.ok()) {
             dbgWarning(D_ORCHESTRATOR) << "Failed to get the update. Error: " << response.getErr();
-            i_orchestration_status->setFieldStatus(
-                OrchestrationStatusFieldType::LAST_UPDATE,
-                OrchestrationStatusResult::FAILED,
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::GENERAL,
+                UpdatesFailureReason::GET_UPDATE_REQUEST,
+                "",
                 "Warning: Agent/Gateway failed during the update process. Contact Check Point support."
-            );
+            ).notify();
 
             return genError(response.getErr());
         }
@@ -925,10 +925,10 @@ private:
         OrchSettings orch_settings  = response.getSettings();
         OrchData orch_data          = response.getData();
 
-        i_orchestration_status->setFieldStatus(
-            OrchestrationStatusFieldType::LAST_UPDATE,
-            OrchestrationStatusResult::SUCCESS
-        );
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::GENERAL
+        ).notify();
         i_orchestration_status->setIsConfigurationUpdated(
             EnumArray<OrchestrationStatusConfigType, bool>(
                 orch_manifest.ok(), orch_policy.ok(), orch_settings.ok(), orch_data.ok()
@@ -1018,6 +1018,10 @@ private:
         }
 
         if (maybe_errors != "") return genError(maybe_errors);
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::GENERAL
+        ).notify();
         return Maybe<void>();
     }
 
@@ -1197,6 +1201,13 @@ private:
         dbgTrace(D_ORCHESTRATOR) << "The settings directory is " << settings_file_path;
         if (!i_orchestration_tools->copyFile(new_settings_file, settings_file_path)) {
             dbgWarning(D_ORCHESTRATOR) << "Failed to update the settings.";
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::SETTINGS,
+                UpdatesFailureReason::HANDLE_FILE,
+                settings_file_path,
+                "Failed to update the settings"
+            ).notify();
             return genError("Failed to update the settings");
         }
 
@@ -1291,6 +1302,23 @@ private:
         }
     }
 
+    void
+    reportCloudMetadata(AgentDataReport &report)
+    {
+        I_DetailsResolver *i_details_resolver = Singleton::Consume<I_DetailsResolver>::by<OrchestrationComp>();
+        auto cloud_metadata = i_details_resolver->readCloudMetadata();
+        if (!cloud_metadata.ok()) {
+            dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
+            return;
+        }
+
+        report << make_pair("cloudAccountId", ::get<0>(cloud_metadata.unpack()));
+        report << make_pair("cloudVpcId", ::get<1>(cloud_metadata.unpack()));
+        report << make_pair("cloudInstanceId", ::get<2>(cloud_metadata.unpack()));
+        report << make_pair("cloudInstanceLocalIp", ::get<3>(cloud_metadata.unpack()));
+        report << make_pair("cloudRegion", ::get<4>(cloud_metadata.unpack()));
+    }
+
     void
     reportAgentDetailsMetaData()
     {
@@ -1330,6 +1358,14 @@ private:
             agent_data_report << AgentReportFieldWithLabel("reverse_proxy", "true");
         }
 
+        if (i_details_resolver->isCloudStorageEnabled()) {
+            agent_data_report << AgentReportFieldWithLabel("cloud_storage_service", "true");
+        } else {
+            agent_data_report << AgentReportFieldWithLabel("cloud_storage_service", "false");
+        }
+
+        reportCloudMetadata(agent_data_report);
+
         if (i_details_resolver->isKernelVersion3OrHigher()) {
             agent_data_report << AgentReportFieldWithLabel("isKernelVersion3OrHigher", "true");
         }
@@ -1417,24 +1453,28 @@ private:
             is_new_success = false;
             sleep_interval = calcSleepInterval(policy.getErrorSleepInterval());
             dbgWarning(D_ORCHESTRATOR)
-                << "Failed during check update from Fog. Error: "
+                << "Failed during check update. Error: "
                 << check_update_result.getErr()
                 << ", new check will be every: "
                 << sleep_interval << " seconds";
-
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::UNHEALTHY,
-                OrchestrationStatusFieldType::LAST_UPDATE,
-                "Failed during check update from Fog. Error: " + check_update_result.getErr()
-            );
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::GENERAL,
+                UpdatesFailureReason::CHECK_UPDATE,
+                "",
+                "Failed during check update. Error: " + check_update_result.getErr()
+            ).notify();
             return;
         }
         failure_count = 0;
         dbgDebug(D_ORCHESTRATOR) << "Check update process completed successfully";
-        health_check_status_listener.setStatus(
-            HealthCheckStatus::HEALTHY,
-            OrchestrationStatusFieldType::LAST_UPDATE
-        );
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::GENERAL,
+            UpdatesFailureReason::CHECK_UPDATE,
+            "",
+            "Check update procces succeeded!"
+        ).notify();
         sleep_interval = policy.getSleepInterval();
         if (!is_new_success) {
             dbgInfo(D_ORCHESTRATOR)
@@ -1469,11 +1509,13 @@ private:
         sleep_interval = policy.getErrorSleepInterval();
         Maybe<void> registration_status(genError("Not running yet."));
         while (!(registration_status = registerToTheFog()).ok()) {
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::UNHEALTHY,
-                OrchestrationStatusFieldType::REGISTRATION,
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::GENERAL,
+                UpdatesFailureReason::REGISTRATION,
+                "",
                 registration_status.getErr()
-            );
+            ).notify();
             sleep_interval = getConfigurationWithDefault<int>(
                 30,
                 "orchestration",
@@ -1493,10 +1535,11 @@ private:
 
         Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->yield(chrono::seconds(1));
 
-        health_check_status_listener.setStatus(
-            HealthCheckStatus::HEALTHY,
-            OrchestrationStatusFieldType::REGISTRATION
-        );
+        UpdatesProcessEvent(
+            UpdatesProcessResult::SUCCESS,
+            UpdatesConfigType::GENERAL,
+            UpdatesFailureReason::REGISTRATION
+        ).notify();
 
         LogGen(
             "Check Point Orchestration nano service successfully started",
@@ -1507,27 +1550,41 @@ private:
             << LogField("agentType", "Orchestration")
             << LogField("agentVersion", Version::get());
 
-        Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->addOneTimeRoutine(
+        auto mainloop = Singleton::Consume<I_MainLoop>::by<OrchestrationComp>();
+        mainloop->addOneTimeRoutine(
             I_MainLoop::RoutineType::Offline,
             sendRegistrationData,
             "Send registration data"
         );
 
+        if (getOrchestrationMode() == OrchestrationMode::HYBRID) {
+            Singleton::Consume<I_MainLoop>::by<OrchestrationComp>()->addRecurringRoutine(
+                I_MainLoop::RoutineType::Offline,
+                chrono::seconds(60),
+                [&] () {
+                    Singleton::Consume<I_UpdateCommunication>::by<OrchestrationComp>()->registerLocalAgentToFog();
+                },
+                "Check For Environment Registration Token"
+            );
+        }
+
         reportAgentDetailsMetaData();
 
         if (!Singleton::Consume<I_ManifestController>::by<OrchestrationComp>()->loadAfterSelfUpdate()) {
             // Should restore from backup
             dbgWarning(D_ORCHESTRATOR) << "Failed to load Orchestration after self-update";
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::UNHEALTHY,
-                OrchestrationStatusFieldType::LAST_UPDATE,
+            UpdatesProcessEvent(
+                UpdatesProcessResult::FAILED,
+                UpdatesConfigType::GENERAL,
+                UpdatesFailureReason::ORCHESTRATION_SELF_UPDATE,
+                "",
                 "Failed to load Orchestration after self-update"
-            );
+            ).notify();
         } else {
-            health_check_status_listener.setStatus(
-                HealthCheckStatus::HEALTHY,
-                OrchestrationStatusFieldType::MANIFEST
-            );
+            UpdatesProcessEvent(
+                UpdatesProcessResult::SUCCESS,
+                UpdatesConfigType::MANIFEST
+            ).notify();
         }
 
         setUpgradeTime();
@@ -1557,6 +1614,11 @@ private:
                 tags.insert(Tags::DEPLOYMENT_EMBEDDED);
                 break;
             }
+            case EnvType::DOCKER: {
+                tags.insert(Tags::DEPLOYMENT_DOCKER);
+                break;
+            }
+            case EnvType::NON_CRD_K8S:
             case EnvType::K8S: {
                 tags.insert(Tags::DEPLOYMENT_K8S);
                 break;
@@ -1760,7 +1822,7 @@ private:
         string orchestration_mode = getConfigurationFlag("orchestration-mode");
         if (
             orchestration_mode == "online_mode" ||
-            orchestration_mode == "hybrid_mode" ||
+            orchestration_mode  ==  "hybrid_mode" ||
             orchestration_mode == "offline_mode"
         ) {
             dbgTrace(D_ORCHESTRATOR) << "Orchestraion mode: " << orchestration_mode;
@@ -1872,7 +1934,7 @@ private:
             ReportIS::Audience::INTERNAL
         );
         hybrid_mode_metric.registerListener();
-        health_check_status_listener.registerListener();
+        updates_process_reporter_listener.registerListener();
     }
 
     void
@@ -1985,7 +2047,7 @@ private:
     unsigned int sleep_interval = 0;
     bool is_new_success = false;
     OrchestrationPolicy policy;
-    HealthCheckStatusListener health_check_status_listener;
+    UpdatesProcessReporter updates_process_reporter_listener;
     HybridModeMetric hybrid_mode_metric;
     EnvDetails env_details;
     chrono::minutes upgrade_delay_time;

components/security_apps/orchestration/orchestration_tools/orchestration_tools.cc

@@ -20,6 +20,7 @@
 #include "cereal/types/set.hpp"
 #include "agent_core_utilities.h"
 #include "namespace_data.h"
+#include "updates_process_event.h"
 
 #include <netdb.h>
 #include <arpa/inet.h>
@@ -469,6 +470,13 @@ OrchestrationTools::Impl::packagesToJsonFile(const map<packageName, Package> &pa
         archive_out(cereal::make_nvp("packages", packges_vector));
     } catch (cereal::Exception &e) {
         dbgDebug(D_ORCHESTRATOR) << "Failed to write vector of packages to JSON file " << path << ", " << e.what();
+        UpdatesProcessEvent(
+            UpdatesProcessResult::FAILED,
+            UpdatesConfigType::MANIFEST,
+            UpdatesFailureReason::HANDLE_FILE,
+            path,
+            string("Failed to write vector of packages to JSON file. Error: ") + e.what()
+        ).notify();
         return false;
     }
     return true;

components/security_apps/orchestration/orchestration_ut/orchestration_multitenant_ut.cc

@@ -39,6 +39,15 @@ Maybe<string> response(
 string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy";
 string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk";
 
+class ExpectInitializer
+{
+public:
+    ExpectInitializer(StrictMock<MockOrchestrationTools> &mock_orchestration_tools)
+    {
+        EXPECT_CALL(mock_orchestration_tools, doesFileExist("/.dockerenv")).WillRepeatedly(Return(false));
+    }
+};
+
 class OrchestrationMultitenancyTest : public Test
 {
 public:
@@ -79,7 +88,7 @@ public:
         // This Holding the Main Routine of the Orchestration.
         EXPECT_CALL(
             mock_ml,
-            addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true)
+            addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Orchestration runner", true)
         ).WillOnce(DoAll(SaveArg<1>(&routine), Return(1)));
 
         EXPECT_CALL(mock_shell_cmd, getExecOutput("openssl version -d | cut -d\" \" -f2 | cut -d\"\\\"\" -f2", _, _))
@@ -134,6 +143,9 @@ public:
 
         map<string, string> resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}});
         EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details));
+        EXPECT_CALL(mock_details_resolver, readCloudMetadata()).WillRepeatedly(
+            Return(Maybe<tuple<string, string, string, string, string>>(genError("No cloud metadata")))
+        );
     }
 
     void
@@ -200,6 +212,7 @@ public:
     NiceMock<MockAgenetDetailsReporter> mock_agent_reporter;
     NiceMock<MockLogging> mock_log;
 
+    ExpectInitializer initializer{mock_orchestration_tools};
     OrchestrationComp orchestration_comp;
 
 private:
@@ -271,7 +284,7 @@ TEST_F(OrchestrationMultitenancyTest, handle_virtual_resource)
     EXPECT_CALL(mock_service_controller, getPolicyVersion())
         .Times(3).WillRepeatedly(ReturnRef(first_policy_version));
 
-    map<string, PortNumber> empty_service_to_port_map;
+    map<string, vector<PortNumber>> empty_service_to_port_map;
     EXPECT_CALL(mock_service_controller, getServiceToPortMap()).WillRepeatedly(Return(empty_service_to_port_map));
 
 

components/security_apps/orchestration/orchestration_ut/orchestration_ut.cc

@@ -22,6 +22,8 @@
 #include "agent_details.h"
 #include "customized_cereal_map.h"
 #include "health_check_status/health_check_status.h"
+#include "updates_process_event.h"
+#include "declarative_policy_utils.h"
 
 using namespace testing;
 using namespace std;
@@ -41,6 +43,15 @@ Maybe<string> response(
 string orchestration_policy_file_path = "/etc/cp/conf/orchestration/orchestration.policy";
 string orchestration_policy_file_path_bk = orchestration_policy_file_path + ".bk";
 
+class ExpectInitializer
+{
+public:
+    ExpectInitializer(StrictMock<MockOrchestrationTools> &mock_orchestration_tools)
+    {
+        EXPECT_CALL(mock_orchestration_tools, doesFileExist("/.dockerenv")).WillRepeatedly(Return(false));
+    }
+};
+
 class OrchestrationTest : public testing::TestWithParam<bool>
 {
 public:
@@ -69,7 +80,7 @@ public:
         EXPECT_CALL(mock_orchestration_tools, setClusterId());
         EXPECT_CALL(
             mock_ml,
-            addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true)
+            addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Orchestration runner", true)
         ).WillOnce(DoAll(SaveArg<1>(&routine), Return(1)));
 
         EXPECT_CALL(
@@ -89,7 +100,7 @@ public:
             )
         );
 
-        map<string, PortNumber> empty_service_to_port_map;
+        map<string, vector<PortNumber>> empty_service_to_port_map;
         EXPECT_CALL(mock_service_controller, getServiceToPortMap()).WillRepeatedly(Return(empty_service_to_port_map));
 
         EXPECT_CALL(rest, mockRestCall(RestAction::SHOW, "orchestration-status", _)).WillOnce(
@@ -150,6 +161,7 @@ public:
         EXPECT_CALL(mock_details_resolver, getPlatform()).WillRepeatedly(Return(string("linux")));
         EXPECT_CALL(mock_details_resolver, getArch()).WillRepeatedly(Return(string("x86_64")));
         EXPECT_CALL(mock_details_resolver, isReverseProxy()).WillRepeatedly(Return(false));
+        EXPECT_CALL(mock_details_resolver, isCloudStorageEnabled()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isKernelVersion3OrHigher()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isGwNotVsx()).WillRepeatedly(Return(false));
         EXPECT_CALL(mock_details_resolver, isVersionAboveR8110()).WillRepeatedly(Return(false));
@@ -159,6 +171,9 @@ public:
 
         map<string, string> resolved_mgmt_details({{"kernel_version", "4.4.0-87-generic"}});
         EXPECT_CALL(mock_details_resolver, getResolvedDetails()).WillRepeatedly(Return(resolved_mgmt_details));
+        EXPECT_CALL(mock_details_resolver, readCloudMetadata()).WillRepeatedly(
+            Return(Maybe<tuple<string, string, string, string, string>>(genError("No cloud metadata")))
+        );
     }
 
     string
@@ -244,6 +259,17 @@ public:
         routine();
     }
 
+    void
+    runRegTokenRoutine()
+    {
+        EXPECT_CALL(
+            mock_ml,
+            addRecurringRoutine(I_MainLoop::RoutineType::Offline, _, _, "Check For Environment Registration Token", _)
+        ).WillOnce(DoAll(SaveArg<2>(&reg_token_routine), Return(0)));
+
+        routine();
+    }
+
     void
     runStatusRoutine()
     {
@@ -303,9 +329,12 @@ public:
     StrictMock<MockDetailsResolver> mock_details_resolver;
     NiceMock<MockAgenetDetailsReporter> mock_agent_reporter;
     NiceMock<MockTenantManager> tenant_manager;
+    ExpectInitializer initializer{mock_orchestration_tools};
     OrchestrationComp orchestration_comp;
+    DeclarativePolicyUtils declarative_policy_utils;
     AgentDetails agent_details;
     I_MainLoop::Routine sending_routine;
+    I_MainLoop::Routine reg_token_routine;
     string message_body;
 
 private:
@@ -326,6 +355,48 @@ private:
     I_MainLoop::Routine status_routine;
 };
 
+
+TEST_F(OrchestrationTest, hybridModeRegisterLocalAgentRoutine)
+{
+    EXPECT_CALL(rest, mockRestCall(_, _, _)).WillRepeatedly(Return(true));
+
+    Singleton::Consume<Config::I_Config>::from(config_comp)->loadConfiguration(
+        vector<string>{"--orchestration-mode=hybrid_mode"}
+    );
+
+    preload();
+    env.init();
+    init();
+
+    EXPECT_CALL(mock_service_controller, updateServiceConfiguration(_, _, _, _, _, _))
+        .WillOnce(Return(Maybe<void>()));
+    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(_, _)).WillRepeatedly(Return(string()));
+    EXPECT_CALL(mock_service_controller, getPolicyVersion()).WillRepeatedly(ReturnRef(first_policy_version));
+    EXPECT_CALL(mock_shell_cmd, getExecOutput(_, _, _)).WillRepeatedly(Return(string()));
+    EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe<void>()));
+    EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false));
+    expectDetailsResolver();
+    EXPECT_CALL(mock_update_communication, getUpdate(_));
+    EXPECT_CALL(mock_status, setLastUpdateAttempt());
+    EXPECT_CALL(mock_status, setIsConfigurationUpdated(_));
+
+    EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
+        .WillOnce(Return())
+        .WillOnce(Invoke([] (chrono::microseconds) { throw invalid_argument("stop while loop"); }));
+
+    EXPECT_CALL(
+        mock_ml,
+        addOneTimeRoutine(I_MainLoop::RoutineType::Offline, _, "Send registration data", false)
+    ).WillOnce(Return(1));
+
+    try {
+        runRegTokenRoutine();
+    } catch (const invalid_argument& e) {}
+
+    EXPECT_CALL(mock_update_communication, registerLocalAgentToFog());
+    reg_token_routine();
+}
+
 TEST_F(OrchestrationTest, testAgentUninstallRest)
 {
     EXPECT_CALL(
@@ -514,7 +585,6 @@ TEST_F(OrchestrationTest, check_sending_registration_data)
     expectDetailsResolver();
     EXPECT_CALL(mock_update_communication, getUpdate(_));
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(mock_status, setFieldStatus(_, _, _));
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(_));
 
     EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
@@ -671,7 +741,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback)
     GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY);
     EXPECT_CALL(
         mock_downloader,
-        downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file)
+        downloadFile(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file)
     ).WillOnce(Return(Maybe<std::string>(new_policy_path)));
 
     vector<string> expected_data_types = {};
@@ -691,10 +761,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdatRollback)
     ).WillOnce(Return(true));
     EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test"));
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -850,7 +916,7 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
     GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY);
     EXPECT_CALL(
         mock_downloader,
-        downloadFileFromFog(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file)
+        downloadFile(new_policy_checksum, Package::ChecksumTypes::SHA256, policy_file)
     ).WillOnce(Return(Maybe<std::string>(new_policy_path)));
 
     vector<string> expected_data_types = {};
@@ -870,10 +936,6 @@ TEST_F(OrchestrationTest, orchestrationPolicyUpdate)
     ).WillOnce(Return(true));
     EXPECT_CALL(mock_update_communication, setAddressExtenesion("/test"));
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -923,7 +985,7 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup)
 
     EXPECT_CALL(
         mock_ml,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Orchestration runner", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Orchestration runner", true)
     );
 
     EXPECT_CALL(
@@ -943,7 +1005,7 @@ TEST_F(OrchestrationTest, loadOrchestrationPolicyFromBackup)
         )
     );
 
-    map<string, PortNumber> empty_service_to_port_map;
+    map<string, vector<PortNumber>> empty_service_to_port_map;
     EXPECT_CALL(mock_service_controller, getServiceToPortMap()).WillRepeatedly(Return(empty_service_to_port_map));
 
     EXPECT_CALL(rest, mockRestCall(RestAction::SHOW, "orchestration-status", _));
@@ -1038,14 +1100,6 @@ TEST_F(OrchestrationTest, manifestUpdate)
     );
 
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::MANIFEST, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -1060,7 +1114,7 @@ TEST_F(OrchestrationTest, manifestUpdate)
 
     GetResourceFile manifest_file(GetResourceFile::ResourceFileType::MANIFEST);
     EXPECT_CALL(mock_downloader,
-        downloadFileFromFog(
+        downloadFile(
             string("new check sum"),
             Package::ChecksumTypes::SHA256,
             manifest_file
@@ -1147,7 +1201,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
     GetResourceFile policy_file(GetResourceFile::ResourceFileType::POLICY);
     EXPECT_CALL(
         mock_downloader,
-            downloadFileFromFog(
+            downloadFile(
             string("111111"),
             Package::ChecksumTypes::SHA256,
             policy_file
@@ -1167,10 +1221,6 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
         .WillOnce(ReturnRef(second_val)
     );
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -1201,7 +1251,7 @@ TEST_F(OrchestrationTest, getBadPolicyUpdate)
     EXPECT_CALL(
         mock_service_controller,
         updateServiceConfiguration(string("policy path"), "", expected_data_types, "", "", _)
-    ).WillOnce(Return(Maybe<void>(genError(string("")))));
+    ).WillOnce(Return(Maybe<void>(genError(string("Fail to load policy")))));
 
     EXPECT_CALL(mock_ml, yield(A<chrono::microseconds>()))
         .WillOnce(
@@ -1258,6 +1308,7 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
 
     EXPECT_CALL(mock_update_communication, authenticateAgent()).WillOnce(Return(Maybe<void>()));
     expectDetailsResolver();
+
     EXPECT_CALL(mock_manifest_controller, loadAfterSelfUpdate()).WillOnce(Return(false));
     EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::SHA256, manifest_file_path))
         .WillOnce(Return(manifest_checksum));
@@ -1289,22 +1340,10 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
     );
 
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    ).Times(1);
 
     string manifest_err =
         "Critical Error: Agent/Gateway was not fully deployed on host 'hostname' "
         "and is not enforcing a security policy. Retry installation or contact Check Point support.";
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(
-            OrchestrationStatusFieldType::MANIFEST,
-            OrchestrationStatusResult::FAILED,
-            manifest_err
-        )
-    ).Times(1);
     EXPECT_CALL(mock_status, getManifestError()).WillOnce(ReturnRef(manifest_err));
 
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
@@ -1324,14 +1363,14 @@ TEST_F(OrchestrationTest, failedDownloadSettings)
     GetResourceFile manifest_file(GetResourceFile::ResourceFileType::MANIFEST);
 
     EXPECT_CALL(mock_downloader,
-            downloadFileFromFog(
+            downloadFile(
             string("manifest-checksum"),
             Package::ChecksumTypes::SHA256,
             manifest_file
         )
     ).WillOnce(Return(download_error));
     EXPECT_CALL(mock_downloader,
-            downloadFileFromFog(
+            downloadFile(
             string("settings-checksum"),
             Package::ChecksumTypes::SHA256,
             settings_file
@@ -1405,10 +1444,6 @@ TEST_P(OrchestrationTest, orchestrationFirstRun)
         .WillOnce(Return(data_checksum));
 
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -1467,23 +1502,6 @@ TEST_P(OrchestrationTest, orchestrationFirstRun)
     } catch (const invalid_argument& e) {}
     EXPECT_CALL(mock_status, writeStatusToFile());
 
-    vector<HealthCheckStatusReply> reply;
-    bool is_named_query = GetParam();
-    if (is_named_query) {
-        auto all_comps_status_reply = HealthCheckStatusEvent().performNamedQuery();
-        for (auto &elem : all_comps_status_reply) {
-            reply.push_back(elem.second);
-        }
-    } else {
-        reply = HealthCheckStatusEvent().query();
-    }
-
-    ASSERT_EQ(reply.size(), 1);
-    EXPECT_EQ(reply[0].getCompName(), "Orchestration");
-    EXPECT_EQ(reply[0].getStatus(), HealthCheckStatus::HEALTHY);
-
-    HealthCheckStatusEvent().notify();
-
     orchestration_comp.fini();
 }
 
@@ -1651,10 +1669,6 @@ TEST_F(OrchestrationTest, dataUpdate)
     );
 
     EXPECT_CALL(mock_status, setLastUpdateAttempt());
-    EXPECT_CALL(
-        mock_status,
-        setFieldStatus(OrchestrationStatusFieldType::LAST_UPDATE, OrchestrationStatusResult::SUCCESS, "")
-    );
     EXPECT_CALL(mock_status, setIsConfigurationUpdated(A<EnumArray<OrchestrationStatusConfigType, bool>>())
     ).WillOnce(
         Invoke(
@@ -1690,7 +1704,7 @@ TEST_F(OrchestrationTest, dataUpdate)
     string new_data_file_path = data_file_path + ".download";
     GetResourceFile data_file(GetResourceFile::ResourceFileType::DATA);
     EXPECT_CALL(mock_downloader,
-        downloadFileFromFog(
+        downloadFile(
             string("new data"),
             Package::ChecksumTypes::SHA256,
             data_file

components/security_apps/orchestration/service_controller/service_controller.cc

@@ -333,7 +333,7 @@ private:
 
     ReconfStatus getUpdatedReconfStatus();
     Maybe<ServiceDetails> getServiceDetails(const string &service_name);
-    map<string, PortNumber> getServiceToPortMap();
+    map<string, vector<PortNumber>> getServiceToPortMap();
 
     template<class Archive>
     void serializeRegisterServices(Archive &ar) { ar(pending_services); }
@@ -358,6 +358,7 @@ private:
     string filesystem_prefix;
     bool is_multi_tenant_env = false;
     set<string> changed_policy_files;
+    ServiceDetails orchestration_service_details;
 
     I_OrchestrationTools *orchestration_tools = nullptr;
     I_MainLoop *mainloop = nullptr;
@@ -374,8 +375,13 @@ public:
         for (auto const& entry: ports_map) {
             string service = entry.first;
             replace(service.begin(), service.end(), ' ', '-');
-            output << service << ":";
-            output << entry.second << ",";
+            output << service;
+            char delim = ':';
+            for (PortNumber port : entry.second) {
+                output << delim << port;
+                delim  = ',';
+            }
+            output << ";";
         }
         ports_list = output.str();
     }
@@ -500,6 +506,7 @@ ServiceController::Impl::loadRegisteredServicesFromFile()
     stringstream ss(maybe_registered_services_str.unpack());
     cereal::JSONInputArchive ar(ss);
     ar(cereal::make_nvp("Registered Services", pending_services));
+    pending_services.erase("cp-nano-orchestration");
 
     dbgInfo(D_SERVICE_CONTROLLER)
         << "Orchestration pending services loaded from file."
@@ -529,16 +536,24 @@ ServiceController::Impl::writeRegisteredServicesToFile()
         "Orchestration registered services"
     );
 
+    map<string, ServiceDetails> registered_services_with_orch = registered_services;
+    if (orchestration_service_details.getServiceID() != "") {
+        registered_services_with_orch.emplace(
+            orchestration_service_details.getServiceID(),
+            orchestration_service_details
+        );
+    }
+
     ofstream ss(registered_services_file);
     cereal::JSONOutputArchive ar(ss);
-    ar(cereal::make_nvp("Registered Services", registered_services));
+    ar(cereal::make_nvp("Registered Services", registered_services_with_orch));
 
     dbgInfo(D_SERVICE_CONTROLLER)
         << "Orchestration registered services file has been updated. File: "
         << registered_services_file
         << ". Registered Services:";
 
-    for (const auto &id_service_pair : registered_services) {
+    for (const auto &id_service_pair : registered_services_with_orch) {
         const auto &service = id_service_pair.second;
         dbgInfo(D_SERVICE_CONTROLLER)
             << "Service name: "
@@ -591,20 +606,20 @@ ServiceController::Impl::cleanUpVirtualFiles()
     }
 }
 
-map<string, PortNumber>
+map<string, vector<PortNumber>>
 ServiceController::Impl::getServiceToPortMap()
 {
-    map<string, PortNumber> ports_map;
+    map<string, vector<PortNumber>> ports_map;
     for (auto const& entry: registered_services) {
-        const string &service = entry.first;
+        const string &service = entry.second.getServiceName();
         PortNumber port = entry.second.getPort();
-        ports_map[service] = port;
+        ports_map[service].push_back(port);
     }
 
     for (auto const& entry: pending_services) {
-        const string &service = entry.first;
+        const string &service = entry.second.getServiceName();
         PortNumber port = entry.second.getPort();
-        ports_map[service] = port;
+        ports_map[service].push_back(port);
     }
 
     return ports_map;
@@ -624,6 +639,12 @@ ServiceController::Impl::registerServiceConfig(
         service_id
     );
 
+    if (service_name == "cp-nano-orchestration") {
+        dbgTrace(D_SERVICE_CONTROLLER) << "Save the orchestration service details";
+        orchestration_service_details = service_config;
+        return;
+    }
+
     pending_services.erase(service_config.getServiceID());
     pending_services.insert({service_config.getServiceID(), service_config});
     refreshPendingServices();
@@ -702,26 +723,6 @@ ServiceController::Impl::createDirectoryForChildTenant(
     return true;
 }
 
-static string
-getChecksum(const string &file_path)
-{
-    auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<ServiceController>();
-    Maybe<string> file_checksum = orchestration_tools->calculateChecksum(
-        Package::ChecksumTypes::MD5,
-        file_path
-    );
-
-    if (file_checksum.ok()) return file_checksum.unpack();
-
-    string checksum = "unknown version";
-    try {
-        checksum = to_string(boost::uuids::random_generator()());
-    } catch (const boost::uuids::entropy_error &e) {
-        dbgDebug(D_SERVICE_CONTROLLER) << "Couldn't generate random checksum";
-    }
-    return checksum;
-}
-
 Maybe<void>
 ServiceController::Impl::updateServiceConfiguration(
     const string &new_policy_path,
@@ -827,7 +828,7 @@ ServiceController::Impl::updateServiceConfiguration(
         if (child_tenant_id.empty() && single_policy.first == versions_param) {
             //In a multi-tenant env, only the parent should handle the versions parameter
             policy_versions = single_policy.second;
-            dbgWarning(D_SERVICE_CONTROLLER) << "Found versions parameter in policy file:" << policy_versions;
+            dbgTrace(D_SERVICE_CONTROLLER) << "Found versions parameter in policy file:" << policy_versions;
         }
 
         dbgDebug(D_SERVICE_CONTROLLER) << "Starting to update policy file. Policy type: " << single_policy.first;
@@ -901,7 +902,7 @@ ServiceController::Impl::updateServiceConfiguration(
     // In a multi-tenant env, we send the signal to the services only on the last iteration
     if (!is_multi_tenant_env || last_iteration) {
         auto is_send_signal_for_services =
-            sendSignalForServices(nano_services_to_update, version_value + ',' + getChecksum(new_policy_path));
+            sendSignalForServices(nano_services_to_update, version_value + ',' + policy_versions);
         was_policy_updated &= is_send_signal_for_services.ok();
         if (!is_send_signal_for_services.ok()) send_signal_for_services_err = is_send_signal_for_services.getErr();
     }
@@ -950,7 +951,7 @@ ServiceController::Impl::sendSignalForServices(
     const set<string> &nano_services_to_update,
     const string &policy_version_to_update)
 {
-    dbgFlow(D_SERVICE_CONTROLLER);
+    dbgFlow(D_SERVICE_CONTROLLER) << "Policy version to update: " << policy_version_to_update;
     for (auto &service_id : nano_services_to_update) {
         auto nano_service = registered_services.find(service_id);
         if (nano_service == registered_services.end()) {

components/security_apps/orchestration/service_controller/service_controller_ut/service_controller_ut.cc

@@ -98,9 +98,12 @@ public:
             "orchestration",
             "Settings file path"
         );
+    }
 
+    void
+    init()
+    {
         service_controller.init();
-
         registerNewService();
     }
 
@@ -173,18 +176,19 @@ public:
     }
 
     void
-    expectNewConfigRequest(const string &req_body, const string &response)
+    expectNewConfigRequest(const string &response)
     {
+        Maybe<HTTPResponse, HTTPResponse> res = HTTPResponse(HTTPStatusCode::HTTP_OK, response);
         EXPECT_CALL(
             mock_message,
             sendSyncMessage(
                 HTTPMethod::POST,
                 "/set-new-configuration",
-                req_body,
+                _,
                 _,
                 _
             )
-        ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response)));
+        ).WillOnce(DoAll(SaveArg<2>(&version_body), Return(res)));
     }
 
     CPTestTempfile                      status_file;
@@ -193,6 +197,7 @@ public:
     ::Environment                       env;
     ConfigComponent                     config;
     DeclarativePolicyUtils              declarative_policy_utils;
+    string version_body;
     string                              configuration_dir;
     string                              policy_extension;
     string                              settings_extension;
@@ -224,16 +229,39 @@ public:
     ostringstream                       capture_debug;
     string                              version_value = "1.0.2";
     string                              old_version = "1.0.1";
+
+    string versions =
+            "[\n"
+            "    {\n"
+            "        \"id\": \"d8c3cc3c-f9df-83c8-f875-322dd8a0c161\",\n"
+            "        \"name\": \"Linux Embedded Agents\",\n"
+            "        \"version\": \"1.0.2\",\n"
+            "        \"profileType\": \"Embedded\"\n"
+            "    }\n"
+            "]";
+    string old_versions =
+            "["
+            "    {"
+            "        \"id\": \"d8c3cc3c-f9df-83c8-f875-322dd8a0c161\","
+            "        \"name\": \"Linux Embedded Agents\","
+            "        \"version\": \"1.0.1\","
+            "        \"profileType\": \"Embedded\""
+            "    }"
+            "]";
+
 };
 
 TEST_F(ServiceControllerTest, doNothing)
 {
+    init();
 }
 
 TEST_F(ServiceControllerTest, UpdateConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -271,7 +299,13 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -284,9 +318,6 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
     EXPECT_EQ(i_service_controller->getPolicyVersions(), "");
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -295,7 +326,7 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
     string general_settings_path = "/my/settings/path";
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
 
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_CALL(
         mock_shell_cmd,
@@ -309,25 +340,30 @@ TEST_F(ServiceControllerTest, UpdateConfiguration)
 
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok());
     EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
-    EXPECT_EQ(i_service_controller->getPolicyVersions(), "");
+    EXPECT_EQ(i_service_controller->getPolicyVersions(), versions);
     EXPECT_EQ(i_service_controller->getUpdatePolicyVersion(), version_value);
+
+    stringstream ver_ss;
+    ver_ss
+        << "{\n"
+        << "    \"id\": 1,\n"
+        << "    \"policy_version\": \"1.0.2,[\\n"
+        << "    {\\n"
+        << "        \\\"id\\\": \\\"d8c3cc3c-f9df-83c8-f875-322dd8a0c161\\\",\\n"
+        << "        \\\"name\\\": \\\"Linux Embedded Agents\\\",\\n"
+        << "        \\\"version\\\": \\\"1.0.2\\\",\\n"
+        << "        \\\"profileType\\\": \\\"Embedded\\\"\\n"
+        << "    }\\n"
+        << "]\"\n}";
+    EXPECT_EQ(
+        version_body,
+        ver_ss.str()
+    );
 }
 
 TEST_F(ServiceControllerTest, supportVersions)
 {
-    string versions = "["
-                    "  {"
-                    "    \"id\" : \"40c4a460-eb24-f002-decb-f4a7f00423fc\","
-                    "    \"name\" : \"Linux Embedded Agents\","
-                    "    \"version\" : 1"
-                    "  },"
-                    "  {"
-                    "    \"id\" : \"93788960-6969-11ee-be56-0242ac120002\","
-                    "    \"name\" : \"Linux SUPER Embedded Agents\","
-                    "    \"version\" : 420"
-                    "  }"
-                    "]";
-
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
                                 "   \"versions\": " + versions +
@@ -386,9 +422,6 @@ TEST_F(ServiceControllerTest, supportVersions)
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
     EXPECT_EQ(i_service_controller->getPolicyVersions(), "");
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -397,7 +430,7 @@ TEST_F(ServiceControllerTest, supportVersions)
     string general_settings_path = "/my/settings/path";
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
 
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_CALL(
         mock_shell_cmd,
@@ -417,8 +450,10 @@ TEST_F(ServiceControllerTest, supportVersions)
 
 TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -456,7 +491,13 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -468,9 +509,6 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
 
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -500,7 +538,7 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
 
     string general_settings_path = "/my/settings/path";
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok());
     EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
@@ -509,13 +547,14 @@ TEST_F(ServiceControllerTest, TimeOutUpdateConfiguration)
 
 TEST_F(ServiceControllerTest, readRegisteredServicesFromFile)
 {
-    int family1_id3_port = 1111;
+    init();
+    uint16_t family1_id3_port = 1111;
     string registered_services_json =  "{\n"
                             "    \"Registered Services\": {\n"
                             "        \"family1_id3\": {\n"
                             "            \"Service name\": \"mock access control\",\n"
                             "            \"Service ID\": \"family1_id3\",\n"
-                            "            \"Service port\": 1111,\n"
+                            "            \"Service port\": " + to_string(family1_id3_port) + ",\n"
                             "            \"Relevant configs\": [\n"
                             "                \"non updated capability\",\n"
                             "                \"l4_firewall\"\n"
@@ -555,13 +594,16 @@ TEST_F(ServiceControllerTest, readRegisteredServicesFromFile)
     service_controller.init();
 
     auto services_to_port_map = i_service_controller->getServiceToPortMap();
-    EXPECT_EQ(services_to_port_map.find("family1_id3")->second, family1_id3_port);
+    vector<PortNumber> ports = {l4_firewall_service_port, family1_id3_port};
+    EXPECT_EQ(services_to_port_map.find("mock access control")->second, ports);
 }
 
 TEST_F(ServiceControllerTest, noPolicyUpdate)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -599,7 +641,13 @@ TEST_F(ServiceControllerTest, noPolicyUpdate)
                             "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -612,11 +660,8 @@ TEST_F(ServiceControllerTest, noPolicyUpdate)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_CALL(
         mock_shell_cmd,
@@ -634,8 +679,10 @@ TEST_F(ServiceControllerTest, noPolicyUpdate)
 
 TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -673,7 +720,12 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -685,9 +737,6 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
 
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -705,7 +754,7 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
 
     string general_settings_path = "/my/settings/path";
     string reply_msg1 = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg1);
+    expectNewConfigRequest(reply_msg1);
 
     // both policy and settings now being updated
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok());
@@ -725,13 +774,15 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
 
     general_settings_path += "/new";
 
     string reply_msg2 = "{\"id\": 2, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 2,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg2);
+    expectNewConfigRequest(reply_msg2);
 
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, general_settings_path).ok());
     EXPECT_EQ(i_service_controller->getPolicyVersion(), version_value);
@@ -739,8 +790,10 @@ TEST_F(ServiceControllerTest, SettingsAndPolicyUpdateCombinations)
 
 TEST_F(ServiceControllerTest, backup)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -779,6 +832,7 @@ TEST_F(ServiceControllerTest, backup)
 
     string old_configuration =  "{"
                                 "   \"version\": \"" + old_version + "\""
+                                "   \"versions\": \"" + old_versions + "\""
                                 "    \"app\": \"netfilter\","
                                 "    \"l4_firewall_rules\": ["
                                 "        {"
@@ -796,7 +850,13 @@ TEST_F(ServiceControllerTest, backup)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -805,9 +865,6 @@ TEST_F(ServiceControllerTest, backup)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(
         mock_orchestration_tools,
         copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
@@ -842,8 +899,10 @@ TEST_F(ServiceControllerTest, backup)
 
 TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -882,6 +941,7 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
 
     string old_configuration =  "{"
                                 "   \"version\": \"" + old_version + "\""
+                                "   \"versions\": \"" + old_versions + "\""
                                 "    \"app\": \"netfilter\","
                                 "    \"l4_firewall_rules\": ["
                                 "        {"
@@ -899,7 +959,14 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -908,9 +975,6 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(
         mock_orchestration_tools,
         copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
@@ -937,7 +1001,7 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
     ).WillRepeatedly(Return(string("registered and running")));
 
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok());
@@ -946,8 +1010,10 @@ TEST_F(ServiceControllerTest, backup_file_doesnt_exist)
 
 TEST_F(ServiceControllerTest, backupAttempts)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -986,6 +1052,7 @@ TEST_F(ServiceControllerTest, backupAttempts)
 
     string old_configuration =  "{"
                                 "   \"version\": \"" + old_version + "\""
+                                "   \"versions\": \"" + old_versions + "\""
                                 "    \"app\": \"netfilter\","
                                 "    \"l4_firewall_rules\": ["
                                 "        {"
@@ -1003,7 +1070,14 @@ TEST_F(ServiceControllerTest, backupAttempts)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -1012,9 +1086,6 @@ TEST_F(ServiceControllerTest, backupAttempts)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(
         mock_orchestration_tools,
         copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
@@ -1041,7 +1112,7 @@ TEST_F(ServiceControllerTest, backupAttempts)
     ).WillRepeatedly(Return(string("registered and running")));
 
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_CALL(mock_ml, yield(false)).Times(2);
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -1053,8 +1124,10 @@ TEST_F(ServiceControllerTest, backupAttempts)
 
 TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -1110,8 +1183,16 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
     Maybe<map<string, string>> json_parser_return = map<string, string>({
         {"version", version_value},
         {"l4_firewall", l4_firewall},
-        {"orchestration", orchestration}
+        {"orchestration", orchestration},
+        {"versions", versions}
     });
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     string orchestration_policy_path =  configuration_dir + "/orchestration/orchestration" + policy_extension;
     string orchestration_settings_path = configuration_dir + "/orchestration/orchestration" + settings_extension;
 
@@ -1125,9 +1206,6 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("orchestration", orchestration_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path, false))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, writeFile(orchestration, orchestration_policy_path, false))
@@ -1148,12 +1226,13 @@ TEST_F(ServiceControllerTest, MultiUpdateConfiguration)
     ).WillRepeatedly(Return(string("registered and running")));
 
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok());
     set<string> changed_policies = {
         "/etc/cp/conf/l4_firewall/l4_firewall.policy",
-        "/etc/cp/conf/orchestration/orchestration.policy"
+        "/etc/cp/conf/orchestration/orchestration.policy",
+        policy_versions_path
     };
     EXPECT_EQ(i_service_controller->moveChangedPolicies(), changed_policies);
 }
@@ -1166,6 +1245,7 @@ public:
 
 TEST_F(ServiceControllerTest, badJsonFile)
 {
+    init();
     Maybe<string> err = genError("Error");
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(err));
     EXPECT_FALSE(i_service_controller->updateServiceConfiguration(file_name, "").ok());
@@ -1173,6 +1253,7 @@ TEST_F(ServiceControllerTest, badJsonFile)
 
 TEST_F(ServiceControllerTest, emptyServices)
 {
+    init();
     Maybe<map<string, string>> json_parser_return = map<string, string>();
     string empty_string = "";
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(empty_string));
@@ -1185,16 +1266,15 @@ TEST_F(ServiceControllerTest, emptyServices)
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_file_path)).WillOnce(Return(true));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_TRUE(i_service_controller->updateServiceConfiguration(file_name, "").ok());
 }
 
 TEST_F(ServiceControllerTest, failingWhileLoadingCurrentConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -1232,23 +1312,31 @@ TEST_F(ServiceControllerTest, failingWhileLoadingCurrentConfiguration)
                                 "}";
 
     Maybe<string> err = genError("Error");
+
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
     EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(err));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
     EXPECT_FALSE(i_service_controller->updateServiceConfiguration(file_name, "").ok());
 }
 
 TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -1287,6 +1375,7 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration)
 
     string old_configuration =  "{"
                                 "   \"version\": \"" + old_version + "\""
+                                "   \"versions\": \"" + old_versions + "\""
                                 "    \"app\": \"netfilter\","
                                 "    \"l4_firewall_rules\": ["
                                 "        {"
@@ -1304,15 +1393,20 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly(
         Return(json_parser_return)
     );
     EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration));
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
     EXPECT_CALL(
         mock_orchestration_tools,
         copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
@@ -1325,10 +1419,12 @@ TEST_F(ServiceControllerTest, failingWhileCopyingCurrentConfiguration)
 
 TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
 {
+    init();
     Debug::setUnitTestFlag(D_SERVICE_CONTROLLER, Debug::DebugLevel::NOISE);
     Debug::setNewDefaultStdout(&capture_debug);
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -1368,7 +1464,14 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
     EXPECT_CALL(time, getWalltime()).WillRepeatedly(Return(chrono::microseconds(0)));
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -1384,9 +1487,6 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY)
     );
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_EQ(i_service_controller->getPolicyVersion(), "");
 
     EXPECT_TRUE(i_service_controller->isServiceInstalled("family1_id2"));
@@ -1417,8 +1517,10 @@ TEST_F(ServiceControllerTest, ErrorUpdateConfigurationRest)
 
 TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration)
 {
+    init();
     string new_configuration =  "{"
                                 "   \"version\": \"" + version_value + "\""
+                                "   \"versions\": \"" + versions + "\""
                                 "   \"l4_firewall\":"
                                 "       {"
                                 "           \"app\": \"netfilter\","
@@ -1457,6 +1559,7 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration)
 
     string old_configuration =  "{"
                                 "   \"version\": \"" + old_version + "\""
+                                "   \"versions\": \"" + old_versions + "\""
                                 "    \"app\": \"netfilter\","
                                 "    \"l4_firewall_rules\": ["
                                 "        {"
@@ -1474,15 +1577,20 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration)
                                 "}";
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).Times(1).WillRepeatedly(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _)).Times(1).WillRepeatedly(
         Return(json_parser_return)
     );
     EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, readFile(l4_firewall_policy_path)).WillOnce(Return(old_configuration));
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
     EXPECT_CALL(
         mock_orchestration_tools,
         copyFile(l4_firewall_policy_path, l4_firewall_policy_path + backup_extension)
@@ -1498,16 +1606,22 @@ TEST_F(ServiceControllerTest, errorWhileWrtingNewConfiguration)
 
 TEST_F(ServiceControllerTest, testPortsRest)
 {
+    init();
     stringstream empty_json;
     empty_json << "{}";
     auto res = get_services_ports->performRestCall(empty_json);
     ASSERT_TRUE(res.ok());
-    EXPECT_THAT(res.unpack(), HasSubstr("family1_id2:8888"));
+    EXPECT_THAT(res.unpack(), HasSubstr("mock-access-control:8888;"));
 }
 
 TEST_F(ServiceControllerTest, testMultitenantConfFiles)
 {
+    setSetting<string>("VirtualNSaaS", "agentType");
+    init();
+
     map<pair<string, string>, pair<string, string>> tenant_files_input = {
+        {make_pair("", ""),
+        make_pair("/etc/cp/conf/policy.json", "")},
         {make_pair("tenant1", "1234"),
         make_pair("/etc/cp/conf/tenant1_profile_1234_policy.json", "/etc/cp/conf/tenant1_profile_1234_settings.json")},
         {make_pair("tenant2", "1235"),
@@ -1517,11 +1631,11 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
     set<string> ids = {"family1_id2"};
     set<string> empty_ids;
 
-    EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillOnce(Return(ids));
-    EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillOnce(Return(empty_ids));
+    EXPECT_CALL(tenant_manager, getInstances("tenant1", "1234")).WillRepeatedly(Return(ids));
+    EXPECT_CALL(tenant_manager, getInstances("tenant2", "1235")).WillRepeatedly(Return(empty_ids));
 
     string reply_msg = "{\"id\": 1, \"error\": false, \"finished\": true, \"error_message\": \"\"}";
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     for(auto entry : tenant_files_input) {
         auto tenant = entry.first.first;
@@ -1532,6 +1646,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
 
         string new_configuration =  "{"
                                     "   \"version\": \"" + version_value + "\""
+                                    "   \"versions\": \"" + versions + "\""
                                     "   \"l4_firewall\":"
                                     "       {"
                                     "           \"app\": \"netfilter\","
@@ -1568,44 +1683,72 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
                                     "    ]"
                                     "}";
 
-        string l4_firewall_policy_path_new =
-            configuration_dir + "/tenant_" + tenant +
-            "_profile_" + profile +"/l4_firewall/l4_firewall" + policy_extension;
         Maybe<map<string, string>> json_parser_return =
-            map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
-
+            map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
         EXPECT_CALL(mock_orchestration_tools, readFile(conf_file_name)).WillOnce(Return(new_configuration));
-
         EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, tenant, profile))
             .WillOnce(Return(json_parser_return));
+        if (!tenant.empty()) {
+            string l4_firewall_policy_path_new =
+                configuration_dir + "/tenant_" + tenant +
+                "_profile_" + profile +"/l4_firewall/l4_firewall" + policy_extension;
+            string policy_versions_path_new =
+                configuration_dir + "/tenant_" + tenant +
+                "_profile_" + profile +"/versions/versions" + policy_extension;
 
-        EXPECT_CALL(
-            mock_orchestration_tools,
-            doesDirectoryExist(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
-        ).WillOnce(Return(false));
+            EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path_new)).WillOnce(Return(false));
+            EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path_new, false))
+                .WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_status,
+                setServiceConfiguration("versions", policy_versions_path_new, OrchestrationStatusConfigType::POLICY));
 
-        EXPECT_CALL(
-            mock_orchestration_tools,
-            createDirectory(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
-        ).WillOnce(Return(true));
+            EXPECT_CALL(
+                mock_orchestration_tools,
+                doesDirectoryExist(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
+            ).WillOnce(Return(false)).WillOnce(Return(true));
 
-        EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path_new)).WillOnce(Return(false));
+            EXPECT_CALL(
+                mock_orchestration_tools,
+                createDirectory(configuration_dir + "/tenant_" + tenant + "_profile_" + profile)
+            ).WillOnce(Return(true));
 
-        EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path_new, false))
-            .WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path_new)).WillOnce(Return(false));
 
-        EXPECT_CALL(mock_orchestration_status, setServiceConfiguration(
-            "l4_firewall", l4_firewall_policy_path_new, OrchestrationStatusConfigType::POLICY)
-        );
+            EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path_new, false))
+                .WillOnce(Return(true));
 
-        EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, conf_file_name))
-            .WillRepeatedly(Return(version_value));
+            string new_policy_file_path =
+                "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json";
+            EXPECT_CALL(
+                mock_orchestration_tools,
+                copyFile(new_policy_file_path, new_policy_file_path + backup_extension)
+            ).WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path))
+                .WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true));
 
-        string new_policy_file_path = "/etc/cp/conf/tenant_" + tenant + "_profile_" + profile + "/" + "policy.json";
-        EXPECT_CALL(mock_orchestration_tools, copyFile(new_policy_file_path, new_policy_file_path + backup_extension))
-            .WillOnce(Return(true));
-        EXPECT_CALL(mock_orchestration_tools, copyFile(conf_file_name, new_policy_file_path)).WillOnce(Return(true));
-        EXPECT_CALL(mock_orchestration_tools, doesFileExist(new_policy_file_path)).WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_status, setServiceConfiguration(
+                "l4_firewall", l4_firewall_policy_path_new, OrchestrationStatusConfigType::POLICY)
+            );
+        } else {
+            EXPECT_CALL(mock_orchestration_tools, doesFileExist(l4_firewall_policy_path)).WillOnce(Return(false));
+            EXPECT_CALL(mock_orchestration_tools, writeFile(l4_firewall, l4_firewall_policy_path, false)).
+                WillOnce(Return(true));
+            EXPECT_CALL(
+                mock_orchestration_status,
+                setServiceConfiguration(
+                    "l4_firewall",
+                    l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY
+                )
+            );
+
+            string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+            EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+            EXPECT_CALL(
+                mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+            EXPECT_CALL(mock_orchestration_status,
+                setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+        }
 
         EXPECT_CALL(
             mock_shell_cmd,
@@ -1623,7 +1766,8 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
                 settings_file_name,
                 {},
                 tenant,
-                profile
+                profile,
+                tenant.empty()
             ).ok()
         );
     }
@@ -1631,6 +1775,7 @@ TEST_F(ServiceControllerTest, testMultitenantConfFiles)
 
 TEST_F(ServiceControllerTest, cleanup_virtual_files)
 {
+    init();
     string agent_tenants_files =
         "111111\n"
         "222222\n"
@@ -1653,9 +1798,11 @@ TEST_F(ServiceControllerTest, cleanup_virtual_files)
 
 TEST_F(ServiceControllerTest, test_delayed_reconf)
 {
+    init();
     string new_configuration =
         "{"
         "   \"version\": \"" + version_value + "\""
+        "   \"versions\": " + versions +
         "   \"l4_firewall\":"
         "       {"
         "           \"app\": \"netfilter\","
@@ -1696,7 +1843,14 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
     setConfiguration(60, "orchestration", "Reconfiguration timeout seconds");
 
     Maybe<map<string, string>> json_parser_return =
-        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}});
+        map<string, string>({{"l4_firewall", l4_firewall}, {"version", version_value}, {"versions", versions}});
+
+    string policy_versions_path = "/etc/cp/conf/versions/versions.policy";
+    EXPECT_CALL(mock_orchestration_tools, doesFileExist(policy_versions_path)).WillOnce(Return(false));
+    EXPECT_CALL(mock_orchestration_tools, writeFile(versions, policy_versions_path, false)).WillOnce(Return(true));
+    EXPECT_CALL(mock_orchestration_status,
+        setServiceConfiguration("versions", policy_versions_path, OrchestrationStatusConfigType::POLICY));
+
     EXPECT_CALL(mock_orchestration_tools, readFile(file_name)).WillOnce(Return(new_configuration));
     EXPECT_CALL(mock_orchestration_tools, jsonObjectSplitter(new_configuration, _, _))
         .WillOnce(Return(json_parser_return));
@@ -1706,9 +1860,6 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
     EXPECT_CALL(mock_orchestration_status,
         setServiceConfiguration("l4_firewall", l4_firewall_policy_path, OrchestrationStatusConfigType::POLICY));
 
-    EXPECT_CALL(mock_orchestration_tools, calculateChecksum(Package::ChecksumTypes::MD5, file_name))
-        .WillOnce(Return(version_value));
-
     EXPECT_CALL(mock_orchestration_tools, copyFile(policy_file_path, policy_file_path + backup_extension))
         .WillOnce(Return(true));
     EXPECT_CALL(mock_orchestration_tools, copyFile(file_name, policy_file_path)).WillOnce(Return(true));
@@ -1737,7 +1888,7 @@ TEST_F(ServiceControllerTest, test_delayed_reconf)
         << "    \"error_message\": \"\""
         << "}";
 
-    expectNewConfigRequest("{\n    \"id\": 1,\n    \"policy_version\": \"1.0.2,1.0.2\"\n}", reply_msg);
+    expectNewConfigRequest(reply_msg);
 
     auto func = [&] (chrono::microseconds) { set_reconf_status->performRestCall(reconf_status); };
     EXPECT_CALL(mock_ml, yield(chrono::microseconds(2000000))).WillOnce(Invoke(func));

components/security_apps/orchestration/update_communication/declarative_policy_utils.cc

@@ -100,11 +100,19 @@ DeclarativePolicyUtils::updateCurrentPolicy(const string &policy_checksum)
 {
     string clean_policy_checksum = getCleanChecksum(policy_checksum);
     auto env = Singleton::Consume<I_EnvDetails>::by<DeclarativePolicyUtils>()->getEnvType();
-    curr_policy = Singleton::Consume<I_LocalPolicyMgmtGen>::by<DeclarativePolicyUtils>()->generateAppSecLocalPolicy(
-        env,
-        clean_policy_checksum,
-        local_policy_path
-    );
+    string maybe_policy =
+        Singleton::Consume<I_LocalPolicyMgmtGen>::by<DeclarativePolicyUtils>()->generateAppSecLocalPolicy(
+            env,
+            clean_policy_checksum,
+            local_policy_path
+        );
+
+    if (maybe_policy.empty()) {
+        dbgWarning(D_ORCHESTRATOR) << "Could not generate appsec local policy";
+        return;
+    }
+
+    curr_policy = maybe_policy;
 }
 
 string

components/security_apps/orchestration/update_communication/fog_authenticator.cc

@@ -68,6 +68,12 @@ FogAuthenticator::RegistrationData::RegistrationData(const string &token)
 {
 }
 
+string
+FogAuthenticator::RegistrationData::getData() const
+{
+    return data;
+}
+
 FogAuthenticator::UserCredentials::UserCredentials(const string &_client_id, const string &_shared_secret)
         :
     client_id(_client_id),
@@ -173,6 +179,17 @@ FogAuthenticator::registerAgent(
         dbgDebug(D_ORCHESTRATOR) << nginx_data.getErr();
     }
 
+    auto cloud_metadata = details_resolver->readCloudMetadata();
+    if (cloud_metadata.ok()) {
+        request << make_pair("cloudAccountId", ::get<0>(cloud_metadata.unpack()));
+        request << make_pair("cloudVpcId", ::get<1>(cloud_metadata.unpack()));
+        request << make_pair("cloudInstanceId", ::get<2>(cloud_metadata.unpack()));
+        request << make_pair("cloudInstanceLocalIp", ::get<3>(cloud_metadata.unpack()));
+        request << make_pair("cloudRegion", ::get<4>(cloud_metadata.unpack()));
+    } else {
+        dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
+    }
+
     for (const pair<string, string> details : details_resolver->getResolvedDetails()) {
         request << details;
     }
@@ -193,6 +210,10 @@ FogAuthenticator::registerAgent(
         request << make_pair("reverse_proxy", "true");
     }
 
+    if (details_resolver->isCloudStorageEnabled()) {
+        request << make_pair("cloud_storage_service", "true");
+    }
+
     if (details_resolver->isKernelVersion3OrHigher()) {
         request << make_pair("isKernelVersion3OrHigher", "true");
     }
@@ -212,6 +233,10 @@ FogAuthenticator::registerAgent(
     if (details_resolver->compareCheckpointVersion(8200, std::greater_equal<int>())) {
         request << make_pair("isCheckpointVersionGER82", "true");
     }
+    auto maybe_vs_id = Singleton::Consume<I_Environment>::by<FogAuthenticator>()->get<string>("VS ID");
+    if (maybe_vs_id.ok()) {
+        request << make_pair("virtualSystemId", maybe_vs_id.unpack());
+    }
 #endif // gaia || smb
 
     dbgDebug(D_ORCHESTRATOR) << "Sending registration request to fog";
@@ -297,36 +322,88 @@ FogAuthenticator::getRegistrationData()
         return reg_data;
     }
 
-    const char *env_otp = getenv("NANO_AGENT_TOKEN");
-    if (env_otp) {
-        dbgInfo(D_ORCHESTRATOR) << "Loading registration token from environment";
-        return RegistrationData(env_otp);
-    }
     if (reg_data.ok()) {
         dbgInfo(D_ORCHESTRATOR) << "Loading registration token from cache";
         return reg_data;
     }
 
+    auto local_env_token = getRegistrationToken();
+    if (local_env_token.ok()) return local_env_token;
+
+    return genError("Failed to load registration token from the environment.");
+}
+
+Maybe<FogAuthenticator::RegistrationData>
+FogAuthenticator::getRegistrationToken()
+{
     auto reg_data_path = getConfigurationWithDefault<string>(
         filesystem_prefix + "/conf/registration-data.json",
         "orchestration",
         "Registration data Path"
     );
+    dbgTrace(D_ORCHESTRATOR) << "Getting registration token from " << reg_data_path;
 
-    dbgDebug(D_ORCHESTRATOR) << "Loading registration data from " << reg_data_path;
     auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<FogAuthenticator>();
     auto raw_reg_data = orchestration_tools->readFile(reg_data_path);
-    if (!raw_reg_data.ok()) return genError(raw_reg_data.getErr());
+    if (raw_reg_data.ok()) {
+        auto decoded_reg_data = orchestration_tools->base64Decode(raw_reg_data.unpack());
+        reg_data = orchestration_tools->jsonStringToObject<RegistrationData>(decoded_reg_data);
 
-    dbgTrace(D_ORCHESTRATOR) << "Successfully loaded the registration data";
-    auto decoded_reg_data = orchestration_tools->base64Decode(raw_reg_data.unpack());
-    reg_data = orchestration_tools->jsonStringToObject<RegistrationData>(decoded_reg_data);
-
-    if (reg_data.ok()) {
-        dbgTrace(D_ORCHESTRATOR) << "Registration token has been converted to an object";
+        if (reg_data.ok()) {
+            dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from: " << reg_data_path;
+            return reg_data;
+        }
     }
 
-    return reg_data;
+    dbgTrace(D_ORCHESTRATOR) << "Getting registration token from container environment.";
+    const char *container_otp = getenv("AGENT_TOKEN");
+    if (container_otp) {
+        dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from container environment";
+        return RegistrationData(container_otp);
+    }
+
+    dbgTrace(D_ORCHESTRATOR) << "Getting registration token from the environment.";
+    const char *env_otp = getenv("NANO_AGENT_TOKEN");
+    if (env_otp) {
+        dbgInfo(D_ORCHESTRATOR) << "Registration token has been loaded from the environment";
+        return RegistrationData(env_otp);
+    }
+
+    return genError("No registration token in the environment");
+}
+
+void
+FogAuthenticator::registerLocalAgentToFog()
+{
+    auto local_reg_token = getRegistrationToken();
+    if (!local_reg_token.ok()) return;
+    dbgInfo(D_ORCHESTRATOR) << "Start local agent registration to the fog";
+
+    string exec_command = "open-appsec-ctl --set-mode --online_mode --token " + local_reg_token.unpack().getData();
+
+    auto i_agent_details = Singleton::Consume<I_AgentDetails>::by<FogAuthenticator>();
+    auto fog_address = i_agent_details->getFogDomain();
+    if (fog_address.ok()) exec_command = exec_command + " --fog https://" + fog_address.unpack();
+
+    auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<FogAuthenticator>();
+    auto maybe_cmd_output = shell_cmd->getExecOutputAndCode(
+        exec_command,
+        300000,
+        true
+    );
+    if (!maybe_cmd_output.ok()){
+        dbgWarning(D_ORCHESTRATOR)
+            << "Failed in local agent registertion to the fog. Error: "
+            << maybe_cmd_output.getErr();
+        return;
+    }
+
+    if (maybe_cmd_output.unpack().second != 0) {
+        dbgWarning(D_ORCHESTRATOR)
+            << "Failed in local agent registertion to the fog. Error: "
+            << maybe_cmd_output.unpack().first;
+        return;
+    }
 }
 
 bool
@@ -378,6 +455,22 @@ FogAuthenticator::getCredentialsFromFile() const
     return orchestration_tools->jsonStringToObject<UserCredentials>(encrypted_cred.unpack());
 }
 
+static string
+getDeplymentType()
+{
+    auto deplyment_type = Singleton::Consume<I_EnvDetails>::by<FogAuthenticator>()->getEnvType();
+    switch (deplyment_type) {
+        case EnvType::LINUX: return "Embedded";
+        case EnvType::DOCKER: return "Docker";
+        case EnvType::NON_CRD_K8S:
+        case EnvType::K8S: return "K8S";
+        case EnvType::COUNT: break;
+    }
+
+    dbgAssert(false) << "Failed to get a legitimate deplyment type: " << static_cast<uint>(deplyment_type);
+    return "Embedded";
+}
+
 Maybe<FogAuthenticator::UserCredentials>
 FogAuthenticator::getCredentials()
 {
@@ -386,7 +479,7 @@ FogAuthenticator::getCredentials()
         return maybe_credentials;
     }
 
-    dbgTrace(D_ORCHESTRATOR) << "Credentials were not not receoived from the file. Getting registration data.";
+    dbgTrace(D_ORCHESTRATOR) << "Credentials were not not received from the file. Getting registration data.";
     auto reg_data = getRegistrationData();
     if (!reg_data.ok()) {
         return genError("Failed to load a valid registration token, Error: " + reg_data.getErr());
@@ -396,17 +489,24 @@ FogAuthenticator::getCredentials()
     Maybe<string> name = details_resolver->getHostname();
     if (!name.ok()) return name.passErr();
 
+    auto maybe_vs_id = Singleton::Consume<I_Environment>::by<FogAuthenticator>()->get<string>("VS ID");
+    string host_name = *name;
+    if (maybe_vs_id.ok()) {
+        host_name.append(":");
+        host_name.append(maybe_vs_id.unpack());
+    }
+
     Maybe<string> platform = details_resolver->getPlatform();
     if (!platform.ok()) return platform.passErr();
 
     Maybe<string> arch = details_resolver->getArch();
     if (!arch.ok()) return arch.passErr();
 
-    string type = getConfigurationWithDefault<string>("Embedded", "orchestration", "Agent type");
-    maybe_credentials = registerAgent(reg_data.unpack(), *name, type, *platform, *arch);
+    string type = getSettingWithDefault(getDeplymentType(), "orchestration", "Agent type");
+    maybe_credentials = registerAgent(reg_data.unpack(), host_name, type, *platform, *arch);
 
     auto orc_status = Singleton::Consume<I_OrchestrationStatus>::by<FogAuthenticator>();
-    orc_status->setRegistrationDetails(*name, type, *platform, *arch);
+    orc_status->setRegistrationDetails(host_name, type, *platform, *arch);
 
     if (!maybe_credentials.ok()) return maybe_credentials;
 
@@ -490,7 +590,7 @@ FogAuthenticator::authenticateAgent()
     auto mainloop = Singleton::Consume<I_MainLoop>::by<FogAuthenticator>();
     if (!mainloop->doesRoutineExist(routine)) {
         routine = mainloop->addOneTimeRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             [this, min_expiration_time] ()
             {
                 uint expiration_time;

components/security_apps/orchestration/update_communication/fog_communication.cc

@@ -96,8 +96,6 @@ FogCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, co
 {
     if (!access_token.ok()) return genError("Acccess Token not available.");
 
-    auto unpacked_access_token = access_token.unpack().getToken();
-
     string policy_mgmt_mode = getSettingWithDefault<string>("management", "profileManagedMode");
     if (policy_mgmt_mode == "declarative" && resourse_file.getFileName() =="policy") {
         dbgDebug(D_ORCHESTRATOR) << "Download policy on declarative mode - returning the local policy";

components/security_apps/orchestration/update_communication/hybrid_communication.cc

@@ -104,8 +104,6 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file,
     if (resourse_file.getFileName() == "manifest") {
         if (!access_token.ok()) return genError("Acccess Token not available.");
 
-        auto unpacked_access_token = access_token.unpack().getToken();
-
         auto attribute_file = Singleton::Consume<I_Messaging>::by<HybridCommunication>()->downloadFile(
             resourse_file.getRequestMethod(),
             agent_resource_api + '/' + resourse_file.getFileName(),

components/security_apps/orchestration/update_communication/local_communication.cc

@@ -40,6 +40,12 @@ LocalCommunication::authenticateAgent()
     return Maybe<void>();
 }
 
+void
+LocalCommunication::registerLocalAgentToFog()
+{
+    return;
+}
+
 string
 LocalCommunication::getChecksum(const string &file_path)
 {

components/security_apps/orchestration/update_communication/update_communication.cc

@@ -32,6 +32,7 @@
 using namespace std;
 
 USE_DEBUG_FLAG(D_ORCHESTRATOR);
+
 class UpdateCommunication::Impl
         :
     public ServerRest,
@@ -69,6 +70,11 @@ public:
         return i_update_comm_impl->authenticateAgent();
     }
 
+    void registerLocalAgentToFog()
+    {
+        i_update_comm_impl->registerLocalAgentToFog();
+    }
+
     Maybe<void>
     getUpdate(CheckUpdateRequest &request) override
     {

components/security_apps/orchestration/update_communication/update_communication_ut/local_communication_ut.cc

@@ -38,6 +38,12 @@ public:
         return local_communication.authenticateAgent();
     }
 
+    void
+    registerLocalAgentToFog()
+    {
+        local_communication.registerLocalAgentToFog();
+    }
+
     Maybe<void>
     sendPolicyVersion(const string &version, const string &policy_versions)
     {
@@ -122,6 +128,11 @@ TEST_F(LocalCommunicationTest, authenticateAgent)
     EXPECT_TRUE(authenticat_res.ok());
 }
 
+TEST_F(LocalCommunicationTest, registerLocalAgentToFog)
+{
+    registerLocalAgentToFog();
+}
+
 TEST_F(LocalCommunicationTest, downloadManifest)
 {
     string new_manifest_string = "new manifest";

components/security_apps/orchestration/updates_process_reporter/CMakeLists.txt

@@ -0,0 +1 @@
+add_library(updates_process_reporter updates_process_event.cc updates_process_reporter.cc)

components/security_apps/orchestration/updates_process_reporter/updates_process_event.cc

@@ -0,0 +1,124 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "updates_process_event.h"
+
+#include <sstream>
+#include <string>
+
+#include "debug.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_UPDATES_PROCESS_REPORTER);
+
+UpdatesProcessEvent::UpdatesProcessEvent(
+    UpdatesProcessResult _result,
+    UpdatesConfigType _type,
+    UpdatesFailureReason _reason,
+    const std::string &_detail,
+    const std::string &_description)
+        :
+    result(_result),
+    type(_type),
+    reason(_reason),
+    detail(_detail),
+    description(_description)
+{
+    string report =
+        "Result: " + convertUpdateProcessResultToStr(result) +
+        ", Reason: " + convertUpdatesFailureReasonToStr(reason) +
+        ", Type: " + convertUpdatesConfigTypeToStr(type) +
+        ", Detail: " + detail +
+        ", Description: " + description;
+    dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Updates process event: " << report;
+}
+
+OrchestrationStatusFieldType
+UpdatesProcessEvent::getStatusFieldType() const
+{
+    if (reason == UpdatesFailureReason::REGISTRATION) {
+        return OrchestrationStatusFieldType::REGISTRATION;
+    }
+    if (type == UpdatesConfigType::MANIFEST) {
+        return  OrchestrationStatusFieldType::MANIFEST;
+    }
+    return OrchestrationStatusFieldType::LAST_UPDATE;
+}
+
+OrchestrationStatusResult
+UpdatesProcessEvent::getOrchestrationStatusResult() const
+{
+    return result == UpdatesProcessResult::SUCCESS ?
+        OrchestrationStatusResult::SUCCESS :
+        OrchestrationStatusResult::FAILED;
+}
+
+string
+UpdatesProcessEvent::parseDescription() const
+{
+    stringstream err;
+    if (description.empty() || result == UpdatesProcessResult::SUCCESS) return "";
+
+    switch (reason) {
+        case UpdatesFailureReason::CHECK_UPDATE: {
+            err << description;
+            break;
+        }
+        case UpdatesFailureReason::REGISTRATION: {
+            err << "Registration failed. Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::GET_UPDATE_REQUEST: {
+            err << "Failed to get update request. Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::DOWNLOAD_FILE : {
+            err << "Failed to download the file " << detail << ". Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::HANDLE_FILE : {
+            err << "Failed to handle the file " << detail << ". " << description;
+            break;
+        }
+        case UpdatesFailureReason::INSTALLATION_QUEUE : {
+            err << "Installation queue creation failed. Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::INSTALL_PACKAGE : {
+            err << "Failed to install the package " << detail << ". Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::CHECKSUM_UNMATCHED : {
+            err << "Checksums do not match for the file: " << detail << ". " << description;
+            break;
+        }
+        case UpdatesFailureReason::POLICY_CONFIGURATION : {
+            err << "Failed to configure policy version: " << detail << ". Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::POLICY_FOG_CONFIGURATION : {
+            err << "Failed to configure the fog address: " << detail << ". Error: " << description;
+            break;
+        }
+        case UpdatesFailureReason::ORCHESTRATION_SELF_UPDATE : {
+            err << description;
+            break;
+        }
+        case UpdatesFailureReason::NONE : {
+            err << description;
+            break;
+        }
+    }
+    return err.str();
+}

components/security_apps/orchestration/updates_process_reporter/updates_process_reporter.cc

@@ -0,0 +1,71 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "updates_process_reporter.h"
+
+#include <sstream>
+#include <string>
+
+#include "debug.h"
+#include "log_generator.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_UPDATES_PROCESS_REPORTER);
+
+vector<UpdatesProcessReport> UpdatesProcessReporter::reports;
+
+void
+UpdatesProcessReporter::upon(const UpdatesProcessEvent &event)
+{
+    if (event.getReason() == UpdatesFailureReason::CHECK_UPDATE) {
+        if (event.getResult() == UpdatesProcessResult::SUCCESS && reports.empty()) {
+            dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Update proccess finished successfully";
+            return;
+        }
+        dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Update proccess finished with errors";
+        reports.emplace_back(
+            UpdatesProcessReport(
+                event.getResult(),
+                event.getType(),
+                event.getReason(),
+                event.parseDescription()
+            )
+        );
+        sendReoprt();
+        return;
+    }
+    if (event.getResult() == UpdatesProcessResult::SUCCESS || event.getResult() == UpdatesProcessResult::UNSET) return;
+    reports.emplace_back(
+        UpdatesProcessReport(event.getResult(), event.getType(), event.getReason(), event.parseDescription())
+    );
+}
+
+void
+UpdatesProcessReporter::sendReoprt()
+{
+    stringstream all_reports;
+    all_reports << "Updates process reports:" << endl;
+    for (const auto &report : reports) {
+        all_reports << report.toString() << endl;
+    }
+    reports.clear();
+    dbgTrace(D_UPDATES_PROCESS_REPORTER) << "Sending updates process report: " << endl << all_reports.str();
+    LogGen(
+        "Updates process report",
+        ReportIS::Audience::INTERNAL,
+        ReportIS::Severity::HIGH,
+        ReportIS::Priority::HIGH,
+        ReportIS::Tags::ORCHESTRATOR
+    ) << LogField("eventMessage", all_reports.str());
+}

components/security_apps/waap/waap_clib/CMakeLists.txt

@@ -87,6 +87,7 @@ add_library(waap_clib
     ParserPairs.cc
     Waf2Util2.cc
     ParserPDF.cc
+    ParserBinaryFile.cc
 )
 
 add_definitions("-Wno-unused-function")

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -27,6 +27,7 @@
 #include "ParserPairs.h"
 #include "ParserDelimiter.h"
 #include "ParserPDF.h"
+#include "ParserBinaryFile.h"
 #include "WaapAssetState.h"
 #include "Waf2Regex.h"
 #include "Waf2Util.h"
@@ -272,54 +273,58 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
     // Detect and decode potential base64 chunks in the value before further processing
 
     bool base64ParamFound = false;
-    dbgTrace(D_WAAP_DEEP_PARSER) << " ===Processing potential base64===";
-    std::string decoded_val, decoded_key;
-    base64_variants base64_status = Waap::Util::b64Test(cur_val, decoded_key, decoded_val);
+    Waap::Util::BinaryFileType base64BinaryFileType = Waap::Util::BinaryFileType::FILE_TYPE_NONE;
+    if (m_depth == 1 && flags == BUFFERED_RECEIVER_F_MIDDLE && m_key.depth() == 1 && m_key.first() != "#base64"){
+        dbgTrace(D_WAAP_DEEP_PARSER) << " === will not check base64 since prev data block was not b64-encoded ===";
+    } else {
+        dbgTrace(D_WAAP_DEEP_PARSER) << " ===Processing potential base64===";
+        std::string decoded_val, decoded_key;
+        base64_variants base64_status = Waap::Util::b64Test(cur_val, decoded_key, decoded_val, base64BinaryFileType);
 
-    dbgTrace(D_WAAP_DEEP_PARSER)
-        << " status = "
-        << base64_status
-        << " key = "
-        << decoded_key
-        << " value = "
-        << decoded_val;
+        dbgTrace(D_WAAP_DEEP_PARSER)
+            << " status = "
+            << base64_status
+            << " key = "
+            << decoded_key
+            << " value = "
+            << decoded_val;
 
-    switch (base64_status) {
-        case SINGLE_B64_CHUNK_CONVERT:
-            cur_val = decoded_val;
-            base64ParamFound = true;
-            break;
-        case KEY_VALUE_B64_PAIR:
-            // going deep with new pair in case value is not empty
-            if (decoded_val.size() > 0) {
+        switch (base64_status) {
+            case SINGLE_B64_CHUNK_CONVERT:
                 cur_val = decoded_val;
                 base64ParamFound = true;
-                rc = onKv(
-                    decoded_key.c_str(),
-                    decoded_key.size(),
-                    cur_val.data(),
-                    cur_val.size(),
-                    flags,
-                    parser_depth
+                break;
+            case KEY_VALUE_B64_PAIR:
+                // going deep with new pair in case value is not empty
+                if (decoded_val.size() > 0) {
+                    cur_val = decoded_val;
+                    base64ParamFound = true;
+                    rc = onKv(
+                        decoded_key.c_str(),
+                        decoded_key.size(),
+                        cur_val.data(),
+                        cur_val.size(),
+                        flags,
+                        parser_depth
                     );
 
-                dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc;
-                if (rc != CONTINUE_PARSING) {
-                    return rc;
+                    dbgTrace(D_WAAP_DEEP_PARSER) << " rc = " << rc;
+                    if (rc != CONTINUE_PARSING) {
+                        return rc;
+                    }
                 }
-            }
-            break;
-        case CONTINUE_AS_IS:
-            break;
-        default:
-            break;
-    }
+                break;
+            case CONTINUE_AS_IS:
+                break;
+            default:
+                break;
+        }
 
-    if (base64ParamFound) {
-        dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): pushing #base64 prefix to the key.";
-        m_key.push("#base64", 7, false);
+        if (base64ParamFound) {
+            dbgTrace(D_WAAP_DEEP_PARSER) << "DeepParser::onKv(): pushing #base64 prefix to the key.";
+            m_key.push("#base64", 7, false);
+        }
     }
-
     // cur_val is later passed through some filters (such as urldecode) before JSON, XML or HTML is detected/decoded
     std::string orig_val = cur_val;
 
@@ -355,7 +360,8 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
             isUrlPayload,
             isUrlParamPayload,
             flags,
-            parser_depth
+            parser_depth,
+            base64BinaryFileType
         );
     } else {
         offset = 0;
@@ -425,7 +431,8 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
                 isUrlParamPayload,
                 flags,
                 parser_depth,
-                base64ParamFound
+                base64ParamFound,
+                base64BinaryFileType
                 );
             if (rc != CONTINUE_PARSING) {
                 return rc;
@@ -468,19 +475,19 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f
     if (rc != CONTINUE_PARSING) {
         return rc;
     }
-
-    if (Waap::Util::detectJSONasParameter(cur_val, decoded_key, decoded_val)) {
+    std::string json_decoded_val, json_decoded_key;
+    if (Waap::Util::detectJSONasParameter(cur_val, json_decoded_key, json_decoded_val)) {
         dbgTrace(D_WAAP_DEEP_PARSER)
             << " detectJSONasParameter was  true: key = "
-            << decoded_key
+            << json_decoded_key
             << " value = "
-            << decoded_val;
+            << json_decoded_val;
 
         rc = onKv(
-            decoded_key.c_str(),
-            decoded_key.size(),
-            decoded_val.data(),
-            decoded_val.size(),
+            json_decoded_key.c_str(),
+            json_decoded_key.size(),
+            json_decoded_val.data(),
+            json_decoded_val.size(),
             flags,
             parser_depth
             );
@@ -798,7 +805,8 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
     bool isUrlParamPayload,
     int flags,
     size_t parser_depth,
-    bool base64ParamFound)
+    bool base64ParamFound,
+    Waap::Util::BinaryFileType b64FileType)
 {
     int offset = -1;
     int rc = 0;
@@ -815,7 +823,8 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
             isUrlPayload,
             isUrlParamPayload,
             flags,
-            parser_depth
+            parser_depth,
+            b64FileType
         );
     } else {
         offset = 0;
@@ -919,7 +928,8 @@ DeepParser::createInternalParser(
     bool isUrlPayload,
     bool isUrlParamPayload,
     int flags,
-    size_t parser_depth
+    size_t parser_depth,
+    Waap::Util::BinaryFileType b64FileType
 )
 {
     dbgTrace(D_WAAP_DEEP_PARSER)
@@ -1152,10 +1162,25 @@ DeepParser::createInternalParser(
                 m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserPDF>>(*this, parser_depth + 1));
                 offset = 0;
             } else {
-                dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file";
-                m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserBinary>>(*this, parser_depth + 1));
-                offset = 0;
+                Waap::Util::BinaryFileType fileType = ParserBinaryFile::detectBinaryFileHeader(cur_val);
+                if (fileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE) {
+                    dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a known binary file (type=" << fileType << ")";
+                    m_parsersDeque.push_back(
+                        std::make_shared<BufferedParser<ParserBinaryFile>>(*this, parser_depth + 1, false, fileType)
+                    );
+                    offset = 0;
+                } else {
+                    dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a binary file";
+                    m_parsersDeque.push_back(std::make_shared<BufferedParser<ParserBinary>>(*this, parser_depth + 1));
+                    offset = 0;
+                }
             }
+        } else if (b64FileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE) {
+            dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a known binary file, base64 encoded";
+            m_parsersDeque.push_back(
+                std::make_shared<BufferedParser<ParserBinaryFile>>(*this, parser_depth + 1, true, b64FileType)
+            );
+            offset = 0;
         }
     }
     if (offset < 0) {

components/security_apps/waap/waap_clib/DeepParser.h

@@ -18,6 +18,7 @@
 #include "KeyStack.h"
 #include "WaapAssetState.h"
 #include "Waf2Regex.h"
+#include "Waf2Util.h"
 #include "maybe_res.h"
 #include <deque>
 
@@ -129,7 +130,8 @@ private:
         bool isUrlPayload,
         bool isUrlParamPayload,
         int flags,
-        size_t parser_depth
+        size_t parser_depth,
+        Waap::Util::BinaryFileType b64FileType
     );
 
     int createUrlParserForJson(
@@ -160,7 +162,8 @@ private:
         bool isUrlParamPayload,
         int flags,
         size_t parser_depth,
-        bool base64ParamFound
+        bool base64ParamFound,
+        Waap::Util::BinaryFileType b64FileType
     );
     int pushValueToTopParser(std::string &cur_val, int flags, bool base64ParamFound, int offset, size_t parser_depth);
     int parseBuffer(

components/security_apps/waap/waap_clib/ParserBase.h

@@ -22,6 +22,7 @@
 #define BUFFERED_RECEIVER_F_LAST 0x02
 #define BUFFERED_RECEIVER_F_BOTH (BUFFERED_RECEIVER_F_FIRST | BUFFERED_RECEIVER_F_LAST)
 #define BUFFERED_RECEIVER_F_UNNAMED 0x04
+#define BUFFERED_RECEIVER_F_MIDDLE 0x00
 
 #if (DISTRO_centos6)
 // pre c++11 compiler doesn' support the "final" keyword

components/security_apps/waap/waap_clib/ParserBinaryFile.cc

@@ -0,0 +1,199 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "ParserBinaryFile.h"
+#include "Waf2Util.h"
+#include "debug.h"
+#include <string.h>
+#include <map>
+#include <tuple>
+
+using namespace std;
+using Waap::Util::BinaryFileType;
+
+USE_DEBUG_FLAG(D_WAAP_PARSER_BINARY_FILE);
+USE_DEBUG_FLAG(D_WAAP);
+
+const string ParserBinaryFile::m_parserName = "ParserBinaryFile";
+
+static const map<BinaryFileType, pair<string, string>> m_head_tail_map = {
+    {BinaryFileType::FILE_TYPE_PNG,
+        {string("\x89\x50\x4E\x47\x0D\x0A\x1A\x0A"), // PNG
+        string("\x49\x45\x4e\x44\xae\x42\x60\x82")}},  // IEND
+    {BinaryFileType::FILE_TYPE_JPEG,
+        {string("\xff\xd8\xff"),
+        string("\xff\xd9")}},
+    {BinaryFileType::FILE_TYPE_PDF,
+        {string("%PDF-"),
+        string("%%EOF")}}
+};
+
+ParserBinaryFile::ParserBinaryFile(
+    IParserStreamReceiver &receiver,
+    size_t parser_depth,
+    bool is_base64,
+    BinaryFileType file_type
+) :
+    m_receiver(receiver),
+    m_state(s_start),
+    m_parser_depth(parser_depth),
+    m_is_base64(is_base64),
+    m_file_type(file_type)
+{}
+
+ParserBinaryFile::~ParserBinaryFile()
+{}
+
+BinaryFileType
+ParserBinaryFile::detectBinaryFileHeader(const string &buf)
+{
+    if (buf.size() < MIN_HEADER_LOOKUP) {
+        dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "Buffer size too small (" << buf.size() << ")";
+        return BinaryFileType::FILE_TYPE_NONE;
+    }
+    const string searchStr = buf.substr(0, MAX_HEADER_LOOKUP);
+    for (const auto &entry : m_head_tail_map) {
+        const string &head = entry.second.first;
+        size_t pos = searchStr.find(head);
+        if (pos != string::npos) {
+            if (buf.size() - pos >= MIN_HEADER_LOOKUP) {
+                dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "Found. type=" << entry.first;
+                return entry.first;
+            } else {
+                dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "Remaining size after header is too small";
+                return BinaryFileType::FILE_TYPE_NONE;
+            }
+        }
+    }
+    return BinaryFileType::FILE_TYPE_NONE;
+}
+
+
+size_t
+ParserBinaryFile::push(const char *buf, size_t len)
+{
+    dbgTrace(D_WAAP_PARSER_BINARY_FILE)
+        << "buf="
+        << buf
+        << "len="
+        << len;
+
+    const char *c;
+
+    if (m_state == s_error) {
+        return 0;
+    }
+    if (len == 0) {
+        dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "end of stream. m_state=" << m_state;
+
+        if (m_state == s_end) {
+            m_receiver.onKvDone();
+        } else if (m_is_base64) {
+            dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "finished parsing";
+            if (m_receiver.onKey("BinaryFileSkip", 14) != 0) {
+                m_state = s_error;
+                return 0;
+            }
+            if (m_receiver.onValue("", 0) != 0) {
+                m_state = s_error;
+                return 0;
+            }
+            m_receiver.onKvDone();
+        } else {
+            m_state = s_error;
+        }
+        return 0;
+    }
+    if (m_head_tail_map.find(m_file_type) == m_head_tail_map.end()) {
+        dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "unknown file type: " << m_file_type;
+        m_state = s_error;
+        return 0;
+    }
+    const string tail = m_head_tail_map.at(m_file_type).second;
+
+    switch (m_state) {
+            case s_start:
+                m_state = s_body;
+                CP_FALL_THROUGH;
+            case s_body:
+                if (m_is_base64) {
+                    dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing base64";
+                    bool keepParsing = true;
+                    for (size_t i = 0; i < len; i++) {
+                        bool isB64AlphaChar =
+                            Waap::Util::isAlphaAsciiFast(buf[i]) || isdigit(buf[i]) || buf[i] == '/' || buf[i] == '+';
+                        if (buf[i] == '=') {
+                            dbgTrace(D_WAAP_PARSER_BINARY_FILE)
+                                << "base64 padding found (offset=" << i << "). end of stream.";
+                            m_state = s_end;
+                            keepParsing = false;
+                            break;
+                        } else if (!isB64AlphaChar) {
+                            dbgTrace(D_WAAP_PARSER_BINARY_FILE)
+                                << "non-base64 char found (c=" << buf[i] << ",offset=" << i << "). return error";
+                            m_state = s_error;
+                            return 0;
+                        }
+                    }
+                    if (keepParsing) { // keep "parsing" on next call to push()
+                        break;
+                    }
+                } else {
+                    dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "parsing binary. Searching for tail: " << tail;
+                    c = strstr(buf + len - tail.size(), tail.c_str());
+                    dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "search result: c=" << c;
+                    if (c) {
+                        m_state = s_end;
+                    } else { // keep "parsing" on next call to push()
+                        break;
+                    }
+                }
+                CP_FALL_THROUGH;
+            case s_end:
+                dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "finished parsing";
+                if (m_receiver.onKey("BinaryFileSkip", 14) != 0) {
+                    m_state = s_error;
+                    return 0;
+                }
+                if (m_receiver.onValue("", 0) != 0) {
+                    m_state = s_error;
+                    return 0;
+                }
+                break;
+            case s_error:
+                dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "error detected";
+                break;
+            default:
+                dbgTrace(D_WAAP_PARSER_BINARY_FILE) << "unknown state: " << m_state;
+                m_state = s_error;
+                return 0;
+    }
+
+    return len;
+}
+
+
+void ParserBinaryFile::finish()
+{
+    push(NULL, 0);
+}
+
+const string& ParserBinaryFile::name() const
+{
+    return m_parserName;
+}
+
+bool ParserBinaryFile::error() const
+{
+    return m_state == s_error;
+}

components/security_apps/waap/waap_clib/ParserBinaryFile.h

@@ -0,0 +1,57 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __PARSER_BINARY_FILE_H__
+#define __PARSER_BINARY_FILE_H__
+
+#include "ParserBase.h"
+#include "Waf2Util.h"
+#include <string.h>
+
+#define MIN_HEADER_LOOKUP 16
+#define MAX_HEADER_LOOKUP 64
+#define MAX_TAIL_LOOKUP 5
+
+class ParserBinaryFile : public ParserBase {
+public:
+    static Waap::Util::BinaryFileType detectBinaryFileHeader(const std::string &buf);
+
+    ParserBinaryFile(
+        IParserStreamReceiver &receiver,
+        size_t parser_depth,
+        bool is_base64,
+        Waap::Util::BinaryFileType file_type);
+    virtual ~ParserBinaryFile();
+    virtual size_t push(const char *buf, size_t len);
+    virtual void finish();
+    virtual const std::string &name() const;
+    virtual bool error() const;
+    virtual size_t depth() { return 1; }
+
+private:
+    enum state {
+        s_start,
+        s_body,
+        s_end,
+        s_error
+    };
+
+    IParserStreamReceiver &m_receiver;
+    enum state m_state;
+    static const std::string m_parserName;
+    size_t m_parser_depth;
+    bool m_is_base64;
+    Waap::Util::BinaryFileType m_file_type;
+};
+
+#endif // __PARSER_BINARY_FILE_H__

components/security_apps/waap/waap_clib/ParserJson.cc

@@ -23,6 +23,7 @@ USE_DEBUG_FLAG(D_WAAP_PARSER_JSON);
 USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
 
 const std::string ParserJson::m_parserName = "jsonParser";
+static const std::string OPERATION_NAME_GRAPHQL = "operationName";
 
 int
 ParserJson::cb_null()
@@ -303,6 +304,7 @@ ParserJson::ParserJson(
     m_key.push("json", 4);
 }
 
+
 ParserJson::~ParserJson()
 {
     // Cleanup JSON

components/security_apps/waap/waap_clib/Serializator.cc

@@ -727,7 +727,6 @@ void SerializeToLocalAndRemoteSyncBase::syncWorker()
             "sync notification for '" + m_assetId + "'",
             ReportIS::AudienceTeam::WAAP,
             syncNotification,
-            false,
             MessageCategory::GENERIC,
             ReportIS::Tags::WAF,
             ReportIS::Notification::SYNC_LEARNING

components/security_apps/waap/waap_clib/Telemetry.cc

@@ -44,6 +44,7 @@ WaapTelemetryBase::sendLog(const LogRest &metric_client_rest) const
         "X-Tenant-Id",
         Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getTenantId()
     );
+    req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
     Singleton::Consume<I_Messaging>::by<GenericMetric>()->sendSyncMessageWithoutResponse(
         HTTPMethod::POST,
         fog_metric_uri,

components/security_apps/waap/waap_clib/TuningDecisions.h

@@ -59,6 +59,7 @@ private:
         if (agentDetails->getOrchestrationMode() != OrchestrationMode::ONLINE) {
             MessageMetadata req_md(getSharedStorageHost(), 80);
             req_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
+            req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
             auto req_status = messaging->sendSyncMessage(
                 method,
                 uri,

components/security_apps/waap/waap_clib/WaapOverride.h

@@ -39,12 +39,19 @@ public:
         m_op = to_lower_copy(m_op);
         m_isCidr = false;
         m_value = "";
+        m_isValid = true;
 
         if (m_op == "basic") {
             // If op == "BASIC" - read numeric value
             ar(cereal::make_nvp("tag", m_tag));
             m_tag = to_lower_copy(m_tag);
 
+            if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" &&
+                m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" &&
+                m_tag != "responsebody" &&  m_tag != "headername" && m_tag != "headervalue" ) {
+                m_isValid = false;
+                dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag;
+            }
             // The name "value" here is misleading. The real meaning is "regex pattern string"
             ar(cereal::make_nvp("value", m_value));
 
@@ -73,12 +80,14 @@ public:
                 m_operand2 = std::make_shared<Match>();
                 ar(cereal::make_nvp("operand2", *m_operand2));
                 m_isOverrideResponse = m_operand1->m_isOverrideResponse || m_operand2->m_isOverrideResponse;
+                m_isValid = m_operand1->m_isValid && m_operand2->m_isValid;
             }
             else if (m_op == "not") {
                 // If op is "NOT" get one operand
                 m_operand1 = std::make_shared<Match>();
                 ar(cereal::make_nvp("operand1", *m_operand1));
                 m_isOverrideResponse = m_operand1->m_isOverrideResponse;
+                m_isValid = m_operand1->m_isValid;
             }
         }
     }
@@ -120,6 +129,10 @@ public:
         return m_isOverrideResponse;
     }
 
+    bool isValidMatch() const{
+        return m_isValid;
+    }
+
 private:
     std::string m_op;
     std::shared_ptr<Match> m_operand1;
@@ -130,6 +143,7 @@ private:
     Waap::Util::CIDRData m_cidr;
     bool        m_isCidr;
     bool        m_isOverrideResponse;
+    bool m_isValid;
 };
 
 class Behavior
@@ -189,6 +203,9 @@ private:
 
 class Rule {
 public:
+
+    Rule(): m_match(), m_isChangingRequestData(false), isValid(true){}
+
     bool operator==(const Rule &other) const;
 
     template <typename _A>
@@ -202,6 +219,11 @@ public:
             m_id.clear();
         }
         ar(cereal::make_nvp("parsedMatch", m_match));
+        if (!m_match.isValidMatch()) {
+            dbgDebug(D_WAAP_OVERRIDE) << "An override rule was not load";
+            isValid = false;
+        }
+
         ar(cereal::make_nvp("parsedBehavior", m_behaviors));
 
         m_isChangingRequestData = false;
@@ -242,6 +264,7 @@ public:
         dbgTrace(D_WAAP_OVERRIDE) << "Rule not matched";
     }
 
+
     bool isChangingRequestData() const {
         return m_isChangingRequestData;
     }
@@ -253,11 +276,16 @@ public:
         return m_id;
     }
 
+    bool isValidRule() const {
+        return isValid;
+    }
+
 private:
     Match m_match;
     bool  m_isChangingRequestData;
     std::vector<Behavior> m_behaviors;
     std::string m_id;
+    bool isValid;
 };
 
 class Policy {
@@ -270,6 +298,10 @@ public:
 
         for (std::vector<Waap::Override::Rule>::const_iterator it = rules.begin(); it != rules.end(); ++it) {
             const Waap::Override::Rule& rule = *it;
+            if (!rule.isValidRule()) {
+                dbgWarning(D_WAAP_OVERRIDE) << "rule is not valid";
+                continue;
+            }
             if (rule.isChangingRequestData())
             {
                 m_RequestOverrides.push_back(rule);

components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc

@@ -145,6 +145,6 @@ bool WaapOverrideFunctor::operator()(const std::string& tag, const boost::regex&
     }
 
     // Unknown tag: should not occur
-    dbgWarning(D_WAAP) << "Invalid override tag: " << tag;
+    dbgDebug(D_WAAP) << "Invalid override tag: " << tag;
     return false;
 }

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -1516,6 +1516,7 @@ Waf2Transaction::decideAfterHeaders()
     return finalizeDecision(sitePolicy, shouldBlock);
 }
 
+
 // Note: the only user of the transactionResult structure filled by this method is waap_automation.
 // TODO: Consider removing this parameter (and provide access to this information by other means)
 int

components/security_apps/waap/waap_clib/Waf2EngineGetters.cc

@@ -567,12 +567,11 @@ Waap::Override::State Waf2Transaction::getOverrideState(IWaapConfig* sitePolicy)
 
     extractEnvSourceIdentifier();
 
-    Waap::Override::State overrideStateResponse;
     if (overridePolicy) { // later we will run response overrides
-        overrideStateResponse.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false);
+        m_overrideState.applyOverride(*overridePolicy, WaapOverrideFunctor(*this), m_matchedOverrideIds, false);
     }
     m_isHeaderOverrideScanRequired = false;
-    return overrideStateResponse;
+    return m_overrideState;
 }
 
 Waf2TransactionFlags &Waf2Transaction::getTransactionFlags()

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -34,6 +34,7 @@
 #include "generic_rulebase/rulebase_config.h"
 #include "user_identifiers_config.h"
 #include "Waf2Regex.h"
+#include "ParserBinaryFile.h"
 
 using boost::algorithm::to_lower_copy;
 using namespace std;
@@ -960,11 +961,12 @@ string filterUTF7(const string& text) {
 //  4. percent of non-printable characters (!isprint())
 //     in decoded data is less than 10% (statistical garbage detection).
 // Returns false above checks fail.
-bool decodeBase64Chunk(
+base64_decode_status decodeBase64Chunk(
         const string& value,
         string::const_iterator it,
         string::const_iterator end,
-        string& decoded)
+        string& decoded,
+        bool clear_on_error)
 {
     decoded.clear();
     uint32_t acc = 0;
@@ -974,13 +976,14 @@ bool decodeBase64Chunk(
     uint32_t spacer_count = 0;
 
     dbgTrace(D_WAAP) << "decodeBase64Chunk: value='" << value << "' match='" << string(it, end) << "'";
+    string::const_iterator begin = it;
 
     // The encoded data length (without the "base64," prefix) should be exactly divisible by 4
     // len % 4 is not 0 i.e. this is not base64
         if ((end - it) % 4 != 0) {
             dbgTrace(D_WAAP_BASE64) <<
                 "b64DecodeChunk: (leave as-is) because encoded data length should be exactly divisible by 4.";
-            return false;
+            return B64_DECODE_INVALID;
         }
 
         while (it != end) {
@@ -992,7 +995,7 @@ bool decodeBase64Chunk(
                     dbgTrace(D_WAAP_BASE64) <<
                         "decodeBase64Chunk: (leave as-is) because terminator characters must all be '='," <<
                         "until end of match.";
-                    return false;
+                    return B64_DECODE_INVALID;
                 }
 
                 // We should see 0, 1 or 2 (no more) terminator characters
@@ -1000,7 +1003,7 @@ bool decodeBase64Chunk(
 
                 if (terminatorCharsSeen > 2) {
                     dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because terminatorCharsSeen > 2";
-                    return false;
+                    return B64_DECODE_INVALID;
                 }
 
                 // allow for more terminator characters
@@ -1033,8 +1036,8 @@ bool decodeBase64Chunk(
             }
             else {
                 dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because of non-base64 character ('" <<
-                        c << "', ASCII " << (unsigned int)c << ")";
-                return false; // non-base64 character
+                        c << "', ASCII " << (unsigned int)c << ", offset " << (it-begin) << ")";
+                return B64_DECODE_INVALID; // non-base64 character
             }
 
             acc = (acc << 6) | val;
@@ -1087,20 +1090,23 @@ bool decodeBase64Chunk(
             }
             else {
                 dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (delete) because decoded.size=" << decoded.size() <<
-                        ", nonPrintableCharsCount=" << nonPrintableCharsCount;
-                decoded.clear();
+                        ", nonPrintableCharsCount=" << nonPrintableCharsCount <<
+                        ", clear_on_error=" << clear_on_error;
+                if (clear_on_error) decoded.clear();
+                return B64_DECODE_INCOMPLETE;
             }
             dbgTrace(D_WAAP_BASE64) << "returning true: successfully decoded."
                 << " Returns decoded data in \"decoded\" parameter";
-            return true; // successfully decoded. Returns decoded data in "decoded" parameter
+            return B64_DECODE_OK; // successfully decoded. Returns decoded data in "decoded" parameter
         }
 
         // If decoded size is too small - leave the encoded value (return false)
         decoded.clear(); // discard partial data
         dbgTrace(D_WAAP_BASE64) << "decodeBase64Chunk: (leave as-is) because decoded too small. decoded.size=" <<
                 decoded.size() <<
-                ", nonPrintableCharsCount=" << nonPrintableCharsCount;
-        return false;
+                ", nonPrintableCharsCount=" << nonPrintableCharsCount <<
+                ", clear_on_error=" << clear_on_error;
+        return B64_DECODE_INVALID;
 }
 
 // Attempts to detect and validate base64 chunk.
@@ -1144,7 +1150,7 @@ b64DecodeChunk(
         }
     }
 
-    return decodeBase64Chunk(value, it, end, decoded);
+    return decodeBase64Chunk(value, it, end, decoded) != B64_DECODE_INVALID;
 }
 
 vector<string> split(const string& s, char delim) {
@@ -1281,6 +1287,8 @@ bool detectBase64Chunk(
 {
     dbgTrace(D_WAAP_BASE64) << " ===detectBase64Chunk===:  starting with = '" << s << "'";
     string::const_iterator it = s.begin();
+    start = s.end();
+    end = s.end();
 
     //detect "base64," prefix to start search after this
     for (; it != s.end()-7; it++) {
@@ -1309,33 +1317,62 @@ bool detectBase64Chunk(
     return false;
 }
 
+base64_decode_status
+processDecodedChunk(
+    const string &s,
+    string::const_iterator start,
+    string::const_iterator end,
+    string &value,
+    BinaryFileType &binaryFileType
+)
+{
+    base64_decode_status retVal = decodeBase64Chunk(s, start, end, value, false);
+    dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: after decode. retVal=" << retVal
+                            << " value.size()=" << value.size();
+    if (retVal != B64_DECODE_INVALID && !value.empty()) {
+        binaryFileType = ParserBinaryFile::detectBinaryFileHeader(value);
+        if (retVal == B64_DECODE_INCOMPLETE) value.clear();
+    }
+    return retVal;
+}
+
 bool isBase64PrefixProcessingOK (
         const string &s,
-        string &value)
+        string &value,
+        BinaryFileType &binaryFileType)
 {
     string::const_iterator start, end;
-    bool retVal = false;
+    base64_decode_status retVal = B64_DECODE_INVALID;
     dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: before regex for prefix for string '" << s << "'";
     if (base64_prefix_detector_re.hasMatch(s)) {
         dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: prefix detected on string '" << s << "'";
         if (detectBase64Chunk(s, start, end)) {
             dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk detected";
             if ((start != s.end()) && (end == s.end())) {
-                retVal = decodeBase64Chunk(s, start, end, value);
+                retVal = processDecodedChunk(s, start, end, value, binaryFileType);
             }
+        } else if (start != s.end()) {
+            dbgTrace(D_WAAP_BASE64) << " ===isBase64PrefixProcessingOK===: chunk not detected."
+                                        " searching for known file header only";
+            end = (start + MAX_HEADER_LOOKUP < s.end()) ? start + MAX_HEADER_LOOKUP : s.end();
+            processDecodedChunk(s, start, end, value, binaryFileType);
+            value.clear();
+            return binaryFileType != Waap::Util::BinaryFileType::FILE_TYPE_NONE;
         }
     }
-    return retVal;
+    return retVal != B64_DECODE_INVALID;
 }
 
 base64_variants b64Test (
         const string &s,
         string &key,
-        string &value)
+        string &value,
+        BinaryFileType &binaryFileType)
 {
 
     key.clear();
     bool retVal;
+    binaryFileType = Waap::Util::BinaryFileType::FILE_TYPE_NONE;
 
     dbgTrace(D_WAAP_BASE64) << " ===b64Test===: string =  " << s
             << " key = " << key << " value = " << value;
@@ -1397,7 +1434,7 @@ base64_variants b64Test (
     }
 
     dbgTrace(D_WAAP_BASE64) << " ===b64Test===: after processing key = '" << key << "'";
-    bool found = isBase64PrefixProcessingOK(s, prefix_decoded_val);
+    bool found = isBase64PrefixProcessingOK(s, prefix_decoded_val, binaryFileType);
     dbgTrace(D_WAAP_BASE64) << " ===b64Test===: after prefix test found = "
             << found << " new value is '" << prefix_decoded_val << "' - done";
     if (found) {
@@ -1421,7 +1458,7 @@ base64_variants b64Test (
         if ((s.end() - start) % 4 != 0) {
             key.clear();
             value.clear();
-            return CONTINUE_AS_IS;;
+            return CONTINUE_AS_IS;
         }
     }
     else {
@@ -1443,7 +1480,7 @@ base64_variants b64Test (
             key.pop_back();
             dbgTrace(D_WAAP_BASE64) << " ===b64Test===: FINAL key = '" << key << "'";
         }
-        retVal = decodeBase64Chunk(s, start, s.end(), value);
+        retVal = decodeBase64Chunk(s, start, s.end(), value) != B64_DECODE_INVALID;
 
         dbgTrace(D_WAAP_BASE64) << " ===b64Test===: After testing and conversion value = "
                 << value << "retVal = '" << retVal <<"'";

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -34,6 +34,7 @@
 
 enum base64_variants {SINGLE_B64_CHUNK_CONVERT, KEY_VALUE_B64_PAIR, CONTINUE_AS_IS};
 enum base64_stage {BEFORE_EQUAL, EQUAL, DONE, MISDETECT};
+enum base64_decode_status {B64_DECODE_INVALID, B64_DECODE_OK, B64_DECODE_INCOMPLETE};
 
 // This is portable version of stricmp(), which is non-standard function (not even in C).
 // Contrary to stricmp(), for a slight optimization, s2 is ASSUMED to be already in lowercase.
@@ -858,12 +859,13 @@ void unescapeUnicode(std::string &text);
 // Try to find and decode UTF7 chunks
 std::string filterUTF7(const std::string &text);
 
-bool
+base64_decode_status
 decodeBase64Chunk(
     const std::string &value,
     std::string::const_iterator it,
     std::string::const_iterator end,
-    std::string &decoded);
+    std::string &decoded,
+    bool clear_on_error = true);
 
 bool
 b64DecodeChunk(
@@ -889,6 +891,13 @@ namespace Util {
             std::string &key,
             std::string &value);
 
+    enum BinaryFileType {
+        FILE_TYPE_NONE,
+        FILE_TYPE_PNG,
+        FILE_TYPE_JPEG,
+        FILE_TYPE_PDF
+    };
+
     void b64Decode(
             const std::string &s,
             RegexSubCallback_f cb,
@@ -899,7 +908,8 @@ namespace Util {
     base64_variants b64Test (
             const std::string &s,
             std::string &key,
-            std::string &value);
+            std::string &value,
+            BinaryFileType &binaryFileType);
 
     // The original stdlib implementation of isalpha() supports locale settings which we do not really need.
     // It is also proven to contribute to slow performance in some of the algorithms using it.

components/security_apps/waap/waap_clib/oasu_key_types.h

@@ -14,7 +14,8 @@ enum SchemaKeyType
     StartObjectKeyName,
     StartArrayKeyName,
     EndArrayKeyName,
-    OtherKey
+    OtherKey,
+    JsonFailure
 };
 
 #endif // __OA_SCHEMA_UPDATER_KEYS_H__

components/security_apps/waap/waap_component_impl.cc

@@ -42,6 +42,7 @@ using namespace std;
 USE_DEBUG_FLAG(D_WAAP);
 USE_DEBUG_FLAG(D_WAAP_ULIMITS);
 USE_DEBUG_FLAG(D_OA_SCHEMA_UPDATER);
+USE_DEBUG_FLAG(D_NGINX_EVENTS);
 
 WaapComponent::Impl::Impl() :
     pending_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT),
@@ -124,7 +125,7 @@ WaapComponent::Impl::getListenerName() const
 EventVerdict
 WaapComponent::Impl::respond(const NewHttpTransactionEvent &event)
 {
-    dbgTrace(D_WAAP) << " * \e[32mNGEN_EVENT: NewTransactionEvent\e[0m";
+    dbgTrace(D_NGINX_EVENTS) << " * \e[32mNGEN_EVENT: NewTransactionEvent\e[0m";
 
     if (waapStateTable->hasState<Waf2Transaction>()) {
         dbgWarning(D_WAAP) << " * \e[31 -- NewTransactionEvent called twice on same entry \e[0m";
@@ -202,7 +203,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event)
     auto &header_name = event.getKey();
     auto &header_value = event.getValue();
 
-    dbgTrace(D_WAAP)
+    dbgTrace(D_NGINX_EVENTS)
         << " * \e[32mNGEN_EVENT: HttpHeaderRequest event: "
         << string(header_name)
         << ": "
@@ -210,7 +211,7 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event)
         << "\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP)
+        dbgWarning(D_NGINX_EVENTS)
             << " * \e[31mNGEN_EVENT: http_header - "
             << "failed to get waf2 transaction, state not exist\e[0m";
 
@@ -257,10 +258,10 @@ WaapComponent::Impl::respond(const HttpRequestHeaderEvent &event)
 EventVerdict
 WaapComponent::Impl::respond(const HttpRequestBodyEvent &event)
 {
-    dbgTrace(D_WAAP) << " * \e[32mNGEN_EVENT: HttpBodyRequest data buffer event\e[0m";
+    dbgTrace(D_NGINX_EVENTS) << " * \e[32mNGEN_EVENT: HttpBodyRequest data buffer event\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP) <<
+        dbgWarning(D_NGINX_EVENTS) <<
             " * \e[31mNGEN_EVENT: data buffer - failed to get waf2 transaction, state not exist\e[0m";
         return drop_response;
     }
@@ -295,10 +296,10 @@ WaapComponent::Impl::respond(const HttpRequestBodyEvent &event)
 EventVerdict
 WaapComponent::Impl::respond(const EndRequestEvent &)
 {
-    dbgTrace(D_WAAP) << " * \e[32mNGEN_EVENT: endRequest event\e[0m";
+    dbgTrace(D_NGINX_EVENTS) << " * \e[32mNGEN_EVENT: endRequest event\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP)
+        dbgWarning(D_NGINX_EVENTS)
                 << "* \e[31mNGEN_EVENT: endRequest - failed to get waf2 transaction, state does not exist\e[0m";
         return drop_response;
     }
@@ -333,13 +334,13 @@ WaapComponent::Impl::respond(const EndRequestEvent &)
 EventVerdict
 WaapComponent::Impl::respond(const ResponseCodeEvent &event)
 {
-    dbgTrace(D_WAAP)
+    dbgTrace(D_NGINX_EVENTS)
         << " * \e[32mNGEN_EVENT: ResponseCodeTransactionEvent event code = "
         << event.getResponseCode()
         << "\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP)
+        dbgWarning(D_NGINX_EVENTS)
                 << " * \e[31mNGEN_EVENT: ResponseCodeTransactionEvent - failed to get waf2 transaction, "
                 << "state does not exist\e[0m";
         return drop_response;
@@ -385,7 +386,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
     auto &header_name = event.getKey();
     auto &header_value = event.getValue();
 
-    dbgTrace(D_WAAP)
+    dbgTrace(D_NGINX_EVENTS)
         << " * \e[32mNGEN_EVENT: HttpHeaderResponse event: "
         << string(header_name)
         << ": "
@@ -393,7 +394,7 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
         << "\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP)
+        dbgWarning(D_NGINX_EVENTS)
             << " * \e[31mNGEN_EVENT: HttpHeaderResponse - "
             << "failed to get waf2 transaction, state does not exist\e[0m";
         return drop_response;
@@ -491,10 +492,10 @@ WaapComponent::Impl::respond(const HttpResponseHeaderEvent &event)
 EventVerdict
 WaapComponent::Impl::respond(const HttpResponseBodyEvent &event)
 {
-    dbgTrace(D_WAAP) << " * \e[32mNGEN_EVENT: HttpBodyResponse data buffer event\e[0m";
+    dbgTrace(D_NGINX_EVENTS) << " * \e[32mNGEN_EVENT: HttpBodyResponse data buffer event\e[0m";
 
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP) <<
+        dbgWarning(D_NGINX_EVENTS) <<
             " * \e[31mNGEN_EVENT: HttpBodyResponse - failed to get waf2 transaction, state does not exist\e[0m";
         return drop_response;
     }
@@ -591,7 +592,7 @@ EventVerdict
 WaapComponent::Impl::respond(const EndTransactionEvent &)
 {
     if (!waapStateTable->hasState<Waf2Transaction>()) {
-        dbgWarning(D_WAAP) <<
+        dbgWarning(D_NGINX_EVENTS) <<
             " * \e[31mNGEN_EVENT: endTransaction - failed to get waf2 transaction, state does not exist\e[0m";
         return EventVerdict(drop_response);
     }

components/signal_handler/signal_handler.cc

@@ -136,8 +136,10 @@ private:
                 string service_underscore_name = service_name;
                 replace(service_underscore_name.begin(), service_underscore_name.end(), ' ', '_');
 
+                string logFilesPath = getLogFilesPathConfig();
+
                 trace_file_path = getConfigurationWithDefault<string>(
-                    "/var/log/nano_agent/trace_export_files/" + service_underscore_name + "_trace_file.dbg",
+                    logFilesPath + "/nano_agent/trace_export_files/" + service_underscore_name + "_trace_file.dbg",
                     "SignalHandler",
                     "outputFilePath"
                 );

components/utils/http_transaction_data/http_transaction_data.cc

@@ -52,6 +52,7 @@ const string HttpTransactionData::req_headers        = "transaction_request_head
 const string HttpTransactionData::req_body           = "transaction_request_body";
 const string HttpTransactionData::source_identifier  = "sourceIdentifiers";
 const string HttpTransactionData::proxy_ip_ctx       = "proxy_ip";
+const string HttpTransactionData::xff_vals_ctx       = "xff_vals";
 
 const CompressionType HttpTransactionData::default_response_content_encoding = CompressionType::NO_COMPRESSION;
 

config/k8s/latest/open-appsec-k8s-default-config-v1beta2.yaml

@@ -0,0 +1,126 @@
+# open-appsec default declarative configuration file
+# based on schema version: "v1beta2"
+# more information on declarative configuration: https://docs.openappsec.io
+
+apiVersion: openappsec.io/v1beta2
+kind: Policy
+metadata:
+  name: default-policy
+spec:
+  default:
+    # start in detect-learn and move to prevent-learn based on learning progress 
+    mode: detect-learn
+    threatPreventionPractices:
+    - default-threat-prevention-practice
+    accessControlPractices:
+    - default-access-control-practice
+    customResponses: default-web-user-response
+    triggers:
+    - default-log-trigger
+  specificRules:
+    - host: www.example.com
+      # this is an example for specific rule, adjust the values as required for the protected app
+      mode: detect-learn
+      threatPreventionPractices:
+      - default-threat-prevention-practice
+      accessControlPractices:
+      - default-access-control-practice
+      triggers:
+      - default-log-trigger
+---
+apiVersion: openappsec.io/v1beta2
+kind: ThreatPreventionPractice
+metadata:
+  name: default-threat-prevention-practice
+spec:
+  practiceMode: inherited
+  webAttacks:
+    overrideMode: inherited
+    minimumConfidence: high
+  intrusionPrevention:
+  # intrusion prevention (IPS) requires "Premium Edition"
+    overrideMode: inherited
+    maxPerformanceImpact: medium
+    minSeverityLevel: medium
+    minCveYear: 2016
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  fileSecurity:
+  # file security requires "Premium Edition"
+    overrideMode: inherited
+    minSeverityLevel: medium
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  snortSignatures:
+    # you must specify snort signatures in configmap or file to activate snort inspection
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  openapiSchemaValidation: # schema validation requires "Premium Edition"
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  antiBot: # antibot requires "Premium Edition"
+    overrideMode: inherited
+    injectedUris: []
+    validatedUris: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: AccessControlPractice
+metadata:
+  name: default-access-control-practice
+spec:
+  practiceMode: inherited
+  rateLimit:
+  # specify one or more rules below to use rate limiting
+    overrideMode: inherited
+    rules: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: LogTrigger
+metadata:
+  name: default-log-trigger
+spec:
+  accessControlLogging:
+    allowEvents: false
+    dropEvents: true
+  appsecLogging:
+    detectEvents: true
+    preventEvents: true
+    allWebRequests: false
+  extendedLogging:
+    urlPath: true
+    urlQuery: true
+    httpHeaders: false
+    requestBody: false      
+  additionalSuspiciousEventsLogging:
+    enabled: true
+    minSeverity: high
+    responseBody: false
+    responseCode: true
+  logDestination:
+    cloud: true
+    logToAgent: false
+    stdout:
+      format: json
+    
+---
+apiVersion: openappsec.io/v1beta2
+kind: CustomResponse
+metadata:
+  name: default-web-user-response
+spec:
+  mode: response-code-only
+  httpResponseCode: 403

config/k8s/latest/open-appsec-k8s-prevent-config-v1beta2.yaml

@@ -0,0 +1,126 @@
+# open-appsec default declarative configuration file
+# based on schema version: "v1beta2"
+# more information on declarative configuration: https://docs.openappsec.io
+
+apiVersion: openappsec.io/v1beta2
+kind: Policy
+metadata:
+  name: default-policy
+spec:
+  default:
+    # start in prevent-learn
+    mode: prevent-learn
+    threatPreventionPractices:
+    - default-threat-prevention-practice
+    accessControlPractices:
+    - default-access-control-practice
+    customResponses: default-web-user-response
+    triggers:
+    - default-log-trigger
+  specificRules:
+    - host: www.example.com
+      # this is an example for specific rule, adjust the values as required for the protected app
+      mode: prevent-learn
+      threatPreventionPractices:
+      - default-threat-prevention-practice
+      accessControlPractices:
+      - default-access-control-practice
+      triggers:
+      - default-log-trigger
+---
+apiVersion: openappsec.io/v1beta2
+kind: ThreatPreventionPractice
+metadata:
+  name: default-threat-prevention-practice
+spec:
+  practiceMode: inherited
+  webAttacks:
+    overrideMode: inherited
+    minimumConfidence: high
+  intrusionPrevention:
+  # intrusion prevention (IPS) requires "Premium Edition"
+    overrideMode: inherited
+    maxPerformanceImpact: medium
+    minSeverityLevel: medium
+    minCveYear: 2016
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  fileSecurity:
+  # file security requires "Premium Edition"
+    overrideMode: inherited
+    minSeverityLevel: medium
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  snortSignatures:
+    # you must specify snort signatures in configmap or file to activate snort inspection
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  openapiSchemaValidation: # schema validation requires "Premium Edition"
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  antiBot: # antibot requires "Premium Edition"
+    overrideMode: inherited
+    injectedUris: []
+    validatedUris: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: AccessControlPractice
+metadata:
+  name: default-access-control-practice
+spec:
+  practiceMode: inherited
+  rateLimit:
+  # specify one or more rules below to use rate limiting
+    overrideMode: inherited
+    rules: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: LogTrigger
+metadata:
+  name: default-log-trigger
+spec:
+  accessControlLogging:
+    allowEvents: false
+    dropEvents: true
+  appsecLogging:
+    detectEvents: true
+    preventEvents: true
+    allWebRequests: false
+  extendedLogging:
+    urlPath: true
+    urlQuery: true
+    httpHeaders: false
+    requestBody: false      
+  additionalSuspiciousEventsLogging:
+    enabled: true
+    minSeverity: high
+    responseBody: false
+    responseCode: true
+  logDestination:
+    cloud: true
+    logToAgent: false
+    stdout:
+      format: json
+    
+---
+apiVersion: openappsec.io/v1beta2
+kind: CustomResponse
+metadata:
+  name: default-web-user-response
+spec:
+  mode: response-code-only
+  httpResponseCode: 403

config/k8s/v1beta1/open-appsec-k8s-default-config-v1beta1.yaml

@@ -0,0 +1,13 @@
+apiVersion: openappsec.io/v1beta1
+kind: Policy
+metadata:
+    name: open-appsec-best-practice-policy
+spec:
+    default:
+        mode: detect-learn
+        practices: [appsec-best-practice]
+        triggers: [appsec-log-trigger]
+        custom-response: 403-forbidden
+        source-identifiers: ""
+        trusted-sources: ""
+        exceptions: []

config/k8s/v1beta1/open-appsec-k8s-prevent-config-v1beta1.yaml

@@ -0,0 +1,13 @@
+apiVersion: openappsec.io/v1beta1
+kind: Policy
+metadata:
+    name: open-appsec-best-practice-policy
+spec:
+    default:
+        mode: prevent-learn
+        practices: [appsec-best-practice]
+        triggers: [appsec-log-trigger]
+        custom-response: 403-forbidden
+        source-identifiers: ""
+        trusted-sources: ""
+        exceptions: []

config/k8s/v1beta2/open-appsec-k8s-default-config-v1beta2.yaml

@@ -0,0 +1,126 @@
+# open-appsec default declarative configuration file
+# based on schema version: "v1beta2"
+# more information on declarative configuration: https://docs.openappsec.io
+
+apiVersion: openappsec.io/v1beta2
+kind: Policy
+metadata:
+  name: default-policy
+spec:
+  default:
+    # start in detect-learn and move to prevent-learn based on learning progress 
+    mode: detect-learn
+    threatPreventionPractices:
+    - default-threat-prevention-practice
+    accessControlPractices:
+    - default-access-control-practice
+    customResponses: default-web-user-response
+    triggers:
+    - default-log-trigger
+  specificRules:
+    - host: www.example.com
+      # this is an example for specific rule, adjust the values as required for the protected app
+      mode: detect-learn
+      threatPreventionPractices:
+      - default-threat-prevention-practice
+      accessControlPractices:
+      - default-access-control-practice
+      triggers:
+      - default-log-trigger
+---
+apiVersion: openappsec.io/v1beta2
+kind: ThreatPreventionPractice
+metadata:
+  name: default-threat-prevention-practice
+spec:
+  practiceMode: inherited
+  webAttacks:
+    overrideMode: inherited
+    minimumConfidence: high
+  intrusionPrevention:
+  # intrusion prevention (IPS) requires "Premium Edition"
+    overrideMode: inherited
+    maxPerformanceImpact: medium
+    minSeverityLevel: medium
+    minCveYear: 2016
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  fileSecurity:
+  # file security requires "Premium Edition"
+    overrideMode: inherited
+    minSeverityLevel: medium
+    highConfidenceEventAction: inherited
+    mediumConfidenceEventAction: inherited
+    lowConfidenceEventAction: detect
+  snortSignatures:
+    # you must specify snort signatures in configmap or file to activate snort inspection
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  openapiSchemaValidation: # schema validation requires "Premium Edition"
+    overrideMode: inherited
+    configmap: []
+    # relevant for deployments on kubernetes
+    # 0 or 1 configmaps supported in array
+    files: []
+    # relevant for docker and linux embedded deployments
+    # 0 or 1 files supported in array
+  antiBot: # antibot requires "Premium Edition"
+    overrideMode: inherited
+    injectedUris: []
+    validatedUris: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: AccessControlPractice
+metadata:
+  name: default-access-control-practice
+spec:
+  practiceMode: inherited
+  rateLimit:
+  # specify one or more rules below to use rate limiting
+    overrideMode: inherited
+    rules: []
+
+---
+apiVersion: openappsec.io/v1beta2
+kind: LogTrigger
+metadata:
+  name: default-log-trigger
+spec:
+  accessControlLogging:
+    allowEvents: false
+    dropEvents: true
+  appsecLogging:
+    detectEvents: true
+    preventEvents: true
+    allWebRequests: false
+  extendedLogging:
+    urlPath: true
+    urlQuery: true
+    httpHeaders: false
+    requestBody: false      
+  additionalSuspiciousEventsLogging:
+    enabled: true
+    minSeverity: high
+    responseBody: false
+    responseCode: true
+  logDestination:
+    cloud: true
+    logToAgent: false
+    stdout:
+      format: json
+    
+---
+apiVersion: openappsec.io/v1beta2
+kind: CustomResponse
+metadata:
+  name: default-web-user-response
+spec:
+  mode: response-code-only
+  httpResponseCode: 403