Add files via upload

Update local_policy.yaml

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required (VERSION 2.8.4)
 project (ngen)
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall -Wno-terminate")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fPIC -Wall -Wno-terminate")
 
 execute_process(COMMAND grep -c "Alpine Linux" /etc/os-release OUTPUT_VARIABLE IS_ALPINE)
 if(NOT IS_ALPINE EQUAL "0")

attachments/nginx/nginx_attachment_util/nginx_attachment_util.cc

@@ -155,6 +155,24 @@ getWaitingForVerdictThreadTimeout()
     return conf_data.getNumericalValue("waiting_for_verdict_thread_timeout_msec");
 }
 
+unsigned int
+getMinRetriesForVerdict()
+{
+    return conf_data.getNumericalValue("min_retries_for_verdict");
+}
+
+unsigned int
+getMaxRetriesForVerdict()
+{
+    return conf_data.getNumericalValue("max_retries_for_verdict");
+}
+
+unsigned int
+getReqBodySizeTrigger()
+{
+    return conf_data.getNumericalValue("body_size_trigger");
+}
+
 int
 isIPAddress(c_str ip_str)
 {

attachments/nginx/nginx_attachment_util/nginx_attachment_util_ut/nginx_attachment_util_ut.cc

@@ -63,31 +63,37 @@ TEST_F(HttpAttachmentUtilTest, GetValidAttachmentConfiguration)
             "\"waiting_for_verdict_thread_timeout_msec\": 75,\n"
             "\"req_header_thread_timeout_msec\": 10,\n"
             "\"ip_ranges\": " + createIPRangesString(ip_ranges) + ",\n"
-            "\"static_resources_path\": \"" + static_resources_path + "\""
+            "\"static_resources_path\": \"" + static_resources_path + "\",\n"
+            "\"min_retries_for_verdict\": 1,\n"
+            "\"max_retries_for_verdict\": 3,\n"
+            "\"body_size_trigger\": 777\n"
         "}\n";
     ofstream valid_configuration_file(attachment_configuration_file_name);
     valid_configuration_file << valid_configuration;
     valid_configuration_file.close();
 
     EXPECT_EQ(initAttachmentConfig(attachment_configuration_file_name.c_str()), 1);
-    EXPECT_EQ(getDbgLevel(), 2);
+    EXPECT_EQ(getDbgLevel(), 2u);
     EXPECT_EQ(getStaticResourcesPath(), static_resources_path);
     EXPECT_EQ(isFailOpenMode(), 0);
-    EXPECT_EQ(getFailOpenTimeout(), 1234);
+    EXPECT_EQ(getFailOpenTimeout(), 1234u);
     EXPECT_EQ(isFailOpenHoldMode(), 1);
-    EXPECT_EQ(getFailOpenHoldTimeout(), 4321);
+    EXPECT_EQ(getFailOpenHoldTimeout(), 4321u);
     EXPECT_EQ(isFailOpenOnSessionLimit(), 1);
-    EXPECT_EQ(getMaxSessionsPerMinute(), 0);
-    EXPECT_EQ(getNumOfNginxIpcElements(), 200);
-    EXPECT_EQ(getKeepAliveIntervalMsec(), 10000);
-    EXPECT_EQ(getResProccessingTimeout(), 420);
-    EXPECT_EQ(getReqProccessingTimeout(), 42);
-    EXPECT_EQ(getRegistrationThreadTimeout(), 101);
-    EXPECT_EQ(getReqHeaderThreadTimeout(), 10);
-    EXPECT_EQ(getReqBodyThreadTimeout(), 155);
-    EXPECT_EQ(getResHeaderThreadTimeout(), 1);
-    EXPECT_EQ(getResBodyThreadTimeout(), 0);
-    EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75);
+    EXPECT_EQ(getMaxSessionsPerMinute(), 0u);
+    EXPECT_EQ(getNumOfNginxIpcElements(), 200u);
+    EXPECT_EQ(getKeepAliveIntervalMsec(), 10000u);
+    EXPECT_EQ(getResProccessingTimeout(), 420u);
+    EXPECT_EQ(getReqProccessingTimeout(), 42u);
+    EXPECT_EQ(getRegistrationThreadTimeout(), 101u);
+    EXPECT_EQ(getReqHeaderThreadTimeout(), 10u);
+    EXPECT_EQ(getReqBodyThreadTimeout(), 155u);
+    EXPECT_EQ(getResHeaderThreadTimeout(), 1u);
+    EXPECT_EQ(getResBodyThreadTimeout(), 0u);
+    EXPECT_EQ(getMinRetriesForVerdict(), 1u);
+    EXPECT_EQ(getMaxRetriesForVerdict(), 3u);
+    EXPECT_EQ(getReqBodySizeTrigger(), 777u);
+    EXPECT_EQ(getWaitingForVerdictThreadTimeout(), 75u);
     EXPECT_EQ(getInspectionMode(), ngx_http_inspection_mode::BLOCKING_THREAD);
 
     EXPECT_EQ(isDebugContext("1.2.3.4", "5.6.7.8", 80, "GET", "test", "/abc"), 1);

build_system/apisix/docker-compose.yaml

@@ -1,15 +1,15 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+# Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
 
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 version: "3"
 

build_system/docker/entry.sh

@@ -11,6 +11,7 @@ var_fog_address=
 var_proxy=
 var_mode=
 var_token=
+var_ignore=
 init=
 
 if [ ! -f /nano-service-installers/$ORCHESTRATION_INSTALLATION_SCRIPT ]; then
@@ -33,6 +34,8 @@ while true; do
         var_proxy="$1"
     elif [ "$1" == "--hybrid-mode" ] || [ "$1" == "--standalone" ]; then
         var_mode="--hybrid_mode"
+    elif [ "$1" == "--no-upgrade" ]; then
+        var_ignore="--ignore all"
     elif [ "$1" == "--token" ]; then
         shift
         var_token="$1"
@@ -41,12 +44,16 @@ while true; do
 done
 
 if [ -z $var_token ] && [ $var_mode != "--hybrid_mode" ]; then
-    echo "Error: Token was not provided as input argument."
-    exit 1
+    var_token=$(env | grep 'AGENT_TOKEN=' | cut -d'=' -f2-)
+    if  [ -z $var_token ]; then
+        echo "Error: Token was not provided as input argument."
+        exit 1
+    fi
 fi
 
 orchestration_service_installation_flags="--container_mode --skip_registration"
 if [ ! -z $var_token ]; then
+    export AGENT_TOKEN="$var_token"
     orchestration_service_installation_flags="$orchestration_service_installation_flags --token $var_token"
 fi
 if [ ! -z $var_fog_address ]; then
@@ -59,6 +66,9 @@ fi
 if [ ! -z $var_mode ]; then
     orchestration_service_installation_flags="$orchestration_service_installation_flags $var_mode"
 fi
+if [ ! -z "$var_ignore" ]; then
+    orchestration_service_installation_flags="$orchestration_service_installation_flags $var_ignore"
+fi
 
 
 /nano-service-installers/$ORCHESTRATION_INSTALLATION_SCRIPT --install $orchestration_service_installation_flags

components/CMakeLists.txt

@@ -1,10 +1,8 @@
-add_subdirectory(report_messaging)
 add_subdirectory(http_manager)
 add_subdirectory(signal_handler)
 add_subdirectory(gradual_deployment)
 add_subdirectory(packet)
 add_subdirectory(pending_key)
-add_subdirectory(health_check_manager)
 
 add_subdirectory(utils)
 add_subdirectory(attachment-intakers)

components/attachment-intakers/attachment_registrator/attachment_registrator.cc

@@ -39,6 +39,8 @@ USE_DEBUG_FLAG(D_ATTACHMENT_REGISTRATION);
 
 using namespace std;
 
+static const AlertInfo alert(AlertTeam::CORE, "attachment registrator");
+
 class AttachmentRegistrator::Impl
 {
 public:
@@ -163,7 +165,7 @@ private:
                 break;
             }
             default:
-                dbgAssert(false) << "Unsupported Attachment " << static_cast<int>(type);
+                dbgAssert(false) << alert << "Unsupported Attachment " << static_cast<int>(type);
         }
 
         if (!family_id.empty()) handler_path << family_id << "_";
@@ -175,7 +177,9 @@ private:
     string
     genRegCommand(const string &family_id, const uint num_of_members, const AttachmentType type) const
     {
-        dbgAssert(num_of_members > 0) << "Failed to generate a registration command for an empty group of attachments";
+        dbgAssert(num_of_members > 0)
+            << alert
+            << "Failed to generate a registration command for an empty group of attachments";
 
         static const string registration_format = "/etc/cp/watchdog/cp-nano-watchdog --register ";
         stringstream registration_command;
@@ -187,7 +191,7 @@ private:
                 break;
             }
             default:
-                dbgAssert(false) << "Unsupported Attachment " << static_cast<int>(type);
+                dbgAssert(false) << alert << "Unsupported Attachment " << static_cast<int>(type);
         }
 
         if (!family_id.empty()) registration_command << " --family " << family_id;
@@ -265,7 +269,7 @@ private:
             return -1;
         }
 
-        dbgAssert(new_socket.unpack() > 0) << "Generated socket is OK yet negative";
+        dbgAssert(new_socket.unpack() > 0) << alert << "Generated socket is OK yet negative";
         return new_socket.unpack();
     }
 
@@ -281,7 +285,7 @@ private:
         }
 
         I_Socket::socketFd client_socket = accepted_socket.unpack();
-        dbgAssert(client_socket > 0) << "Generated client socket is OK yet negative";
+        dbgAssert(client_socket > 0) << alert << "Generated client socket is OK yet negative";
         auto close_socket_on_exit = make_scope_exit([&]() { i_socket->closeSocket(client_socket); });
 
         Maybe<uint8_t> attachment_id = readNumericParam(client_socket);
@@ -375,7 +379,7 @@ private:
         }
 
         I_Socket::socketFd client_socket = accepted_socket.unpack();
-        dbgAssert(client_socket > 0) << "Generated client socket is OK yet negative";
+        dbgAssert(client_socket > 0) << alert << "Generated client socket is OK yet negative";
         auto close_socket_on_exit = make_scope_exit([&]() { i_socket->closeSocket(client_socket); });
 
         Maybe<AttachmentType> attachment_type = readAttachmentType(client_socket);

components/attachment-intakers/nginx_attachment/nginx_attachment.cc

@@ -76,6 +76,7 @@ using namespace std;
 using ChunkType = ngx_http_chunk_type_e;
 
 static const uint32_t corrupted_session_id = CORRUPTED_SESSION_ID;
+static const AlertInfo alert(AlertTeam::CORE, "nginx attachment");
 
 class FailopenModeListener : public Listener<FailopenModeEvent>
 {
@@ -410,7 +411,10 @@ private:
     bool
     registerAttachmentProcess(uint32_t nginx_user_id, uint32_t nginx_group_id, I_Socket::socketFd new_socket)
     {
-        dbgAssert(server_sock > 0) << "Registration attempt occurred while registration socket is uninitialized";
+        dbgAssert(server_sock > 0)
+        << alert
+        << "Registration attempt occurred while registration socket is uninitialized";
+
 #ifdef FAILURE_TEST
         bool did_fail_on_purpose = false;
 #endif
@@ -802,10 +806,10 @@ private:
         case ChunkType::HOLD_DATA:
             return "HOLD_DATA";
         case ChunkType::COUNT:
-            dbgAssert(false) << "Invalid 'COUNT' ChunkType";
+            dbgAssert(false) << alert << "Invalid 'COUNT' ChunkType";
             return "";
         }
-        dbgAssert(false) << "ChunkType was not handled by the switch case";
+        dbgAssert(false) << alert << "ChunkType was not handled by the switch case";
         return "";
     }
 
@@ -1131,18 +1135,26 @@ private:
             "webUserResponse"
         );
 
+        bool remove_event_id_param =
+            getProfileAgentSettingWithDefault<string>("false", "nginxAttachment.removeRedirectEventId") == "true";
+
         string uuid;
+        string redirectUrl;
         if (i_transaction_table->hasState<NginxAttachmentOpaque>()) {
             NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
             uuid = opaque.getSessionUUID();
         }
-        web_response_data.uuid_size =
-            string("Incident Id: ").length() + uuid.size();
+        web_response_data.uuid_size = uuid.size();
 
         if (web_trigger_conf.getDetailsLevel() == "Redirect") {
             web_response_data.response_data.redirect_data.redirect_location_size =
                 web_trigger_conf.getRedirectURL().size();
-            web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0;
+            bool add_event = web_trigger_conf.getAddEventId();
+            if (add_event && !remove_event_id_param) {
+                web_response_data.response_data.redirect_data.redirect_location_size +=
+                    strlen("?event_id=") + uuid.size();
+            }
+            web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0;
             web_response_data.web_repsonse_type = static_cast<uint8_t>(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE);
         } else {
             web_response_data.response_data.custom_response_data.title_size =
@@ -1156,8 +1168,13 @@ private:
         verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t));
 
         if (web_trigger_conf.getDetailsLevel() == "Redirect") {
-            verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getRedirectURL().data()));
-            verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size());
+            redirectUrl = web_trigger_conf.getRedirectURL();
+            if (!remove_event_id_param && web_trigger_conf.getAddEventId()) {
+                redirectUrl += "?event-id=" + uuid;
+            }
+
+            verdict_data.push_back(reinterpret_cast<const char *>(redirectUrl.data()));
+            verdict_data_sizes.push_back(redirectUrl.size());
         } else {
             verdict_data.push_back(reinterpret_cast<const char *>(web_trigger_conf.getResponseTitle().data()));
             verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size());
@@ -1583,7 +1600,7 @@ private:
             case WAIT:
                 return "WAIT";
         }
-        dbgAssert(false) << "Invalid EventVerdict enum: " << static_cast<int>(verdict.getVerdict());
+        dbgAssert(false) << alert << "Invalid EventVerdict enum: " << static_cast<int>(verdict.getVerdict());
         return string();
     }
 
@@ -1634,13 +1651,14 @@ private:
             return false;
         }
 
-        dbgAssert(sock.unpack() > 0) << "The generated server socket is OK, yet negative";
+        dbgAssert(sock.unpack() > 0) << alert << "The generated server socket is OK, yet negative";
         server_sock = sock.unpack();
 
         I_MainLoop::Routine accept_attachment_routine =
             [this] ()
             {
                 dbgAssert(inst_awareness->getUniqueID().ok())
+                    << alert
                     << "NGINX attachment Initialized without Instance Awareness";
 
                 bool did_fail_on_purpose = false;
@@ -1653,7 +1671,7 @@ private:
                     << (did_fail_on_purpose ? "Intentional Failure" : new_sock.getErr());
                     return;
                 }
-                dbgAssert(new_sock.unpack() > 0) << "The generated client socket is OK, yet negative";
+                dbgAssert(new_sock.unpack() > 0) << alert << "The generated client socket is OK, yet negative";
                 I_Socket::socketFd new_attachment_socket = new_sock.unpack();
 
                 Maybe<string> uid = getUidFromSocket(new_attachment_socket);
@@ -1699,7 +1717,7 @@ private:
                 }
             };
         mainloop->addFileRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             server_sock,
             accept_attachment_routine,
             "Nginx Attachment registration listener",
@@ -1712,7 +1730,9 @@ private:
     Maybe<string>
     getUidFromSocket(I_Socket::socketFd new_attachment_socket)
     {
-        dbgAssert(server_sock > 0) << "Registration attempt occurred while registration socket is uninitialized";
+        dbgAssert(server_sock > 0)
+            << alert
+            << "Registration attempt occurred while registration socket is uninitialized";
 
         bool did_fail_on_purpose = false;
         DELAY_IF_NEEDED(IntentionalFailureHandler::FailureType::ReceiveDataFromSocket);

components/attachment-intakers/nginx_attachment/nginx_attachment_config.cc

@@ -42,6 +42,7 @@ HttpAttachmentConfig::init()
     setNumOfNginxIpcElements();
     setDebugByContextValues();
     setKeepAliveIntervalMsec();
+    setRetriesForVerdict();
 }
 
 bool
@@ -215,6 +216,31 @@ HttpAttachmentConfig::setFailOpenTimeout()
     conf_data.setNumericalValue("nginx_inspection_mode", inspection_mode);
 }
 
+void
+HttpAttachmentConfig::setRetriesForVerdict()
+{
+    conf_data.setNumericalValue("min_retries_for_verdict", getAttachmentConf<uint>(
+        3,
+        "agent.minRetriesForVerdict.nginxModule",
+        "HTTP manager",
+        "Min retries for verdict"
+    ));
+
+    conf_data.setNumericalValue("max_retries_for_verdict", getAttachmentConf<uint>(
+        15,
+        "agent.maxRetriesForVerdict.nginxModule",
+        "HTTP manager",
+        "Max retries for verdict"
+    ));
+
+    conf_data.setNumericalValue("body_size_trigger", getAttachmentConf<uint>(
+        200000,
+        "agent.reqBodySizeTrigger.nginxModule",
+        "HTTP manager",
+        "Request body size trigger"
+    ));
+}
+
 void
 HttpAttachmentConfig::setFailOpenWaitMode()
 {

components/attachment-intakers/nginx_attachment/nginx_attachment_config.h

@@ -70,6 +70,8 @@ private:
 
     void setDebugByContextValues();
 
+    void setRetriesForVerdict();
+
     WebTriggerConf web_trigger_conf;
     HttpAttachmentConfiguration conf_data;
 };

components/attachment-intakers/nginx_attachment/user_identifiers_config.cc

@@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector<CIDRSData> &cidr_values)
 }
 
 Maybe<string>
-UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
+UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const
 {
     vector<string> header_values = split(str);
 
@@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
     vector<string> xff_values = getHeaderValuesFromConfig("x-forwarded-for");
     vector<CIDRSData> cidr_values(xff_values.begin(), xff_values.end());
 
-    for (const string &value : header_values) {
-        if (!IPAddr::createIPAddr(value).ok()) {
-            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value;
+    for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) {
+        if (!IPAddr::createIPAddr(*it).ok()) {
+            dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it;
             return genError("Invalid IP address");
         }
-        if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found");
+        if (type == ExtractType::PROXYIP) continue;
+        if (!isIpTrusted(*it, cidr_values)) {
+            dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it;
+            return *it;
+        }
+    }
+
+    if (!IPAddr::createIPAddr(header_values[0]).ok()) {
+        dbgWarning(D_NGINX_ATTACHMENT_PARSER)
+            << "Invalid IP address found in the xff header IPs list: "
+            << header_values[0];
+        return genError("Invalid IP address");
     }
 
     return header_values[0];
@@ -306,22 +317,28 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const
 void
 UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, ExtractType type) const
 {
-    auto value = parseXForwardedFor(header.getValue());
+    auto i_transaction_table = Singleton::Consume<I_TableSpecific<SessionID>>::by<NginxAttachment>();
+    if (!i_transaction_table || !i_transaction_table->hasState<NginxAttachmentOpaque>()) {
+        dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Can't get the transaction table";
+        return;
+    }
+    NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
+    auto value = parseXForwardedFor(header.getValue(), type);
     if (!value.ok()) {
         dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header";
         return;
     };
-    auto i_transaction_table = Singleton::Consume<I_TableSpecific<SessionID>>::by<NginxAttachment>();
-    if (!i_transaction_table || !i_transaction_table->hasState<NginxAttachmentOpaque>()) {
-        dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Can't get the transaction table";
-        return;
-    }
-    NginxAttachmentOpaque &opaque = i_transaction_table->getState<NginxAttachmentOpaque>();
+
     if (type == ExtractType::SOURCEIDENTIFIER) {
         opaque.setSourceIdentifier(header.getKey(), value.unpack());
         dbgDebug(D_NGINX_ATTACHMENT_PARSER)
-            << "Added source identifir to XFF "
+            << "Added source identifier from XFF header"
             <<  value.unpack();
+        opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue());
+        opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack());
+        dbgTrace(D_NGINX_ATTACHMENT_PARSER)
+            << "XFF found, set ctx with value from header: "
+            << static_cast<string>(header.getValue());
     } else {
         opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack());
     }

components/gradual_deployment/gradual_deployment.cc

@@ -128,7 +128,7 @@ private:
                 break;
             }
             default:
-                dbgAssert(false) << "Unsupported IP type";
+                dbgAssert(false) << AlertInfo(AlertTeam::CORE, "gradual deployment") << "Unsupported IP type";
         }
         return address;
     }

components/health_check_manager/health_check_manager_ut/CMakeLists.txt

@@ -1,8 +0,0 @@
-include_directories(${CMAKE_SOURCE_DIR}/components/include)
-link_directories(${BOOST_ROOT}/lib)
-
-add_unit_test(
-    health_check_manager_ut
-    "health_check_manager_ut.cc"
-    "singleton;messaging;mainloop;health_check_manager;event_is;metric;-lboost_regex"
-)

components/http_manager/http_manager.cc

@@ -46,7 +46,10 @@ operator<<(ostream &os, const EventVerdict &event)
         case ngx_http_cp_verdict_e::TRAFFIC_VERDICT_WAIT: return os << "Wait";
     }
 
-    dbgAssert(false) << "Illegal Event Verdict value: " << static_cast<uint>(event.getVerdict());
+    dbgAssert(false)
+        << AlertInfo(AlertTeam::CORE, "http manager")
+        << "Illegal Event Verdict value: "
+        << static_cast<uint>(event.getVerdict());
     return os;
 }
 
@@ -321,8 +324,11 @@ private:
 
             state.setApplicationVerdict(respond.first, respond.second.getVerdict());
         }
-
-        return state.getCurrVerdict();
+        FilterVerdict aggregated_verdict = state.getCurrVerdict();
+        if (aggregated_verdict.getVerdict() == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+            SecurityAppsDropEvent(state.getCurrentDropVerdictCausers()).notify();
+        }
+        return aggregated_verdict;
     }
 
     static void

components/http_manager/http_manager_opaque.cc

@@ -69,6 +69,7 @@ HttpManagerOpaque::getCurrVerdict() const
                 break;
             default:
                 dbgAssert(false)
+                    << AlertInfo(AlertTeam::CORE, "http manager")
                     << "Received unknown verdict "
                     << static_cast<int>(app_verdic_pair.second);
         }
@@ -77,6 +78,25 @@ HttpManagerOpaque::getCurrVerdict() const
     return accepted_apps == applications_verdicts.size() ? ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT : verdict;
 }
 
+std::set<std::string>
+HttpManagerOpaque::getCurrentDropVerdictCausers() const
+{
+    std::set<std::string> causers;
+    if (manager_verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+        causers.insert(HTTP_MANAGER_NAME);
+    }
+    for (const auto &app_verdic_pair : applications_verdicts) {
+        bool was_dropped = app_verdic_pair.second == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+        dbgTrace(D_HTTP_MANAGER)
+            << "The verdict from: " << app_verdic_pair.first
+            << (was_dropped ? " is \"drop\"" : " is not \"drop\" ");
+        if (was_dropped) {
+            causers.insert(app_verdic_pair.first);
+        }
+    }
+    return causers;
+}
+
 void
 HttpManagerOpaque::saveCurrentDataToCache(const Buffer &full_data)
 {

components/http_manager/http_manager_opaque.h

@@ -20,6 +20,8 @@
 #include "table_opaque.h"
 #include "nginx_attachment_common.h"
 
+static const std::string HTTP_MANAGER_NAME = "HTTP Manager";
+
 class HttpManagerOpaque : public TableOpaqueSerialize<HttpManagerOpaque>
 {
 public:
@@ -30,6 +32,7 @@ public:
     void setManagerVerdict(ngx_http_cp_verdict_e verdict) { manager_verdict = verdict; }
     ngx_http_cp_verdict_e getManagerVerdict() const { return manager_verdict; }
     ngx_http_cp_verdict_e getCurrVerdict() const;
+    std::set<std::string> getCurrentDropVerdictCausers() const;
     void saveCurrentDataToCache(const Buffer &full_data);
     void setUserDefinedValue(const std::string &value) { user_defined_value = value; }
     Maybe<std::string> getUserDefinedValue() const { return user_defined_value; }

components/include/env_details.h

@@ -29,12 +29,15 @@ public:
 
     virtual EnvType getEnvType() override;
     virtual std::string getToken() override;
+    virtual std::string getNameSpace() override;
 
 private:
     std::string retrieveToken();
+    std::string retrieveNamespace();
     std::string readFileContent(const std::string &file_path);
 
     std::string token;
+    std::string agent_namespace;
     EnvType env_type;
 };
 

components/include/external_sdk_server.h

@@ -24,7 +24,8 @@ class ExternalSdkServer
         :
     public Component,
     Singleton::Provide<I_ExternalSdkServer>,
-    Singleton::Consume<I_RestApi>
+    Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Messaging>
 {
 public:
     ExternalSdkServer();

components/include/generic_rulebase/match_query.h

@@ -89,7 +89,9 @@ private:
     bool matchAttributesRegEx(const std::set<std::string> &values,
             std::set<std::string> &matched_override_keywords) const;
     bool matchAttributesString(const std::set<std::string> &values) const;
+    bool matchAttributesIp(const std::set<std::string> &values) const;
     bool isRegEx() const;
+    void sortAndMergeIpRangesValues();
 
     MatchType type;
     Operators operator_type;

components/include/health_checker.h

@@ -21,6 +21,7 @@
 #include "i_shell_cmd.h"
 #include "i_orchestration_status.h"
 #include "component.h"
+#include "i_service_controller.h"
 
 class HealthChecker
         :
@@ -29,7 +30,8 @@ class HealthChecker
     Singleton::Consume<I_Socket>,
     Singleton::Consume<I_Health_Check_Manager>,
     Singleton::Consume<I_ShellCmd>,
-    Singleton::Consume<I_OrchestrationStatus>
+    Singleton::Consume<I_OrchestrationStatus>,
+    Singleton::Consume<I_ServiceController>
 {
 public:
     HealthChecker();

components/include/http_event_impl/i_http_event_impl.h

@@ -50,9 +50,11 @@ public:
         position(mod_position)
     {
         dbgAssert(mod_type != ModificationType::APPEND || position == injection_pos_irrelevant)
+            << AlertInfo(AlertTeam::CORE, "http manager")
             << "Injection position is not applicable to a modification of type \"Append\"";
 
         dbgAssert(mod_type != ModificationType::INJECT || position >= 0)
+            << AlertInfo(AlertTeam::CORE, "http manager")
             << "Invalid injection position: must be non-negative. Position: "
             << position;
     }
@@ -166,6 +168,7 @@ private:
             }
             default:
                 dbgAssert(false)
+                    << AlertInfo(AlertTeam::CORE, "http manager")
                     << "Unknown type of ModificationType: "
                     << static_cast<int>(modification_type);
         }

components/include/http_geo_filter.h

@@ -15,7 +15,8 @@ class HttpGeoFilter
     public Component,
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_GeoLocation>,
-    Singleton::Consume<I_GenericRulebase>
+    Singleton::Consume<I_GenericRulebase>,
+    Singleton::Consume<I_Environment>
 {
 public:
     HttpGeoFilter();

components/include/http_inspection_events.h

@@ -183,4 +183,16 @@ class WaitTransactionEvent : public Event<WaitTransactionEvent, EventVerdict>
 {
 };
 
+class SecurityAppsDropEvent : public Event<SecurityAppsDropEvent>
+{
+public:
+    SecurityAppsDropEvent(
+        const std::set<std::string> &apps_names)
+            :
+        apps_names(apps_names) {}
+    const std::set<std::string> & getAppsNames() const { return apps_names; }
+
+private:
+    const std::set<std::string> apps_names;
+};
 #endif // __HTTP_INSPECTION_EVENTS_H__

components/include/http_transaction_data.h

@@ -136,6 +136,7 @@ public:
     static const std::string req_body;
     static const std::string source_identifier;
     static const std::string proxy_ip_ctx;
+    static const std::string xff_vals_ctx;
 
     static const CompressionType default_response_content_encoding;
 

components/include/i_details_resolver.h

@@ -29,7 +29,9 @@ public:
     virtual bool isGwNotVsx() = 0;
     virtual bool isVersionAboveR8110() = 0;
     virtual bool isReverseProxy() = 0;
+    virtual bool isCloudStorageEnabled() = 0;
     virtual Maybe<std::tuple<std::string, std::string, std::string>> parseNginxMetadata() = 0;
+    virtual Maybe<std::tuple<std::string, std::string, std::string, std::string, std::string>> readCloudMetadata() = 0;
     virtual std::map<std::string, std::string> getResolvedDetails() = 0;
 #if defined(gaia) || defined(smb)
     virtual bool compareCheckpointVersion(int cp_version, std::function<bool(int, int)> compare_operator) const = 0;

components/include/i_downloader.h

@@ -22,7 +22,7 @@
 class I_Downloader
 {
 public:
-    virtual Maybe<std::string> downloadFileFromFog(
+    virtual Maybe<std::string> downloadFile(
         const std::string &checksum,
         Package::ChecksumTypes,
         const GetResourceFile &resourse_file

components/include/i_service_controller.h

@@ -64,7 +64,9 @@ public:
         const std::string &service_id
     ) = 0;
 
-    virtual std::map<std::string, PortNumber> getServiceToPortMap() = 0;
+    virtual std::map<std::string, std::vector<PortNumber>> getServiceToPortMap() = 0;
+
+    virtual bool getServicesPolicyStatus() const = 0;
 
 protected:
     virtual ~I_ServiceController() {}

components/include/i_update_communication.h

@@ -32,6 +32,7 @@ public:
         const std::string &policy_versions
     ) const = 0;
     virtual Maybe<void> authenticateAgent() = 0;
+    virtual void registerLocalAgentToFog() = 0;
     virtual Maybe<void> getUpdate(CheckUpdateRequest &request) = 0;
     virtual Maybe<std::string> downloadAttributeFile(
         const GetResourceFile &resourse_file,

components/include/ip_utilities.h

@@ -28,8 +28,9 @@
 
 // LCOV_EXCL_START Reason: temporary until we add relevant UT until 07/10
 bool operator<(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
-
 bool operator==(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<=(const IpAddress &this_ip_addr, const IpAddress &other_ip_addr);
+bool operator<(const IPRange &range1, const IPRange &range2);
 // LCOV_EXCL_STOP
 
 Maybe<std::pair<std::string, int>> extractAddressAndMaskSize(const std::string &cidr);

components/include/orchestration_comp.h

@@ -31,6 +31,7 @@
 #include "i_environment.h"
 #include "i_tenant_manager.h"
 #include "i_package_handler.h"
+#include "i_proxy_configuration.h"
 #include "i_env_details.h"
 #include "component.h"
 
@@ -54,7 +55,8 @@ class OrchestrationComp
     Singleton::Consume<I_UpdateCommunication>,
     Singleton::Consume<I_Downloader>,
     Singleton::Consume<I_ManifestController>,
-    Singleton::Consume<I_EnvDetails>
+    Singleton::Consume<I_EnvDetails>,
+    Singleton::Consume<I_ProxyConfiguration>
 {
 public:
     OrchestrationComp();

components/include/orchestrator/rest_api/get_resource_file.h

@@ -115,7 +115,7 @@ public:
             case ResourceFileType::VIRTUAL_SETTINGS: return "virtualSettings";
             case ResourceFileType::VIRTUAL_POLICY:   return "virtualPolicy";
             default:
-                dbgAssert(false) << "Unknown file type";
+                dbgAssert(false) << AlertInfo(AlertTeam::CORE, "update process") << "Unknown file type";
         }
         return std::string();
     }

components/include/package.h

@@ -56,7 +56,7 @@ private:
             if (mapped_type.second == type) return mapped_type.first;
         }
 
-        dbgAssert(false) << "Unsupported type " << static_cast<int>(type);
+        dbgAssert(false) << AlertInfo(AlertTeam::CORE, "packaging") << "Unsupported type " << static_cast<int>(type);
         // Just satisfying the compiler, this return never reached
         return std::string();
     }

components/include/reverse_proxy_defaults.h

@@ -7,24 +7,28 @@ static const std::string product_name = getenv("DOCKER_RPM_ENABLED") ? "CloudGua
 static const std::string default_cp_cert_file = "/etc/cp/cpCert.pem";
 static const std::string default_cp_key_file = "/etc/cp/cpKey.key";
 static const std::string default_rpm_conf_path = "/etc/cp/conf/rpmanager/";
+
 static const std::string default_certificate_path = "/etc/cp/rpmanager/certs";
+static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/";
+static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers";
+static const std::string default_rpm_prepare_path = "/etc/cp/conf/rpmanager/prepare/servers";
+
+static const std::string default_nginx_log_files_path = "/var/log/nginx/";
 static const std::string default_additional_files_path = "/etc/cp/conf/rpmanager/include";
 static const std::string default_server_config = "additional_server_config.conf";
 static const std::string default_location_config = "additional_location_config.conf";
 static const std::string default_trusted_ca_suffix = "_user_ca_bundle.crt";
-static const std::string default_nginx_log_files_path = "/var/log/nginx/";
 static const std::string default_log_files_host_path = "/var/log/nano_agent/rpmanager/nginx_log/";
-static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers";
 static const std::string default_template_path = "/etc/cp/conf/rpmanager/nginx-template-clear";
-static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/";
 static const std::string default_server_certificate_path = "/etc/cp/rpmanager/certs/sslCertificate_";
 static const std::string default_server_certificate_key_path = "/etc/cp/rpmanager/certs/sslPrivateKey_";
 static const std::string default_container_name = "cp_nginx_gaia";
 static const std::string default_docker_image = "cp_nginx_gaia";
 static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/nginx.conf";
+static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf";
 static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
 static const std::string default_nginx_config_include_file =
-    "/etc/cp/conf/rpmanager/servers/nginx_conf_include";
+    "/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf";
 static const std::string default_global_conf_include_template =
     "/etc/cp/conf/rpmanager/nginx-conf-include-template";
 static const std::string default_global_conf_include_template_no_responses =

components/include/service_health_status.h

@@ -0,0 +1,39 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __SERVICE_HEALTH_STATUS_H__
+#define __SERVICE_HEALTH_STATUS_H__
+
+#include "singleton.h"
+#include "i_rest_api.h"
+#include "i_environment.h"
+#include "component.h"
+
+class ServiceHealthStatus
+        :
+    public Component,
+    Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_Environment>
+{
+public:
+    ServiceHealthStatus();
+    ~ServiceHealthStatus();
+
+    void init() override;
+
+private:
+    class Impl;
+    std::unique_ptr<Impl> pimpl;
+};
+
+#endif // __SERVICE_HEALTH_STATUS_H__

components/include/user_identifiers_config.h

@@ -58,7 +58,7 @@ private:
         const std::string::const_iterator &end,
         const std::string &key) const;
     Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const;
-    Maybe<std::string> parseXForwardedFor(const std::string &str) const;
+    Maybe<std::string> parseXForwardedFor(const std::string &str, ExtractType type) const;
 
     std::vector<UsersIdentifiersConfig> user_identifiers;
 };

components/include/waap.h

@@ -34,6 +34,8 @@ class I_Messaging;
 class I_AgentDetails;
 class I_Encryptor;
 
+const std::string WAAP_APPLICATION_NAME = "waap application";
+
 class WaapComponent
         :
     public Component,

components/packet/packet.cc

@@ -563,7 +563,10 @@ Packet::parsePacket(PktType type, IPType proto)
             return parseFromL3v6();
         }
         default: {
-            dbgAssert(false) << "Unknown (neither IPv4, nor IPv6), or uninitialized packet type: " << proto;
+            dbgAssert(false)
+                << AlertInfo(AlertTeam::CORE, "packet")
+                << "Unknown (neither IPv4, nor IPv6), or uninitialized packet type: "
+                << proto;
         }
     }
 

components/pending_key/pending_key.cc

@@ -43,7 +43,9 @@ PendingKey::print(ostream &os) const
 size_t
 PendingKey::hash() const
 {
-    dbgAssert(src.type != IPType::UNINITIALIZED) << "PendingKey::hash was called on an uninitialized object";
+    dbgAssert(src.type != IPType::UNINITIALIZED)
+        << AlertInfo(AlertTeam::CORE, "pending key")
+        << "PendingKey::hash was called on an uninitialized object";
     size_t seed = 0;
     hashCombine(seed, static_cast<u_char>(src.type));
     hashCombine(seed, src.proto);

components/report_messaging/report_messaging_ut/CMakeLists.txt

@@ -1,3 +0,0 @@
-link_directories(${BOOST_ROOT}/lib)
-
-add_unit_test(report_messaging_ut "report_messaging_ut.cc" "report_messaging;report;messaging;singleton;-lboost_regex")

components/security_apps/http_geo_filter/CMakeLists.txt

@@ -1 +1,5 @@
+include_directories(../waap/include)
+include_directories(../waap/waap_clib)
+include_directories(../../attachment-intakers/nginx_attachment)
+
 add_library(http_geo_filter http_geo_filter.cc)

components/security_apps/http_geo_filter/http_geo_filter.cc

@@ -4,10 +4,16 @@
 #include <unistd.h>
 #include <stddef.h>
 #include <algorithm>
+#include <sstream>
+#include <string>
+#include <vector>
+#include <boost/algorithm/string.hpp>
 
+#include "cidrs_data.h"
 #include "generic_rulebase/generic_rulebase.h"
 #include "generic_rulebase/parameters_config.h"
 #include "generic_rulebase/triggers_config.h"
+#include "user_identifiers_config.h"
 #include "debug.h"
 #include "config.h"
 #include "rest.h"
@@ -21,9 +27,10 @@ USE_DEBUG_FLAG(D_GEO_FILTER);
 
 static const LogTriggerConf default_triger;
 
-class HttpGeoFilter::Impl : public Listener<NewHttpTransactionEvent>
+class HttpGeoFilter::Impl : public Listener<HttpRequestHeaderEvent>
 {
 public:
+
     void
     init()
     {
@@ -55,32 +62,42 @@ public:
     }
 
     EventVerdict
-    respond(const NewHttpTransactionEvent &event) override
+    respond(const HttpRequestHeaderEvent &event) override
     {
         dbgTrace(D_GEO_FILTER) << getListenerName() << " new transaction event";
 
-        if (!ParameterException::isGeoLocationExceptionExists() &&
-            !getConfiguration<GeoConfig>("rulebase", "httpGeoFilter").ok()
-        ) {
-            dbgTrace(D_GEO_FILTER) << "No geo location practice nor exception was found. Returning default verdict";
+        if (!event.isLastHeader()) return EventVerdict(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_INSPECT);
+        std::set<std::string> ip_set;
+        auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
+        auto maybe_xff = env->get<std::string>(HttpTransactionData::xff_vals_ctx);
+        if (!maybe_xff.ok()) {
+            dbgTrace(D_GEO_FILTER) << "failed to get xff vals from env";
+        } else {
+            ip_set = split(maybe_xff.unpack(), ',');
+        }
+        dbgDebug(D_GEO_FILTER) << getListenerName() << " last header, start lookup";
+
+        if (ip_set.size() > 0) {
+            removeTrustedIpsFromXff(ip_set);
+        } else {
+            dbgDebug(D_GEO_FILTER) << "xff not found in headers";
+        }
+
+        auto maybe_source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+        if (!maybe_source_ip.ok()) {
+            dbgWarning(D_GEO_FILTER) << "failed to get source ip from env";
             return EventVerdict(default_action);
         }
 
-        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
-        auto asset_location = i_geo_location->lookupLocation(event.getSourceIP());
-        if (!asset_location.ok()) {
-            dbgTrace(D_GEO_FILTER) << "Lookup location failed, Error: " << asset_location.getErr();
-            return EventVerdict(default_action);
-        }
+        auto source_ip = convertIpAddrToString(maybe_source_ip.unpack());
+        ip_set.insert(source_ip);
 
-        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data = asset_location.unpack();
-
-        ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(event, geo_location_data);
+        ngx_http_cp_verdict_e exception_verdict = getExceptionVerdict(ip_set);
         if (exception_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
             return EventVerdict(exception_verdict);
         }
 
-        ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(event, geo_location_data);
+        ngx_http_cp_verdict_e geo_lookup_verdict = getGeoLookupVerdict(ip_set);
         if (geo_lookup_verdict != ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT) {
             return EventVerdict(geo_lookup_verdict);
         }
@@ -88,6 +105,73 @@ public:
     }
 
 private:
+    std::set<std::string>
+    split(const std::string& s, char delim) {
+        std::set<std::string> elems;
+        std::stringstream ss(s);
+        std::string value;
+        while (std::getline(ss, value, delim)) {
+            elems.insert(trim(value));
+        }
+        return elems;
+    }
+
+    static inline std::string &ltrim(std::string &s) {
+        s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+            [] (char c) { return !std::isspace(c); }));
+        return s;
+    }
+
+    // trim from end
+    static inline std::string &rtrim(std::string &s) {
+        s.erase(std::find_if(s.rbegin(), s.rend(),
+            [] (char c) { return !std::isspace(c); }).base(), s.end());
+        return s;
+    }
+
+    // trim from both ends
+    static inline std::string &trim(std::string &s) {
+        return ltrim(rtrim(s));
+    }
+
+    void
+    removeTrustedIpsFromXff(std::set<std::string> &xff_set)
+    {
+        auto identify_config = getConfiguration<UsersAllIdentifiersConfig>(
+            "rulebase",
+            "usersIdentifiers"
+        );
+        if (!identify_config.ok()) {
+            dbgDebug(D_GEO_FILTER) << "did not find users identifiers definition in policy";
+        } else {
+            auto trusted_ips = (*identify_config).getHeaderValuesFromConfig("x-forwarded-for");
+            for (auto it = xff_set.begin(); it != xff_set.end();) {
+                if (isIpTrusted(*it, trusted_ips)) {
+                    dbgTrace(D_GEO_FILTER) << "xff value is in trusted ips: " << *it;
+                    it = xff_set.erase(it);
+                } else {
+                    dbgTrace(D_GEO_FILTER) << "xff value is not in trusted ips: " << *it;
+                    ++it;
+                }
+            }
+        }
+    }
+
+    bool
+    isIpTrusted(const string &ip, const vector<string> &trusted_ips)
+    {
+        for (const auto &trusted_ip : trusted_ips) {
+            CIDRSData cidr_data(trusted_ip);
+            if (
+                ip == trusted_ip ||
+                (cidr_data.contains(ip))
+            ) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     string
     convertIpAddrToString(const IPAddr &ip_to_convert)
     {
@@ -117,54 +201,75 @@ private:
     }
 
     ngx_http_cp_verdict_e
-    getGeoLookupVerdict(
-        const NewHttpTransactionEvent &event,
-        const EnumArray<I_GeoLocation::GeoLocationField, std::string> &geo_location_data)
+    getGeoLookupVerdict(const std::set<std::string> &sources)
     {
         auto maybe_geo_config = getConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
         if (!maybe_geo_config.ok()) {
-            dbgWarning(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
+            dbgTrace(D_GEO_FILTER) << "Failed to load HTTP Geo Filter config. Error:" << maybe_geo_config.getErr();
             return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
         }
         GeoConfig geo_config = maybe_geo_config.unpack();
-        string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
+        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
 
-        if (geo_config.isAllowedCountry(country_code)) {
-            dbgTrace(D_GEO_FILTER)
-                << "geo verdict ACCEPT, practice id: "
-                << geo_config.getId()
-                << ", country code: "
-                << country_code;
-            generateVerdictLog(
-                ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
-                event,
-                geo_config.getId(),
-                true,
-                geo_location_data
-            );
-            return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
-        }
-        if (geo_config.isBlockedCountry(country_code)) {
-            dbgTrace(D_GEO_FILTER)
-                << "geo verdict DROP, practice id: "
-                << geo_config.getId()
-                << ", country code: "
-                << country_code;
-            generateVerdictLog(
-                ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
-                event,
-                geo_config.getId(),
-                true,
-                geo_location_data
-            );
-            return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+        for (const std::string& source : sources) {
+            Maybe<IPAddr> maybe_source_ip = IPAddr::createIPAddr(source);
+            if (!maybe_source_ip.ok()){
+                dbgWarning(D_GEO_FILTER) <<
+                "create ip address failed for source: " <<
+                source <<
+                ", Error: " <<
+                maybe_source_ip.getErr();
+                continue;
+            }
+            auto asset_location = i_geo_location->lookupLocation(maybe_source_ip.unpack());
+            if (!asset_location.ok()) {
+                dbgWarning(D_GEO_FILTER) <<
+                "Lookup location failed for source: " <<
+                source <<
+                ", Error: " <<
+                asset_location.getErr();
+                continue;
+            }
+
+            geo_location_data = asset_location.unpack();
+
+            string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+
+            if (geo_config.isAllowedCountry(country_code)) {
+                dbgTrace(D_GEO_FILTER)
+                    << "geo verdict ACCEPT, practice id: "
+                    << geo_config.getId()
+                    << ", country code: "
+                    << country_code;
+                generateVerdictLog(
+                    ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT,
+                    geo_config.getId(),
+                    true,
+                    geo_location_data
+                );
+                return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT;
+            }
+            if (geo_config.isBlockedCountry(country_code)) {
+                dbgTrace(D_GEO_FILTER)
+                    << "geo verdict DROP, practice id: "
+                    << geo_config.getId()
+                    << ", country code: "
+                    << country_code;
+                generateVerdictLog(
+                    ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP,
+                    geo_config.getId(),
+                    true,
+                    geo_location_data
+                );
+                return ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP;
+            }
         }
         dbgTrace(D_GEO_FILTER)
             << "No matched practice. Returned default action: "
             << geo_config.getDefaultAction();
         generateVerdictLog(
             convertActionToVerdict(geo_config.getDefaultAction()),
-            event,
             geo_config.getId(),
             true,
             geo_location_data,
@@ -176,7 +281,6 @@ private:
     Maybe<pair<ngx_http_cp_verdict_e, string>>
     getBehaviorsVerdict(
         const unordered_map<string, set<string>> &behaviors_map_to_search,
-        const NewHttpTransactionEvent &event,
         EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data)
     {
         bool is_matched = false;
@@ -193,7 +297,6 @@ private:
                 dbgTrace(D_GEO_FILTER) << "behavior verdict: DROP, exception id: " << behavior.getId();
                 generateVerdictLog(
                     matched_verdict,
-                    event,
                     behavior.getId(),
                     false,
                     geo_location_data
@@ -218,63 +321,74 @@ private:
     }
 
     ngx_http_cp_verdict_e
-    getExceptionVerdict(
-        const NewHttpTransactionEvent &event,
-        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data
-    ){
-        string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
-        string country_name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME];
-        string source_ip = convertIpAddrToString(event.getSourceIP());
+    getExceptionVerdict(const std::set<std::string> &sources) {
 
         pair<ngx_http_cp_verdict_e, string> curr_matched_behavior;
         ngx_http_cp_verdict_e verdict = ngx_http_cp_verdict_e::TRAFFIC_VERDICT_IRRELEVANT;
+        I_GeoLocation *i_geo_location = Singleton::Consume<I_GeoLocation>::by<HttpGeoFilter>();
+        EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data;
 
-        dbgTrace(D_GEO_FILTER)
+        for (const std::string& source : sources) {
+
+            Maybe<IPAddr> maybe_source_ip = IPAddr::createIPAddr(source);
+            if (!maybe_source_ip.ok()){
+                dbgWarning(D_GEO_FILTER) <<
+                "create ip address failed for source: " <<
+                source <<
+                ", Error: " <<
+                maybe_source_ip.getErr();
+                continue;
+            }
+
+
+            auto asset_location = i_geo_location->lookupLocation(maybe_source_ip.unpack());
+            if (!asset_location.ok()) {
+                dbgWarning(D_GEO_FILTER) << "Lookup location failed for source: " <<
+                source <<
+                ", Error: " <<
+                asset_location.getErr();
+                continue;
+            }
+            geo_location_data = asset_location.unpack();
+            string country_code = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE];
+            string country_name = geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_NAME];
+            dbgTrace(D_GEO_FILTER)
             << "Get exception verdict. "
             << "country code: "
             << country_code
             << ", country name: "
             << country_name
             << ", source ip address: "
-            << source_ip;
+            << source;
 
-        unordered_map<string, set<string>> exception_value_source_ip = {{"sourceIP", {source_ip}}};
-        auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_source_ip, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
+            unordered_map<string, set<string>> exception_value_country_code = {
+                {"countryCode", {country_code}}
+            };
+            auto matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, geo_location_data);
+            if (matched_behavior_maybe.ok()) {
+                curr_matched_behavior = matched_behavior_maybe.unpack();
+                verdict = curr_matched_behavior.first;
+                if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+                    return verdict;
+                }
+            }
+
+            unordered_map<string, set<string>> exception_value_country_name = {
+                {"countryName", {country_name}}
+            };
+            matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, geo_location_data);
+            if (matched_behavior_maybe.ok()) {
+                curr_matched_behavior = matched_behavior_maybe.unpack();
+                verdict = curr_matched_behavior.first;
+                if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
+                    return verdict;
+                }
             }
         }
 
-        unordered_map<string, set<string>> exception_value_country_code = {
-            {"countryCode", {country_code}}
-        };
-        matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_code, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
-            }
-        }
-
-        unordered_map<string, set<string>> exception_value_country_name = {
-            {"countryName", {country_name}}
-        };
-        matched_behavior_maybe = getBehaviorsVerdict(exception_value_country_name, event, geo_location_data);
-        if (matched_behavior_maybe.ok()) {
-            curr_matched_behavior = matched_behavior_maybe.unpack();
-            verdict = curr_matched_behavior.first;
-            if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP) {
-                return verdict;
-            }
-        }
         if (verdict == ngx_http_cp_verdict_e::TRAFFIC_VERDICT_ACCEPT) {
             generateVerdictLog(
                 verdict,
-                event,
                 curr_matched_behavior.second,
                 false,
                 geo_location_data
@@ -286,7 +400,6 @@ private:
     void
     generateVerdictLog(
         const ngx_http_cp_verdict_e &verdict,
-        const NewHttpTransactionEvent &event,
         const string &matched_id,
         bool is_geo_filter,
         const EnumArray<I_GeoLocation::GeoLocationField, std::string> geo_location_data,
@@ -307,14 +420,27 @@ private:
             LogField(matched_on, matched_id),
             ReportIS::Tags::HTTP_GEO_FILTER
         );
-        log
-            << LogField("sourceIP", convertIpAddrToString(event.getSourceIP()))
-            << LogField("sourcePort", event.getSourcePort())
-            << LogField("hostName", event.getDestinationHost())
-            << LogField("httpMethod", event.getHttpMethod())
-            << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
+        auto env = Singleton::Consume<I_Environment>::by<HttpGeoFilter>();
+        auto source_ip = env->get<IPAddr>(HttpTransactionData::client_ip_ctx);
+        if (source_ip.ok()) log << LogField("sourceIP", convertIpAddrToString(source_ip.unpack()));
+
+        auto source_identifier = env->get<string>(HttpTransactionData::source_identifier);
+        if (source_identifier.ok()) log << LogField("httpSourceId", source_identifier.unpack());
+
+        auto source_port = env->get<string>(HttpTransactionData::client_port_ctx);
+        if (source_port.ok()) log << LogField("sourcePort", source_port.unpack());
+
+        auto host_name = env->get<string>(HttpTransactionData::host_name_ctx);
+        if (host_name.ok()) log << LogField("hostName", host_name.unpack());
+
+        auto method = env->get<string>(HttpTransactionData::method_ctx);
+        if (method.ok()) log << LogField("httpMethod", method.unpack());
+
+        log << LogField("securityAction", is_prevent ? "Prevent" : "Detect");
 
         if (is_default_action) log << LogField("isDefaultSecurityAction", true);
+        auto xff = env->get<string>(HttpTransactionData::xff_vals_ctx);
+        if (xff.ok()) log << LogField("proxyIP", xff.unpack());
 
         log
             << LogField("sourceCountryCode", geo_location_data[I_GeoLocation::GeoLocationField::COUNTRY_CODE])
@@ -343,5 +469,6 @@ void
 HttpGeoFilter::preload()
 {
     registerExpectedConfiguration<GeoConfig>("rulebase", "httpGeoFilter");
+    registerExpectedConfiguration<UsersAllIdentifiersConfig>("rulebase", "usersIdentifiers");
     registerConfigLoadCb([this]() { pimpl->loadDefaultAction(); });
 }

components/security_apps/ips/compound_protection.cc

@@ -43,7 +43,10 @@ CompoundProtection::Impl::getMatch(const set<PMPattern> &matched) const
         case Operation::ORDERED_AND: return getMatchOrderedAnd(matched);
     }
 
-    dbgAssert(false) << "Unknown compound operation: " << static_cast<uint>(operation);
+    dbgAssert(false)
+        << AlertInfo(AlertTeam::CORE, "ips")
+        << "Unknown compound operation: "
+        << static_cast<uint>(operation);
     return MatchType::NO_MATCH;
 }
 

components/security_apps/ips/ips_configuration.cc

@@ -8,7 +8,9 @@ IPSConfiguration::Context::Context(ContextType _type, uint history) : type(_type
 uint
 IPSConfiguration::Context::getHistorySize() const
 {
-    dbgAssert(type == ContextType::HISTORY) << "Try to access history size for non-history context";
+    dbgAssert(type == ContextType::HISTORY)
+        << AlertInfo(AlertTeam::CORE, "ips")
+        << "Try to access history size for non-history context";
     return history_size;
 }
 
@@ -69,6 +71,8 @@ uint
 IPSConfiguration::getHistorySize(const string &name) const
 {
     auto context = context_config.find(name);
-    dbgAssert(context != context_config.end()) << "Try to access history size for non-exiting context";
+    dbgAssert(context != context_config.end())
+        << AlertInfo(AlertTeam::CORE, "ips")
+        << "Try to access history size for non-exiting context";
     return context->second.getHistorySize();
 }

components/security_apps/ips/ips_entry.cc

@@ -26,6 +26,8 @@ static const map<string, IPSConfiguration::Context> default_conf_mapping = {
 };
 
 static const IPSConfiguration default_conf(default_conf_mapping);
+static const IPSSignatures default_ips_sigs;
+static const SnortSignatures default_snort_sigs;
 
 IPSEntry::IPSEntry() : TableOpaqueSerialize<IPSEntry>(this) {}
 
@@ -51,9 +53,9 @@ IPSEntry::respond(const ParsedContext &parsed)
     ctx.registerValue(name, buf);
 
     ctx.activate();
-    auto &signatures = getConfigurationWithDefault(IPSSignatures(), "IPS", "IpsProtections");
+    auto &signatures = getConfigurationWithDefault(default_ips_sigs, "IPS", "IpsProtections");
     bool should_drop = signatures.isMatchedPrevent(parsed.getName(), buf);
-    auto &snort_signatures = getConfigurationWithDefault(SnortSignatures(), "IPSSnortSigs", "SnortProtections");
+    auto &snort_signatures = getConfigurationWithDefault(default_snort_sigs, "IPSSnortSigs", "SnortProtections");
     should_drop |= snort_signatures.isMatchedPrevent(parsed.getName(), buf);
     ctx.deactivate();
 

components/security_apps/ips/ips_signatures.cc

@@ -84,7 +84,7 @@ IPSSignatureMetaData::getSeverityString() const
             return "Critical";
     }
 
-    dbgAssert(false) << "Illegal severity value: " << static_cast<uint>(severity);
+    dbgAssert(false) << AlertInfo(AlertTeam::CORE, "ips") << "Illegal severity value: " << static_cast<uint>(severity);
     return "Critical";
 }
 
@@ -116,7 +116,10 @@ IPSSignatureMetaData::getPerformanceString() const
             return "Critical";
     }
 
-    dbgAssert(false) << "Illegal performance value: " << static_cast<uint>(performance);
+    dbgAssert(false)
+        << AlertInfo(AlertTeam::CORE, "ips")
+        << "Illegal performance value: "
+        << static_cast<uint>(performance);
     return "Critical";
 }
 

components/security_apps/ips/ips_ut/configuration.cc

@@ -7,7 +7,7 @@ TEST(configuration, basic_context)
 
     IPSConfiguration::Context ctx1(IPSConfiguration::ContextType::HISTORY, 254);
     EXPECT_EQ(ctx1.getType(), IPSConfiguration::ContextType::HISTORY);
-    EXPECT_EQ(ctx1.getHistorySize(), 254);
+    EXPECT_EQ(ctx1.getHistorySize(), 254u);
 
     IPSConfiguration::Context ctx2(IPSConfiguration::ContextType::NORMAL, 0);
     EXPECT_EQ(ctx2.getType(), IPSConfiguration::ContextType::NORMAL);
@@ -42,7 +42,7 @@ TEST(configuration, read_configuration)
 
     auto body = conf.getContext("HTTP_REQUEST_BODY");
     EXPECT_EQ(body.getType(), IPSConfiguration::ContextType::HISTORY);
-    EXPECT_EQ(conf.getHistorySize("HTTP_REQUEST_BODY"), 100);
+    EXPECT_EQ(conf.getHistorySize("HTTP_REQUEST_BODY"), 100u);
 
     auto header = conf.getContext("HTTP_REQUEST_HEADER");
     EXPECT_EQ(header.getType(), IPSConfiguration::ContextType::KEEP);

components/security_apps/ips/ips_ut/entry_ut.cc

@@ -137,8 +137,8 @@ private:
 TEST_F(EntryTest, basic_inherited_functions)
 {
     EXPECT_EQ(IPSEntry::name(), "IPS");
-    EXPECT_EQ(IPSEntry::currVer(), 0);
-    EXPECT_EQ(IPSEntry::minVer(), 0);
+    EXPECT_EQ(IPSEntry::currVer(), 0u);
+    EXPECT_EQ(IPSEntry::minVer(), 0u);
     EXPECT_NE(IPSEntry::prototype(), nullptr);
     EXPECT_EQ(entry.getListenerName(), IPSEntry::name());
 

components/security_apps/ips/ips_ut/resource_ut.cc

@@ -71,7 +71,7 @@ TEST(resources, basic_resource)
     Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(resource);
 
     auto loaded_resources = getSettingWithDefault(IPSSignaturesResource(), "IPS", "protections");
-    EXPECT_EQ(loaded_resources.getSignatures().size(), 2);
+    EXPECT_EQ(loaded_resources.getSignatures().size(), 2u);
     auto version = getSettingWithDefault<string>("", "IPS", "VersionId");
     EXPECT_EQ(version, "1234567");
 }

components/security_apps/layer_7_access_control/layer_7_access_control.cc

@@ -385,8 +385,29 @@ Layer7AccessControl::Impl::init()
     i_intelligence = Singleton::Consume<I_Intelligence_IS_V2>::by<Layer7AccessControl>();
     i_mainloop = Singleton::Consume<I_MainLoop>::by<Layer7AccessControl>();
 
-    chrono::minutes expiration(
-        getProfileAgentSettingWithDefault<uint>(60u, "layer7AccessControl.crowdsec.cacheExpiration")
+    int cache_expiration_in_seconds = 30;
+    string cache_expiration_env = getenv("CROWDSEC_CACHE_EXPIRATION") ? getenv("CROWDSEC_CACHE_EXPIRATION") : "";
+    if (!cache_expiration_env.empty()) {
+        if (
+            all_of(cache_expiration_env.begin(), cache_expiration_env.end(), ::isdigit)
+            && stoi(cache_expiration_env) > 0
+        ) {
+            cache_expiration_in_seconds = stoi(cache_expiration_env);
+            dbgInfo(D_L7_ACCESS_CONTROL)
+                << "Successfully read cache expiration value from env: "
+                << cache_expiration_env;
+        } else {
+            dbgWarning(D_L7_ACCESS_CONTROL)
+                << "An invalid cache expiration value was provided in env: "
+                << cache_expiration_env;
+        }
+    }
+
+    chrono::seconds expiration(
+        getProfileAgentSettingWithDefault<uint>(
+            cache_expiration_in_seconds,
+            "layer7AccessControl.crowdsec.cacheExpiration"
+        )
     );
 
     ip_reputation_cache.startExpiration(

components/security_apps/layer_7_access_control/layer_7_access_control_ut/layer_7_access_control_ut.cc

@@ -142,6 +142,13 @@ string disabled_settings =
         "}"
     "],\n";
 
+
+string local_intelligence =
+    "\"intelligence\":{"
+    "  \"local intelligence server ip\":\"127.0.0.1\","
+    "  \"local intelligence server primary port\":9090"
+    "}\n,";
+
 string policy =
     "\"rulebase\": {"
         "\"usersIdentifiers\": ["
@@ -240,7 +247,9 @@ Layer7AccessControlTest::verifyReport(
     string log = reportToStr(report);
     dbgTrace(D_L7_ACCESS_CONTROL) << "Report: " << log;
 
-    if (!source_identifier.empty()) EXPECT_THAT(log, HasSubstr("\"httpSourceId\": \"" + source_identifier + "\""));
+    if (!source_identifier.empty()) {
+        EXPECT_THAT(log, HasSubstr("\"httpSourceId\": \"" + source_identifier + "\""));
+    }
     EXPECT_THAT(log, HasSubstr("\"securityAction\": \"" + security_action + "\""));
     EXPECT_THAT(log, HasSubstr("\"eventName\": \"Access Control External Vendor Reputation\""));
     EXPECT_THAT(log, HasSubstr("\"httpHostName\": \"juice-shop.checkpoint.com\""));
@@ -259,7 +268,7 @@ Layer7AccessControlTest::verifyReport(
 
 TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string intelligence_response_ok = loadIntelligenceResponse("data/ok_intelligence_response.json");
@@ -305,7 +314,7 @@ TEST_F(Layer7AccessControlTest, ReturnAcceptVerdict)
 
 TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
     
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -351,7 +360,7 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictOnMaliciousReputation)
 
 TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -403,7 +412,7 @@ TEST_F(Layer7AccessControlTest, ReturnDropVerdictCacheBased)
 
 TEST_F(Layer7AccessControlTest, AcceptOnDetect)
 {
-    stringstream ss_conf(detect_settings + policy);
+    stringstream ss_conf(detect_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
     
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");
@@ -449,7 +458,7 @@ TEST_F(Layer7AccessControlTest, AcceptOnDetect)
 
 TEST_F(Layer7AccessControlTest, FallbackToSourceIPAndDrop)
 {
-    stringstream ss_conf(prevent_settings + policy);
+    stringstream ss_conf(prevent_settings + local_intelligence + policy);
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss_conf);
 
     string malicious_intelligence_response = loadIntelligenceResponse("data/malicious_intelligence_response.json");

components/security_apps/local_policy_mgmt_gen/access_control_practice.cc

@@ -228,7 +228,11 @@ AccessControlPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
 
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
-    parseAppsecJSONKey<string>("practiceMode", mode, archive_in);
+    parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Access control practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Access control practice mode invalid: " + mode);
+    }
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<AccessControlRateLimit>("rateLimit", rate_limit, archive_in);
 }

components/security_apps/local_policy_mgmt_gen/appsec_practice_section.cc

@@ -19,7 +19,14 @@ using namespace std;
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 // LCOV_EXCL_START Reason: no test exist
 
-static const set<string> valid_modes = {"prevent-learn", "detect-learn", "prevent", "detect", "inactive"};
+static const set<string> valid_modes = {
+    "prevent-learn",
+    "detect-learn",
+    "prevent",
+    "detect",
+    "inactive",
+    "as-top-level"
+};
 static const set<string> valid_confidences = {"medium", "high", "critical"};
 
 void
@@ -138,15 +145,11 @@ AppSecPracticeWebAttacks::load(cereal::JSONInputArchive &archive_in)
         dbgWarning(D_LOCAL_POLICY) << "AppSec practice override mode invalid: " << mode;
     }
 
-    if (getMode() == "Prevent") {
-        parseAppsecJSONKey<string>("minimum-confidence", minimum_confidence, archive_in, "critical");
-        if (valid_confidences.count(minimum_confidence) == 0) {
-            dbgWarning(D_LOCAL_POLICY)
-                << "AppSec practice override minimum confidence invalid: "
-                << minimum_confidence;
-        }
-    } else {
-        minimum_confidence = "Transparent";
+    parseAppsecJSONKey<string>("minimum-confidence", minimum_confidence, archive_in, "critical");
+    if (valid_confidences.count(minimum_confidence) == 0) {
+        dbgWarning(D_LOCAL_POLICY)
+            << "AppSec practice override minimum confidence invalid: "
+            << minimum_confidence;
     }
     parseAppsecJSONKey<int>("max-body-size-kb", max_body_size_kb, archive_in, 1000000);
     parseAppsecJSONKey<int>("max-header-size-bytes", max_header_size_bytes, archive_in, 102400);
@@ -189,7 +192,10 @@ AppSecPracticeWebAttacks::getMode(const string &default_mode) const
 {
     if (isModeInherited(mode) || (key_to_practices_val2.find(mode) == key_to_practices_val2.end())) {
         dbgError(D_LOCAL_POLICY) << "Couldn't find a value for key: " << mode << ". Returning " << default_mode;
-        return default_mode;
+        if(key_to_practices_val2.find(default_mode) == key_to_practices_val2.end()) {
+            return default_mode;
+        }
+        return key_to_practices_val2.at(default_mode);
     }
     return key_to_practices_val2.at(mode);
 }
@@ -404,6 +410,7 @@ AppsecPracticeAntiBotSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 // LCOV_EXCL_START Reason: no test exist
+// Used for V1Beta1
 WebAppSection::WebAppSection(
     const string &_application_urls,
     const string &_asset_id,
@@ -417,7 +424,7 @@ WebAppSection::WebAppSection(
     const LogTriggerSection &parsed_log_trigger,
     const string &default_mode,
     const AppSecTrustedSources &parsed_trusted_sources,
-    const vector<InnerException> &parsed_exceptions)
+    const std::map<std::string, std::vector<InnerException>> &exceptions)
     :
     application_urls(_application_urls),
     asset_id(_asset_id),
@@ -427,21 +434,34 @@ WebAppSection::WebAppSection(
     practice_id(_practice_id),
     practice_name(_practice_name),
     context(_context),
-    web_attack_mitigation_severity(parsed_appsec_spec.getWebAttacks().getMinimumConfidence()),
     web_attack_mitigation_mode(parsed_appsec_spec.getWebAttacks().getMode(default_mode)),
     csrf_protection_mode("Disabled"),
     open_redirect_mode("Disabled"),
     error_disclosure_mode("Disabled"),
+    schema_validation_mode("Disabled"),
+    schema_validation_enforce_level("fullSchema"),
     practice_advanced_config(parsed_appsec_spec),
     anti_bots(parsed_appsec_spec.getAntiBot()),
     trusted_sources({ parsed_trusted_sources })
 {
+    auto mitigation_sevirity = parsed_appsec_spec.getWebAttacks().getMinimumConfidence();
+    if (key_to_mitigation_severity.find(mitigation_sevirity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << mitigation_sevirity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + mitigation_sevirity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(mitigation_sevirity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
+    web_attack_mitigation_severity =
+        web_attack_mitigation_mode != "Prevent" ? "Transparent" :
+        web_attack_mitigation_severity;
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     triggers.push_back(TriggersInWaapSection(parsed_log_trigger));
@@ -449,8 +469,11 @@ WebAppSection::WebAppSection(
         overrides.push_back(AppSecOverride(source_ident));
     }
 
-    for (const InnerException &exception : parsed_exceptions) {
-        overrides.push_back(AppSecOverride(exception));
+    for (const auto &exception : exceptions) {
+
+        for (const auto &inner_exception : exception.second) {
+            overrides.push_back(AppSecOverride(inner_exception));
+        }
     }
 }
 
@@ -466,6 +489,10 @@ WebAppSection::WebAppSection(
     const string &_context,
     const string &_web_attack_mitigation_severity,
     const string &_web_attack_mitigation_mode,
+    const string &_bot_protection,
+    const string &_schema_validation_mode,
+    const string &_schema_validation_enforce_level,
+    const vector<string> &_schema_validation_oas,
     const PracticeAdvancedConfig &_practice_advanced_config,
     const AppsecPracticeAntiBotSection &_anti_bots,
     const LogTriggerSection &parsed_log_trigger,
@@ -480,18 +507,29 @@ WebAppSection::WebAppSection(
     practice_id(_practice_id),
     practice_name(_practice_name),
     context(_context),
-    web_attack_mitigation_severity(_web_attack_mitigation_severity),
     web_attack_mitigation_mode(_web_attack_mitigation_mode),
+    bot_protection(_bot_protection),
+    schema_validation_mode(_schema_validation_mode),
+    schema_validation_enforce_level(_schema_validation_enforce_level),
+    schema_validation_oas(_schema_validation_oas),
     practice_advanced_config(_practice_advanced_config),
     anti_bots(_anti_bots),
     trusted_sources({ parsed_trusted_sources })
 {
+    if (key_to_mitigation_severity.find(_web_attack_mitigation_severity) == key_to_mitigation_severity.end()) {
+        dbgWarning(D_LOCAL_POLICY)
+        << "web attack mitigation severity invalid: "
+        << _web_attack_mitigation_severity;
+        throw PolicyGenException("web attack mitigation severity invalid: " + _web_attack_mitigation_severity);
+    } else {
+        web_attack_mitigation_severity = key_to_mitigation_severity.at(_web_attack_mitigation_severity);
+    }
     web_attack_mitigation = web_attack_mitigation_mode != "Disabled";
     web_attack_mitigation_action =
         web_attack_mitigation_mode != "Prevent" ? "Transparent" :
-        web_attack_mitigation_severity == "critical" ? "low" :
-        web_attack_mitigation_severity == "high" ? "balanced" :
-        web_attack_mitigation_severity == "medium" ? "high" :
+        web_attack_mitigation_severity == "Critical" ? "Low" :
+        web_attack_mitigation_severity == "High" ? "Balanced" :
+        web_attack_mitigation_severity == "Medium" ? "High" :
         "Error";
 
     csrf_protection_mode = protections.getCsrfProtectionMode(_web_attack_mitigation_mode);
@@ -502,6 +540,7 @@ WebAppSection::WebAppSection(
     for (const SourcesIdentifiers &source_ident : parsed_trusted_sources.getSourcesIdentifiers()) {
         overrides.push_back(AppSecOverride(source_ident));
     }
+
 }
 
 // LCOV_EXCL_STOP
@@ -509,36 +548,35 @@ WebAppSection::WebAppSection(
 void
 WebAppSection::save(cereal::JSONOutputArchive &out_ar) const
 {
-    string disabled_str = "Disabled";
-    string detect_str = "Detect";
     vector<string> empty_list;
     out_ar(
-        cereal::make_nvp("context",                     context),
-        cereal::make_nvp("webAttackMitigation",         web_attack_mitigation),
-        cereal::make_nvp("webAttackMitigationSeverity", web_attack_mitigation_severity),
-        cereal::make_nvp("webAttackMitigationAction",   web_attack_mitigation_action),
-        cereal::make_nvp("webAttackMitigationMode",     web_attack_mitigation_mode),
-        cereal::make_nvp("practiceAdvancedConfig",      practice_advanced_config),
-        cereal::make_nvp("csrfProtection",              csrf_protection_mode),
-        cereal::make_nvp("openRedirect",                open_redirect_mode),
-        cereal::make_nvp("errorDisclosure",             error_disclosure_mode),
-        cereal::make_nvp("practiceId",                  practice_id),
-        cereal::make_nvp("practiceName",                practice_name),
-        cereal::make_nvp("assetId",                     asset_id),
-        cereal::make_nvp("assetName",                   asset_name),
-        cereal::make_nvp("ruleId",                      rule_id),
-        cereal::make_nvp("ruleName",                    rule_name),
-        cereal::make_nvp("schemaValidation",               false),
-        cereal::make_nvp("schemaValidation_v2",            disabled_str),
-        cereal::make_nvp("oas",                            empty_list),
-        cereal::make_nvp("triggers",                    triggers),
-        cereal::make_nvp("applicationUrls",             application_urls),
-        cereal::make_nvp("overrides",                   overrides),
-        cereal::make_nvp("trustedSources",              trusted_sources),
-        cereal::make_nvp("waapParameters",              empty_list),
-        cereal::make_nvp("botProtection",               false),
-        cereal::make_nvp("antiBot",                     anti_bots),
-        cereal::make_nvp("botProtection_v2",            detect_str)
+        cereal::make_nvp("context",                         context),
+        cereal::make_nvp("webAttackMitigation",             web_attack_mitigation),
+        cereal::make_nvp("webAttackMitigationSeverity",     web_attack_mitigation_severity),
+        cereal::make_nvp("webAttackMitigationAction",       web_attack_mitigation_action),
+        cereal::make_nvp("webAttackMitigationMode",         web_attack_mitigation_mode),
+        cereal::make_nvp("practiceAdvancedConfig",          practice_advanced_config),
+        cereal::make_nvp("csrfProtection",                  csrf_protection_mode),
+        cereal::make_nvp("openRedirect",                    open_redirect_mode),
+        cereal::make_nvp("errorDisclosure",                 error_disclosure_mode),
+        cereal::make_nvp("practiceId",                      practice_id),
+        cereal::make_nvp("practiceName",                    practice_name),
+        cereal::make_nvp("assetId",                         asset_id),
+        cereal::make_nvp("assetName",                       asset_name),
+        cereal::make_nvp("ruleId",                          rule_id),
+        cereal::make_nvp("ruleName",                        rule_name),
+        cereal::make_nvp("schemaValidation",                schema_validation_mode == "Prevent"),
+        cereal::make_nvp("schemaValidation_v2",             schema_validation_mode),
+        cereal::make_nvp("oas",                             schema_validation_oas),
+        cereal::make_nvp("schemaValidationEnforceLevel",    schema_validation_enforce_level),
+        cereal::make_nvp("triggers",                        triggers),
+        cereal::make_nvp("applicationUrls",                 application_urls),
+        cereal::make_nvp("overrides",                       overrides),
+        cereal::make_nvp("trustedSources",                  trusted_sources),
+        cereal::make_nvp("waapParameters",                  empty_list),
+        cereal::make_nvp("botProtection",                   false),
+        cereal::make_nvp("antiBot",                         anti_bots),
+        cereal::make_nvp("botProtection_v2",                bot_protection != "" ? bot_protection : string("Detect"))
     );
 }
 

components/security_apps/local_policy_mgmt_gen/exceptions_section.cc

@@ -146,7 +146,9 @@ AppsecException::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec exception";
     parseAppsecJSONKey<string>("name", name, archive_in);
-    archive_in(CEREAL_NVP(exception_spec));
+    AppsecExceptionSpec single_exception_spec;
+    single_exception_spec.load(archive_in);
+    exception_spec.push_back(single_exception_spec);
 }
 
 void
@@ -174,7 +176,7 @@ ExceptionMatch::ExceptionMatch(const AppsecExceptionSpec &parsed_exception)
 {
     bool single_condition = parsed_exception.isOneCondition();
     for (auto &attrib : attributes) {
-        auto &attrib_name = attrib.first;
+        auto attrib_name = (attrib.first == "sourceIp" ? "sourceIP" : attrib.first);
         auto &attrib_getter = attrib.second;
         auto exceptions_value = attrib_getter(parsed_exception);
         if (exceptions_value.empty()) continue;

components/security_apps/local_policy_mgmt_gen/include/appsec_practice_section.h

@@ -275,7 +275,7 @@ public:
         const LogTriggerSection &parsed_log_trigger,
         const std::string &default_mode,
         const AppSecTrustedSources &parsed_trusted_sources,
-        const std::vector<InnerException> &parsed_exceptions
+        const std::map<std::string, std::vector<InnerException>> &exceptions
     );
 
     // used for V1beta2
@@ -290,6 +290,10 @@ public:
         const std::string &_context,
         const std::string &_web_attack_mitigation_severity,
         const std::string &_web_attack_mitigation_mode,
+        const std::string &_bot_protection,
+        const std::string &schema_validation_mode,
+        const std::string &schema_validation_enforce_level,
+        const std::vector<std::string> &schema_validation_oas,
         const PracticeAdvancedConfig &_practice_advanced_config,
         const AppsecPracticeAntiBotSection &_anti_bots,
         const LogTriggerSection &parsed_log_trigger,
@@ -301,26 +305,30 @@ public:
     bool operator< (const WebAppSection &other) const;
 
 private:
-    std::string application_urls;
-    std::string asset_id;
-    std::string asset_name;
-    std::string rule_id;
-    std::string rule_name;
-    std::string practice_id;
-    std::string practice_name;
-    std::string context;
-    std::string web_attack_mitigation_action;
-    std::string web_attack_mitigation_severity;
-    std::string web_attack_mitigation_mode;
-    std::string csrf_protection_mode;
-    std::string open_redirect_mode;
-    std::string error_disclosure_mode;
-    bool web_attack_mitigation;
-    std::vector<TriggersInWaapSection> triggers;
-    PracticeAdvancedConfig practice_advanced_config;
-    AppsecPracticeAntiBotSection anti_bots;
-    std::vector<AppSecTrustedSources> trusted_sources;
-    std::vector<AppSecOverride> overrides;
+    bool                                    web_attack_mitigation;
+    std::string                             application_urls;
+    std::string                             asset_id;
+    std::string                             asset_name;
+    std::string                             rule_id;
+    std::string                             rule_name;
+    std::string                             practice_id;
+    std::string                             practice_name;
+    std::string                             context;
+    std::string                             web_attack_mitigation_action;
+    std::string                             web_attack_mitigation_severity;
+    std::string                             web_attack_mitigation_mode;
+    std::string                             csrf_protection_mode;
+    std::string                             open_redirect_mode;
+    std::string                             error_disclosure_mode;
+    std::string                             bot_protection;
+    std::string                             schema_validation_mode;
+    std::string                             schema_validation_enforce_level;
+    std::vector<std::string>                schema_validation_oas;
+    PracticeAdvancedConfig                  practice_advanced_config;
+    AppsecPracticeAntiBotSection            anti_bots;
+    std::vector<AppSecOverride>             overrides;
+    std::vector<AppSecTrustedSources>       trusted_sources;
+    std::vector<TriggersInWaapSection>      triggers;
 };
 
 class WebAPISection
@@ -408,7 +416,7 @@ class ParsedRule
 {
 public:
     ParsedRule() {}
-    ParsedRule(const std::string &_host) : host(_host) {}
+    ParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
     const std::vector<std::string> & getExceptions() const;

components/security_apps/local_policy_mgmt_gen/include/exceptions_section.h

@@ -44,7 +44,7 @@ public:
     bool isOneCondition() const;
 
 private:
-    int conditions_number;
+    int conditions_number = 0;
     std::string action;
     std::vector<std::string> country_code;
     std::vector<std::string> country_name;

components/security_apps/local_policy_mgmt_gen/include/ingress_data.h

@@ -79,6 +79,7 @@ class DefaultBackend
 {
 public:
     void load(cereal::JSONInputArchive &);
+    bool doesExist() const;
 
 private:
     bool is_exists = false;
@@ -90,6 +91,7 @@ public:
     void load(cereal::JSONInputArchive &archive_in);
 
     const std::vector<IngressDefinedRule> & getRules() const;
+    bool doesDefaultBackendExist() const;
 
 private:
     std::string ingress_class_name;

components/security_apps/local_policy_mgmt_gen/include/k8s_policy_utils.h

@@ -24,6 +24,7 @@
 #include "maybe_res.h"
 #include "i_orchestration_tools.h"
 #include "i_shell_cmd.h"
+#include "i_encryptor.h"
 #include "i_messaging.h"
 #include "i_env_details.h"
 #include "i_agent_details.h"
@@ -40,6 +41,7 @@ class K8sPolicyUtils
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_ShellCmd>,
     Singleton::Consume<I_EnvDetails>,
+    Singleton::Consume<I_Encryptor>,
     Singleton::Consume<I_AgentDetails>
 {
 public:
@@ -80,6 +82,8 @@ private:
 
     void createSnortFile(std::vector<NewAppSecPracticeSpec> &practices) const;
 
+    void createSchemaValidationOas(std::vector<NewAppSecPracticeSpec> &practices) const;
+
     template<class T>
     std::vector<T> extractV1Beta2ElementsFromCluster(
         const std::string &crd_plural,
@@ -112,6 +116,7 @@ private:
     I_Messaging* messaging = nullptr;
     EnvType env_type;
     std::string token;
+    std::string agent_ns;
 };
 
 #endif // __K8S_POLICY_UTILS_H__

components/security_apps/local_policy_mgmt_gen/include/local_policy_common.h

@@ -49,6 +49,13 @@ static const std::unordered_map<std::string, TriggerType> string_to_trigger_type
     { "WebUserResponse", TriggerType::WebUserResponse }
 };
 
+static const std::unordered_map<std::string, std::string> key_to_mitigation_severity = {
+    { "high", "High"},
+    { "medium", "Medium"},
+    { "critical", "Critical"},
+    { "Transparent", "Transparent"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -57,6 +64,14 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val =
     { "inactive", "Inactive"}
 };
 
+static const std::unordered_map<std::string, std::string> key_to_practices_mode_val = {
+    { "prevent-learn", "Prevent"},
+    { "detect-learn", "Detect"},
+    { "prevent", "Prevent"},
+    { "detect", "Detect"},
+    { "inactive", "Disabled"}
+};
+
 static const std::unordered_map<std::string, std::string> key_to_practices_val2 = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Learn"},
@@ -66,6 +81,8 @@ static const std::unordered_map<std::string, std::string> key_to_practices_val2
 };
 
 static const std::string default_appsec_url = "http://*:*";
+static const std::string default_appsec_name = "Any";
+
 
 class PolicyGenException : public std::exception
 {

components/security_apps/local_policy_mgmt_gen/include/new_appsec_policy_crd_parser.h

@@ -31,7 +31,7 @@ class NewParsedRule
 {
 public:
     NewParsedRule() {}
-    NewParsedRule(const std::string &_host) : host(_host) {}
+    NewParsedRule(const std::string &_host, const std::string &_mode) : host(_host), mode(_mode) {}
 
     void load(cereal::JSONInputArchive &archive_in);
 

components/security_apps/local_policy_mgmt_gen/include/new_log_trigger.h

@@ -129,7 +129,7 @@ public:
     bool shouldBeautifyLogs() const;
 
     bool getCloud() const;
-    bool isK8SNeeded() const;
+    bool isContainerNeeded() const;
     bool isCefNeeded() const;
     bool isSyslogNeeded() const;
     const std::string & getSyslogServerIpv4Address() const;
@@ -140,7 +140,7 @@ private:
     const NewLoggingService & getCefServiceData() const;
 
     bool cloud = false;
-    bool k8s_service = false;
+    bool container_service = false;
     bool agent_local = true;
     bool beautify_logs = true;
     NewLoggingService syslog_service;

components/security_apps/local_policy_mgmt_gen/include/new_practice.h

@@ -23,6 +23,8 @@
 #include "config.h"
 #include "debug.h"
 #include "local_policy_common.h"
+#include "i_orchestration_tools.h"
+#include "i_encryptor.h"
 
 bool isModeInherited(const std::string &mode);
 
@@ -88,6 +90,8 @@ public:
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
+    bool operator<(const IpsProtectionsSection &other) const;
+
 private:
     std::string                                 context;
     std::string                                 name;
@@ -105,7 +109,7 @@ public:
     // LCOV_EXCL_START Reason: no test exist
     IPSSection() {};
 
-    IPSSection(const std::vector<IpsProtectionsSection> &_ips) : ips(_ips) {};
+    IPSSection(const std::vector<IpsProtectionsSection> &_ips);
     // LCOV_EXCL_STOP
 
     void save(cereal::JSONOutputArchive &out_ar) const;
@@ -138,6 +142,12 @@ public:
     const std::string & getMode(const std::string &default_mode = "inactive") const;
 
 private:
+
+    const std::string & getRulesMode(
+        const std::string &mode,
+        const std::string &default_mode = "inactive"
+    ) const;
+
     std::string override_mode;
     std::string max_performance_impact;
     std::string min_severity_level;
@@ -487,15 +497,16 @@ private:
     SnortSection snort;
 };
 
-class NewSnortSignaturesAndOpenSchemaAPI
+class NewSnortSignatures
 {
 public:
-    NewSnortSignaturesAndOpenSchemaAPI() : is_temporary(false) {};
+    NewSnortSignatures() : is_temporary(false) {};
 
     void load(cereal::JSONInputArchive &archive_in);
 
     void addFile(const std::string &file_name);
     const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
     const std::vector<std::string> & getConfigMap() const;
     const std::vector<std::string> & getFiles() const;
     bool isTemporary() const;
@@ -503,35 +514,48 @@ public:
 
 private:
     std::string override_mode;
+    std::string enforcement_level;
     std::vector<std::string> config_map;
     std::vector<std::string> files;
     bool is_temporary;
 };
 
-class NewAppSecWebBotsURI
+class NewOpenApiSchema : Singleton::Consume<I_OrchestrationTools>, Singleton::Consume<I_Encryptor>
 {
 public:
+    NewOpenApiSchema() {};
+
     void load(cereal::JSONInputArchive &archive_in);
 
-    const std::string & getURI() const;
+    void addOas(const std::string &file);
+    const std::string & getOverrideMode(const std::string &default_mode = "inactive") const;
+    const std::string & getEnforceLevel() const;
+    const std::vector<std::string> & getConfigMap() const;
+    const std::vector<std::string> & getFiles() const;
+    const std::vector<std::string> & getOas() const;
 
 private:
-    std::string uri;
+    std::string override_mode;
+    std::string enforcement_level;
+    std::vector<std::string> config_map;
+    std::vector<std::string> files;
+    std::vector<std::string> oas;
 };
 
 class NewAppSecPracticeAntiBot
 {
 public:
-    std::vector<std::string> getIjectedUris() const;
-    std::vector<std::string> getValidatedUris() const;
+    const std::vector<std::string> & getIjectedUris() const;
+    const std::vector<std::string> & getValidatedUris() const;
+    const std::string & getMode(const std::string &default_mode = "inactive") const;
 
     void load(cereal::JSONInputArchive &archive_in);
     void save(cereal::JSONOutputArchive &out_ar) const;
 
 private:
     std::string override_mode;
-    std::vector<NewAppSecWebBotsURI> injected_uris;
-    std::vector<NewAppSecWebBotsURI> validated_uris;
+    std::vector<std::string> injected_uris;
+    std::vector<std::string> validated_uris;
 };
 
 class NewAppSecWebAttackProtections
@@ -579,8 +603,8 @@ class NewAppSecPracticeSpec
 public:
     void load(cereal::JSONInputArchive &archive_in);
 
-    NewSnortSignaturesAndOpenSchemaAPI & getSnortSignatures();
-    const NewSnortSignaturesAndOpenSchemaAPI & getOpenSchemaValidation() const;
+    NewSnortSignatures & getSnortSignatures();
+    NewOpenApiSchema & getOpenSchemaValidation();
     const NewAppSecPracticeWebAttacks & getWebAttacks() const;
     const NewAppSecPracticeAntiBot & getAntiBot() const;
     const NewIntrusionPrevention & getIntrusionPrevention() const;
@@ -593,8 +617,8 @@ public:
 private:
     NewFileSecurity                             file_security;
     NewIntrusionPrevention                      intrusion_prevention;
-    NewSnortSignaturesAndOpenSchemaAPI          openapi_schema_validation;
-    NewSnortSignaturesAndOpenSchemaAPI          snort_signatures;
+    NewOpenApiSchema                            openapi_schema_validation;
+    NewSnortSignatures                          snort_signatures;
     NewAppSecPracticeWebAttacks                 web_attacks;
     NewAppSecPracticeAntiBot                    anti_bot;
     std::string                                 appsec_class_name;

components/security_apps/local_policy_mgmt_gen/include/policy_maker_utils.h

@@ -111,7 +111,7 @@ private:
     SecurityAppsWrapper security_apps;
 };
 
-class PolicyMakerUtils
+class PolicyMakerUtils : Singleton::Consume<I_EnvDetails>
 {
 public:
     std::string proccesSingleAppsecPolicy(

components/security_apps/local_policy_mgmt_gen/include/rules_config_section.h

@@ -123,6 +123,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 
@@ -145,6 +146,7 @@ public:
     );
 
     const std::string & getIdentifier() const;
+    const std::string & getIdentifierValue() const;
 
     void save(cereal::JSONOutputArchive &out_ar) const;
 

components/security_apps/local_policy_mgmt_gen/include/triggers_section.h

@@ -39,7 +39,7 @@ public:
         bool _logToAgent,
         bool _logToCef,
         bool _logToCloud,
-        bool _logToK8sService,
+        bool _logTolocalTuning,
         bool _logToSyslog,
         bool _responseBody,
         bool _tpDetect,
@@ -73,7 +73,7 @@ private:
     bool logToAgent;
     bool logToCef;
     bool logToCloud;
-    bool logToK8sService;
+    bool logTolocalTuning;
     bool logToSyslog;
     bool responseBody;
     bool tpDetect;
@@ -258,7 +258,7 @@ public:
     bool shouldBeautifyLogs() const;
 
     bool getCloud() const;
-    bool isK8SNeeded() const;
+    bool isContainerNeeded() const;
     bool isCefNeeded() const;
     bool isSyslogNeeded() const;
     const std::string & getSyslogServerIpv4Address() const;
@@ -269,7 +269,7 @@ private:
     const LoggingService & getCefServiceData() const;
 
     bool cloud = false;
-    bool k8s_service = false;
+    bool container_service = false;
     bool agent_local = true;
     bool beautify_logs = true;
     LoggingService syslog_service;

components/security_apps/local_policy_mgmt_gen/ingress_data.cc

@@ -86,6 +86,12 @@ DefaultBackend::load(cereal::JSONInputArchive &)
     is_exists = true;
 }
 
+bool
+DefaultBackend::doesExist() const
+{
+    return is_exists;
+}
+
 void
 IngressSpec::load(cereal::JSONInputArchive &archive_in)
 {
@@ -101,6 +107,12 @@ IngressSpec::getRules() const
     return rules;
 }
 
+bool
+IngressSpec::doesDefaultBackendExist() const
+{
+    return default_backend.doesExist();
+}
+
 void
 SingleIngressData::load(cereal::JSONInputArchive &archive_in)
 {

components/security_apps/local_policy_mgmt_gen/k8s_policy_utils.cc

@@ -35,6 +35,14 @@ convertAnnotationKeysTostring(const AnnotationKeys &key)
     }
 }
 
+string
+getAppSecScopeType()
+{
+    auto env_res = getenv("CRDS_SCOPE");
+    if (env_res != nullptr) return env_res;
+    return "cluster";
+}
+
 void
 K8sPolicyUtils::init()
 {
@@ -42,6 +50,7 @@ K8sPolicyUtils::init()
     env_type = env_details->getEnvType();
     if (env_type == EnvType::K8S) {
         token = env_details->getToken();
+        agent_ns = getAppSecScopeType() == "namespaced" ? env_details->getNameSpace() + "/" : "";
         messaging = Singleton::Consume<I_Messaging>::by<K8sPolicyUtils>();
     }
 }
@@ -140,10 +149,12 @@ extractElementsFromNewRule(
     const NewParsedRule &rule,
     map<AnnotationTypes, unordered_set<string>> &policy_elements_names)
 {
-    policy_elements_names[AnnotationTypes::EXCEPTION].insert(
-        rule.getExceptions().begin(),
-        rule.getExceptions().end()
-    );
+    if (rule.getExceptions().size() > 0) {
+        policy_elements_names[AnnotationTypes::EXCEPTION].insert(
+            rule.getExceptions().begin(),
+            rule.getExceptions().end()
+        );
+    }
     policy_elements_names[AnnotationTypes::THREAT_PREVENTION_PRACTICE].insert(
         rule.getPractices().begin(),
         rule.getPractices().end()
@@ -152,14 +163,24 @@ extractElementsFromNewRule(
         rule.getAccessControlPractices().begin(),
         rule.getAccessControlPractices().end()
     );
-    policy_elements_names[AnnotationTypes::TRIGGER].insert(
-        rule.getLogTriggers().begin(),
-        rule.getLogTriggers().end()
-    );
-    policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
-    policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
-    policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
-    policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
+    if (rule.getLogTriggers().size() > 0) {
+        policy_elements_names[AnnotationTypes::TRIGGER].insert(
+            rule.getLogTriggers().begin(),
+            rule.getLogTriggers().end()
+        );
+    }
+    if (rule.getCustomResponse() != "" ) {
+        policy_elements_names[AnnotationTypes::WEB_USER_RES].insert(rule.getCustomResponse());
+    }
+    if (rule.getSourceIdentifiers() != "" ) {
+        policy_elements_names[AnnotationTypes::SOURCE_IDENTIFIERS].insert(rule.getSourceIdentifiers());
+    }
+    if (rule.getTrustedSources() != "" ) {
+        policy_elements_names[AnnotationTypes::TRUSTED_SOURCES].insert(rule.getTrustedSources());
+    }
+    if (rule.getUpgradeSettings() != "" ) {
+        policy_elements_names[AnnotationTypes::UPGRADE_SETTINGS].insert(rule.getUpgradeSettings());
+    }
 }
 
 map<AnnotationTypes, unordered_set<string>>
@@ -259,9 +280,11 @@ K8sPolicyUtils::extractV1Beta2ElementsFromCluster(
     dbgTrace(D_LOCAL_POLICY) << "Retrieve AppSec elements. type: " << crd_plural;
     vector<T> elements;
     for (const string &element_name : elements_names) {
+        string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+        string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
         dbgTrace(D_LOCAL_POLICY) << "AppSec element name: " << element_name;
         auto maybe_appsec_element = getObjectFromCluster<AppsecSpecParser<T>>(
-            "/apis/openappsec.io/v1beta2/" + crd_plural + "/" + element_name
+            "/apis/openappsec.io/v1beta2/" + ns + agent_ns + crd_plural + ns_suffix + "/" + element_name
         );
 
         if (!maybe_appsec_element.ok()) {
@@ -362,8 +385,9 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
         practice.getSnortSignatures().setTemporary(true);
         for (const string &config_map : practice.getSnortSignatures().getConfigMap())
         {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
             auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
-                "/api/v1/namespaces/default/configmaps/" + config_map
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
             );
             if (!maybe_configmap.ok())  {
                 dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
@@ -381,6 +405,28 @@ K8sPolicyUtils::createSnortFile(vector<NewAppSecPracticeSpec> &practices) const
     }
 }
 
+void
+K8sPolicyUtils::createSchemaValidationOas(vector<NewAppSecPracticeSpec> &practices) const
+{
+    for (NewAppSecPracticeSpec &practice : practices) {
+        vector<string> res;
+        for (const string &config_map : practice.getOpenSchemaValidation().getConfigMap())
+        {
+            string ns = agent_ns == "" ? "default/" : agent_ns;
+            auto maybe_configmap = getObjectFromCluster<ConfigMaps>(
+                "/api/v1/namespaces/" + ns + "configmaps/" + config_map
+            );
+            if (!maybe_configmap.ok())  {
+                dbgWarning(D_LOCAL_POLICY) << "Failed to get configMaps from the cluster.";
+                continue;
+            }
+            string file_content = maybe_configmap.unpack().getFileContent();
+            string res = Singleton::Consume<I_Encryptor>::by<K8sPolicyUtils>()->base64Encode(file_content);
+            practice.getOpenSchemaValidation().addOas(res);
+        }
+    }
+}
+
 Maybe<V1beta2AppsecLinuxPolicy>
 K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     const AppsecSpecParser<NewAppsecPolicySpec> &appsec_policy_spec,
@@ -396,6 +442,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
     }
 
     if (default_rule.getMode().empty() && !ingress_mode.empty()) {
+        dbgTrace(D_LOCAL_POLICY) << "setting the policy default rule mode to the ingress mode: " << ingress_mode;
         default_rule.setMode(ingress_mode);
     }
 
@@ -411,6 +458,7 @@ K8sPolicyUtils::createAppsecPolicyK8sFromV1beta2Crds(
         );
 
     createSnortFile(threat_prevention_practices);
+    createSchemaValidationOas(threat_prevention_practices);
 
     vector<AccessControlPracticeSpec> access_control_practices =
         extractV1Beta2ElementsFromCluster<AccessControlPracticeSpec>(
@@ -493,9 +541,12 @@ K8sPolicyUtils::createAppsecPolicyK8s(const string &policy_name, const string &i
                 maybe_appsec_policy_spec.ok() ? "There is no v1beta1 policy" : maybe_appsec_policy_spec.getErr();
             dbgWarning(D_LOCAL_POLICY
             ) << "Failed to retrieve Appsec policy with crds version: v1beta1, Trying version: v1beta2";
+            string ns_suffix = getAppSecScopeType() == "namespaced" ? "ns" : "";
+            string ns = getAppSecScopeType() == "namespaced" ? "namespaces/" : "";
             auto maybe_v1beta2_appsec_policy_spec = getObjectFromCluster<AppsecSpecParser<NewAppsecPolicySpec>>(
-                "/apis/openappsec.io/v1beta2/policies/" + policy_name
+                "/apis/openappsec.io/v1beta2/" + ns + agent_ns + "policies" + ns_suffix + "/" + policy_name
             );
+
             if (!maybe_v1beta2_appsec_policy_spec.ok()) {
                 dbgWarning(D_LOCAL_POLICY)
                     << "Failed to retrieve AppSec policy. Error: " << maybe_v1beta2_appsec_policy_spec.getErr();
@@ -532,26 +583,38 @@ K8sPolicyUtils::createPolicy(
     map<AnnotationKeys, string> &annotations_values,
     const SingleIngressData &item) const
 {
+    if (policies.find(annotations_values[AnnotationKeys::PolicyKey]) == policies.end()) {
+        policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
+    }
+    auto default_mode = appsec_policy.getAppsecPolicySpec().getDefaultRule().getMode();
+    if (item.getSpec().doesDefaultBackendExist()) {
+        dbgTrace(D_LOCAL_POLICY)
+            << "Inserting Any host rule to the specific asset set";
+        K ingress_rule = K("*", default_mode);
+        policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
+    }
+
     for (const IngressDefinedRule &rule : item.getSpec().getRules()) {
-        string url = rule.getHost();
+        string host = rule.getHost();
         for (const IngressRulePath &uri : rule.getPathsWrapper().getRulePaths()) {
-            if (!appsec_policy.getAppsecPolicySpec().isAssetHostExist(url + uri.getPath())) {
+            if (uri.getPath() != "/") {
+                host = host + uri.getPath();
+            }
+            if (!appsec_policy.getAppsecPolicySpec().isAssetHostExist(host)) {
                 dbgTrace(D_LOCAL_POLICY)
                     << "Inserting Host data to the specific asset set:"
                     << "URL: '"
-                    << url
+                    << rule.getHost()
                     << "' uri: '"
                     << uri.getPath()
                     << "'";
-                K ingress_rule = K(url + uri.getPath());
-                appsec_policy.addSpecificRule(ingress_rule);
+                K ingress_rule = K(host, default_mode);
+                policies[annotations_values[AnnotationKeys::PolicyKey]].addSpecificRule(ingress_rule);
             }
         }
     }
-    policies[annotations_values[AnnotationKeys::PolicyKey]] = appsec_policy;
 }
 
-
 std::tuple<map<string, AppsecLinuxPolicy>, map<string, V1beta2AppsecLinuxPolicy>>
 K8sPolicyUtils::createAppsecPoliciesFromIngresses()
 {

components/security_apps/local_policy_mgmt_gen/new_appsec_policy_crd_parser.cc

@@ -126,6 +126,7 @@ NewAppsecPolicySpec::load(cereal::JSONInputArchive &archive_in)
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec policy spec";
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseAppsecJSONKey<NewParsedRule>("default", default_rule, archive_in);
+    default_rule.setHost("*");
     parseAppsecJSONKey<vector<NewParsedRule>>("specificRules", specific_rules, archive_in);
 }
 

components/security_apps/local_policy_mgmt_gen/new_log_trigger.cc

@@ -27,7 +27,7 @@ NewAppsecTriggerAccessControlLogging::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Access Control Logging";
     parseAppsecJSONKey<bool>("allowEvents", ac_allow_events, archive_in, false);
-    parseAppsecJSONKey<bool>("dropEvents", ac_drop_events, archive_in, false);
+    parseAppsecJSONKey<bool>("dropEvents", ac_drop_events, archive_in, true);
 }
 
 void
@@ -36,8 +36,7 @@ NewAppsecTriggerAdditionalSuspiciousEventsLogging::load(cereal::JSONInputArchive
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger - Additional Suspicious Events Logging";
     parseAppsecJSONKey<bool>("enabled", enabled, archive_in, true);
     parseAppsecJSONKey<bool>("responseBody", response_body, archive_in, false);
-    //the old code didn't parse the responsecode so ask Noam what is the currenct default value for it
-    parseAppsecJSONKey<bool>("responseCode", response_code, archive_in, false);
+    parseAppsecJSONKey<bool>("responseCode", response_code, archive_in, true);
     parseAppsecJSONKey<string>("minSeverity", minimum_severity, archive_in, "high");
     if (valid_severities.count(minimum_severity) == 0) {
         dbgWarning(D_LOCAL_POLICY)
@@ -133,7 +132,7 @@ void
 NewLoggingService::load(cereal::JSONInputArchive &archive_in)
 {
     parseAppsecJSONKey<string>("address", address, archive_in);
-    parseAppsecJSONKey<string>("proto", proto, archive_in);
+    parseAppsecJSONKey<string>("proto", proto, archive_in, "tcp");
     if (valid_protocols.count(proto) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Logging Service - proto invalid: " << proto;
         throw PolicyGenException("AppSec Logging Service - proto invalid: " + proto);
@@ -175,16 +174,26 @@ void
 NewAppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination";
-    // TBD: support "file"
-    parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
-    auto mode = Singleton::Consume<I_AgentDetails>::by<NewAppsecTriggerLogDestination>()->getOrchestrationMode();
-    auto env_type = Singleton::Consume<I_EnvDetails>::by<NewAppsecTriggerLogDestination>()->getEnvType();
-    bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S);
-    parseAppsecJSONKey<bool>("k8s-service", k8s_service, archive_in, k8s_service_default);
+    if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") {
+        // TBD: support "file"
+        parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    } else {
+        cloud = false;
+    }
+    bool local_tuning_default = false;
+    // check ENV VAR LOCAL_TUNING_ENABLED
+    char * tuning_enabled = getenv("LOCAL_TUNING_ENABLED");
+    if (tuning_enabled != NULL) {
+        for (unsigned int i = 0; i < strlen(tuning_enabled); i++) {
+            tuning_enabled[i] = tolower(tuning_enabled[i]);
+        }
+        local_tuning_default = string(tuning_enabled) == "true";
+    }
+    parseAppsecJSONKey<bool>("local-tuning", container_service, archive_in, local_tuning_default);
 
     NewStdoutLogging stdout_log;
     parseAppsecJSONKey<NewStdoutLogging>("stdout", stdout_log, archive_in);
-    agent_local = !(stdout_log.getFormat().empty());
+    parseAppsecJSONKey<bool>("logToAgent", agent_local, archive_in, true);
     beautify_logs = stdout_log.getFormat() == "json-formatted";
     parseAppsecJSONKey<NewLoggingService>("syslogService", syslog_service, archive_in);
     parseAppsecJSONKey<NewLoggingService>("cefService", cef_service, archive_in);
@@ -221,9 +230,9 @@ NewAppsecTriggerLogDestination::getCloud() const
 }
 
 bool
-NewAppsecTriggerLogDestination::isK8SNeeded() const
+NewAppsecTriggerLogDestination::isContainerNeeded() const
 {
-    return k8s_service;
+    return container_service;
 }
 
 bool

components/security_apps/local_policy_mgmt_gen/new_practice.cc

@@ -22,6 +22,7 @@ static const set<string> performance_impacts    = {"low", "medium", "high"};
 static const set<string> severity_levels        = {"low", "medium", "high", "critical"};
 static const set<string> size_unit              = {"bytes", "KB", "MB", "GB"};
 static const set<string> confidences_actions    = {"prevent", "detect", "inactive", "as-top-level", "inherited"};
+static const set<string> valied_enforcement_level = {"fullSchema", "endpointOnly"};
 static const set<string> valid_modes = {
     "prevent",
     "detect",
@@ -32,31 +33,38 @@ static const set<string> valid_modes = {
     "inherited"
 };
 static const set<string> valid_confidences      = {"medium", "high", "critical"};
-static const std::unordered_map<std::string, std::string> key_to_performance_impact_val = {
+static const unordered_map<string, string> key_to_performance_impact_val = {
     { "low", "Low or lower"},
     { "medium", "Medium or lower"},
     { "high", "High or lower"}
 };
-static const std::unordered_map<std::string, std::string> key_to_severity_level_val = {
+static const unordered_map<string, string> key_to_severity_level_val = {
     { "low", "Low or above"},
     { "medium", "Medium or above"},
     { "high", "High or above"},
     { "critical", "Critical"}
 };
-static const std::unordered_map<std::string, std::string> key_to_mode_val = {
+static const unordered_map<string, string> key_to_mode_val = {
     { "prevent-learn", "Prevent"},
     { "detect-learn", "Detect"},
     { "prevent", "Prevent"},
     { "detect", "Detect"},
     { "inactive", "Inactive"}
 };
-static const std::unordered_map<std::string, uint64_t> unit_to_int = {
+static const unordered_map<string, string> anti_bot_key_to_mode_val = {
+    { "prevent-learn", "Prevent"},
+    { "detect-learn", "Detect"},
+    { "prevent", "Prevent"},
+    { "detect", "Detect"},
+    { "inactive", "Disabled"}
+};
+static const unordered_map<string, uint64_t> unit_to_int = {
     { "bytes", 1},
     { "KB", 1024},
     { "MB", 1048576},
     { "GB", 1073741824}
 };
-static const std::string TRANSPARENT_MODE = "Transparent";
+static const string TRANSPARENT_MODE = "Transparent";
 
 bool
 isModeInherited(const string &mode)
@@ -64,11 +72,11 @@ isModeInherited(const string &mode)
     return mode == "as-top-level" || mode == "inherited";
 }
 
-const std::string &
+const string &
 getModeWithDefault(
-    const std::string &mode,
-    const std::string &default_mode,
-    const std::unordered_map<std::string, std::string> &key_to_val)
+    const string &mode,
+    const string &default_mode,
+    const unordered_map<string, string> &key_to_val)
 {
     if (isModeInherited(mode) && (key_to_val.find(default_mode) != key_to_val.end())) {
         dbgError(D_LOCAL_POLICY) << "Setting to top-level mode: " << default_mode;
@@ -81,57 +89,43 @@ getModeWithDefault(
     return key_to_val.at(mode);
 }
 
-void
-NewAppSecWebBotsURI::load(cereal::JSONInputArchive &archive_in)
+const vector<string> &
+NewAppSecPracticeAntiBot::getIjectedUris() const
 {
-    dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots URI";
-    parseAppsecJSONKey<string>("uri", uri, archive_in);
+    return injected_uris;
+}
+
+const vector<string> &
+NewAppSecPracticeAntiBot::getValidatedUris() const
+{
+    return validated_uris;
 }
 
 const string &
-NewAppSecWebBotsURI::getURI() const
+NewAppSecPracticeAntiBot::getMode(const string &default_mode) const
 {
-    return uri;
-}
-
-std::vector<std::string>
-NewAppSecPracticeAntiBot::getIjectedUris() const
-{
-    vector<string> injected;
-    for (const NewAppSecWebBotsURI &uri : injected_uris) injected.push_back(uri.getURI());
-    return injected;
-}
-
-std::vector<std::string>
-NewAppSecPracticeAntiBot::getValidatedUris() const
-{
-    vector<string> validated;
-    for (const NewAppSecWebBotsURI &uri : validated_uris) validated.push_back(uri.getURI());
-    return validated;
+    return getModeWithDefault(override_mode, default_mode, anti_bot_key_to_mode_val);
 }
 
 void
 NewAppSecPracticeAntiBot::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Web Bots";
-    parseAppsecJSONKey<vector<NewAppSecWebBotsURI>>("injectedUris", injected_uris, archive_in);
-    parseAppsecJSONKey<vector<NewAppSecWebBotsURI>>("validatedUris", validated_uris, archive_in);
+    parseAppsecJSONKey<vector<string>>("injectedUris", injected_uris, archive_in);
+    parseAppsecJSONKey<vector<string>>("validatedUris", validated_uris, archive_in);
     parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
     if (valid_modes.count(override_mode) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Web Bots override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Web Bots override mode invalid: " + override_mode);
     }
 }
 
 void
 NewAppSecPracticeAntiBot::save(cereal::JSONOutputArchive &out_ar) const
 {
-    vector<string> injected;
-    vector<string> validated;
-    for (const NewAppSecWebBotsURI &uri : injected_uris) injected.push_back(uri.getURI());
-    for (const NewAppSecWebBotsURI &uri : validated_uris) validated.push_back(uri.getURI());
     out_ar(
-        cereal::make_nvp("injected", injected),
-        cereal::make_nvp("validated", validated)
+        cereal::make_nvp("injected", injected_uris),
+        cereal::make_nvp("validated", validated_uris)
     );
 }
 
@@ -248,14 +242,14 @@ NewAppSecPracticeWebAttacks::getProtections() const
 }
 
 SnortProtectionsSection::SnortProtectionsSection(
-    const std::string               &_context,
-    const std::string               &_asset_name,
-    const std::string               &_asset_id,
-    const std::string               &_practice_name,
-    const std::string               &_practice_id,
-    const std::string               &_source_identifier,
-    const std::string               &_mode,
-    const std::vector<std::string>  &_files)
+    const string               &_context,
+    const string               &_asset_name,
+    const string               &_asset_id,
+    const string               &_practice_name,
+    const string               &_practice_id,
+    const string               &_source_identifier,
+    const string               &_mode,
+    const vector<string>  &_files)
         :
     context(_context),
     asset_name(_asset_name),
@@ -284,10 +278,10 @@ SnortProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 DetectionRules::DetectionRules(
-    const std::string                   &_type,
-    const std::string                   &_SSM,
-    const std::string                   &_keywords,
-    const std::vector<std::string>      &_context)
+    const string                   &_type,
+    const string                   &_SSM,
+    const string                   &_keywords,
+    const vector<string>      &_context)
         :
     type(_type),
     SSM(_SSM),
@@ -320,14 +314,14 @@ DetectionRules::save(cereal::JSONOutputArchive &out_ar) const
 
 ProtectionMetadata::ProtectionMetadata(
     bool                                _silent,
-    const std::string                   &_protection_name,
-    const std::string                   &_severity,
-    const std::string                   &_confidence_level,
-    const std::string                   &_performance_impact,
-    const std::string                   &_last_update,
-    const std::string                   &_maintrain_id,
-    const std::vector<std::string>      &_tags,
-    const std::vector<std::string>      &_cve_list)
+    const string                   &_protection_name,
+    const string                   &_severity,
+    const string                   &_confidence_level,
+    const string                   &_performance_impact,
+    const string                   &_last_update,
+    const string                   &_maintrain_id,
+    const vector<string>      &_tags,
+    const vector<string>      &_cve_list)
         :
     silent(_silent),
     protection_name(_protection_name),
@@ -400,9 +394,9 @@ ProtectionsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 ProtectionsSection::ProtectionsSection(
-    const std::vector<ProtectionsProtectionsSection>    &_protections,
-    const std::string                                   &_name,
-    const std::string                                   &_modification_time)
+    const vector<ProtectionsProtectionsSection>    &_protections,
+    const string                                   &_name,
+    const string                                   &_modification_time)
         :
     protections(_protections),
     name(_name),
@@ -466,12 +460,16 @@ SnortSectionWrapper::save(cereal::JSONOutputArchive &out_ar) const
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
+NewSnortSignatures::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Snort Signatures practice";
     parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
     parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
     parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Snort Signatures override mode invalid: " + override_mode);
+    }
     is_temporary = false;
     if (valid_modes.count(override_mode) == 0) {
         dbgWarning(D_LOCAL_POLICY) << "AppSec Snort Signatures override mode invalid: " << override_mode;
@@ -480,42 +478,107 @@ NewSnortSignaturesAndOpenSchemaAPI::load(cereal::JSONInputArchive &archive_in)
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::addFile(const string &file_name)
+NewSnortSignatures::addFile(const string &file_name)
 {
     files.push_back(file_name);
 }
 
 const string &
-NewSnortSignaturesAndOpenSchemaAPI::getOverrideMode(const string &default_mode) const
+NewSnortSignatures::getOverrideMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getFiles() const
+NewSnortSignatures::getFiles() const
 {
     return files;
 }
 
 const vector<string> &
-NewSnortSignaturesAndOpenSchemaAPI::getConfigMap() const
+NewSnortSignatures::getConfigMap() const
 {
     return config_map;
 }
 
 bool
-NewSnortSignaturesAndOpenSchemaAPI::isTemporary() const
+NewSnortSignatures::isTemporary() const
 {
     return is_temporary;
 }
 
 void
-NewSnortSignaturesAndOpenSchemaAPI::setTemporary(bool val)
+NewSnortSignatures::setTemporary(bool val)
 {
     is_temporary = val;
 }
 
+void
+NewOpenApiSchema::load(cereal::JSONInputArchive &archive_in)
+{
+    dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Schema Validation practice";
+    parseMandatoryAppsecJSONKey<string>("overrideMode", override_mode, archive_in, "inactive");
+    parseAppsecJSONKey<vector<string>>("configmap", config_map, archive_in);
+    parseAppsecJSONKey<vector<string>>("files", files, archive_in);
+    parseAppsecJSONKey<string>("enforcementLevel", enforcement_level, archive_in, "fullSchema");
+    if (valied_enforcement_level.count(enforcement_level) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation enforcement level invalid: " << enforcement_level;
+        throw PolicyGenException("AppSec Schema Validation enforcement level invalid: " + enforcement_level);
+    }
+    if (valid_modes.count(override_mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Schema Validation override mode invalid: " << override_mode;
+        throw PolicyGenException("AppSec Schema Validation override mode invalid: " + override_mode);
+    }
+    for (const string &file : files)
+    {
+        auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<NewOpenApiSchema>();
+        auto file_content = i_orchestration_tools->readFile(file);
+        if (!file_content.ok()) {
+            dbgWarning(D_LOCAL_POLICY) << "Couldn't open the schema validation file";
+            continue;
+        }
+        oas.push_back(Singleton::Consume<I_Encryptor>::by<NewOpenApiSchema>()->base64Encode(file_content.unpack()));
+    }
+}
+
+void
+NewOpenApiSchema::addOas(const string &file)
+{
+    oas.push_back(file);
+}
+
+const string &
+NewOpenApiSchema::getOverrideMode(const string &default_mode) const
+{
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val2);
+    return res;
+}
+
+const string &
+NewOpenApiSchema::getEnforceLevel() const
+{
+    return enforcement_level;
+}
+
+const vector<string> &
+NewOpenApiSchema::getFiles() const
+{
+    return files;
+}
+
+const vector<string> &
+NewOpenApiSchema::getConfigMap() const
+{
+    return config_map;
+}
+
+const vector<string> &
+NewOpenApiSchema::getOas() const
+{
+    return oas;
+}
+
 void
 IpsProtectionsRulesSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -554,7 +617,7 @@ IpsProtectionsSection::IpsProtectionsSection(
 {
 }
 
-std::string &
+string &
 IpsProtectionsSection::getMode()
 {
     return mode;
@@ -576,6 +639,20 @@ IpsProtectionsSection::save(cereal::JSONOutputArchive &out_ar) const
     );
 }
 
+bool
+IpsProtectionsSection::operator<(const IpsProtectionsSection &other) const
+{
+    // for sorting from the most specific to the least specific rule
+    if (name == default_appsec_name) return false;
+    if (other.name == default_appsec_name) return true;
+    return name.size() > other.name.size();
+}
+
+IPSSection::IPSSection(const vector<IpsProtectionsSection> &_ips) : ips(_ips)
+{
+    sort(ips.begin(), ips.end());
+}
+
 void
 IPSSection::save(cereal::JSONOutputArchive &out_ar) const
 {
@@ -654,7 +731,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     vector<IpsProtectionsRulesSection> ips_rules;
     IpsProtectionsRulesSection high_rule(
         min_cve_Year,
-        getModeWithDefault(high_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(high_confidence_event_action, default_mode),
         string("High"),
         max_performance_impact,
         string(""),
@@ -664,7 +741,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection med_rule(
         min_cve_Year,
-        getModeWithDefault(medium_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(medium_confidence_event_action, default_mode),
         string("Medium"),
         max_performance_impact,
         string(""),
@@ -674,7 +751,7 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
 
     IpsProtectionsRulesSection low_rule(
         min_cve_Year,
-        getModeWithDefault(low_confidence_event_action, default_mode, key_to_practices_val),
+        getRulesMode(low_confidence_event_action, default_mode),
         string("Low"),
         max_performance_impact,
         string(""),
@@ -685,33 +762,45 @@ NewIntrusionPrevention::createIpsRules(const string &default_mode) const
     return ips_rules;
 }
 
-const std::string &
-NewIntrusionPrevention::getMode(const std::string &default_mode) const
+const string &
+NewIntrusionPrevention::getMode(const string &default_mode) const
 {
-    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_val);
+    const string &res = getModeWithDefault(override_mode, default_mode, key_to_practices_mode_val);
     return res;
 }
 
+const string &
+NewIntrusionPrevention::getRulesMode(const string &mode, const string &default_mode) const
+{
+    if (isModeInherited(mode)) return default_mode;
+
+    if (key_to_practices_mode_val.find(mode) == key_to_practices_mode_val.end()) {
+        dbgError(D_LOCAL_POLICY) << "Given mode: " << mode << " or top-level: " << default_mode << " is invalid.";
+        return key_to_practices_mode_val.at("inactive");
+    }
+    return key_to_practices_mode_val.at(mode);
+}
+
 FileSecurityProtectionsSection::FileSecurityProtectionsSection(
     uint64_t                _file_size_limit,
     uint64_t                _archive_file_size_limit,
     bool                    _allow_files_without_name,
     bool                    _required_file_size_limit,
     bool                    _required_archive_extraction,
-    const std::string       &_context,
-    const std::string       &_name,
-    const std::string       &_asset_id,
-    const std::string       &_practice_name,
-    const std::string       &_practice_id,
-    const std::string       &_action,
-    const std::string       &_files_without_name_action,
-    const std::string       &_high_confidence_action,
-    const std::string       &_medium_confidence_action,
-    const std::string       &_low_confidence_action,
-    const std::string       &_severity_level,
-    const std::string       &_file_size_limit_action,
-    const std::string       &_multi_level_archive_action,
-    const std::string       &_unopened_archive_action)
+    const string       &_context,
+    const string       &_name,
+    const string       &_asset_id,
+    const string       &_practice_name,
+    const string       &_practice_id,
+    const string       &_action,
+    const string       &_files_without_name_action,
+    const string       &_high_confidence_action,
+    const string       &_medium_confidence_action,
+    const string       &_low_confidence_action,
+    const string       &_severity_level,
+    const string       &_file_size_limit_action,
+    const string       &_multi_level_archive_action,
+    const string       &_unopened_archive_action)
         :
     file_size_limit(_file_size_limit),
     archive_file_size_limit(_archive_file_size_limit),
@@ -837,13 +926,13 @@ NewFileSecurityArchiveInspection::getrequiredArchiveExtraction() const
     return extract_archive_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getMultiLevelArchiveAction() const
 {
     return archived_files_within_archived_files;
 }
 
-const std::string &
+const string &
 NewFileSecurityArchiveInspection::getUnopenedArchiveAction() const
 {
     return archived_files_where_content_extraction_failed;
@@ -892,7 +981,7 @@ NewFileSecurityLargeFileInspection::getFileSizeLimit() const
     return (file_size_limit * unit_to_int.at(file_size_limit_unit));
 }
 
-const std::string &
+const string &
 NewFileSecurityLargeFileInspection::getFileSizeLimitAction() const
 {
     return files_exceeding_size_limit_action;
@@ -1013,7 +1102,7 @@ void
 NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec practice spec";
-    parseAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>(
+    parseAppsecJSONKey<NewOpenApiSchema>(
         "schemaValidation",
         openapi_schema_validation,
         archive_in
@@ -1021,11 +1110,15 @@ NewAppSecPracticeSpec::load(cereal::JSONInputArchive &archive_in)
     parseAppsecJSONKey<string>("appsecClassName", appsec_class_name, archive_in);
     parseMandatoryAppsecJSONKey<NewFileSecurity>("fileSecurity", file_security, archive_in);
     parseMandatoryAppsecJSONKey<NewIntrusionPrevention>("intrusionPrevention", intrusion_prevention, archive_in);
-    parseMandatoryAppsecJSONKey<NewSnortSignaturesAndOpenSchemaAPI>("snortSignatures", snort_signatures, archive_in);
+    parseMandatoryAppsecJSONKey<NewSnortSignatures>("snortSignatures", snort_signatures, archive_in);
     parseMandatoryAppsecJSONKey<NewAppSecPracticeWebAttacks>("webAttacks", web_attacks, archive_in);
     parseAppsecJSONKey<NewAppSecPracticeAntiBot>("antiBot", anti_bot, archive_in);
     parseAppsecJSONKey<string>("name", practice_name, archive_in);
     parseAppsecJSONKey<string>("practiceMode", mode, archive_in, "inherited");
+    if (valid_modes.count(mode) == 0) {
+        dbgWarning(D_LOCAL_POLICY) << "AppSec Threat prevention practice mode invalid: " << mode;
+        throw PolicyGenException("AppSec Threat prevention practice mode invalid: " + mode);
+    }
 }
 
 void
@@ -1034,13 +1127,13 @@ NewAppSecPracticeSpec::setName(const string &_name)
     practice_name = _name;
 }
 
-const NewSnortSignaturesAndOpenSchemaAPI &
-NewAppSecPracticeSpec::getOpenSchemaValidation() const
+NewOpenApiSchema &
+NewAppSecPracticeSpec::getOpenSchemaValidation()
 {
     return openapi_schema_validation;
 }
 
-NewSnortSignaturesAndOpenSchemaAPI &
+NewSnortSignatures &
 NewAppSecPracticeSpec::getSnortSignatures()
 {
     return snort_signatures;

components/security_apps/local_policy_mgmt_gen/policy_maker_utils.cc

@@ -21,6 +21,15 @@
 using namespace std;
 
 USE_DEBUG_FLAG(D_NGINX_POLICY);
+USE_DEBUG_FLAG(D_LOCAL_POLICY);
+
+static const std::unordered_map<std::string, std::string> key_to_source_identefier_val = {
+    { "sourceip", "Source IP"},
+    { "cookie", "Cookie:"},
+    { "headerkey", "Header:"},
+    { "JWTKey", ""},
+    { "x-forwarded-for", "X-Forwarded-For"}
+};
 
 void
 SecurityAppsWrapper::save(cereal::JSONOutputArchive &out_ar) const
@@ -147,7 +156,7 @@ string
 PolicyMakerUtils::dumpPolicyToFile(
     const PolicyWrapper &policy,
     const string &policy_path,
-    const string &settings_path)
+    const string &)
 {
     clearElementsMaps();
 
@@ -170,6 +179,7 @@ PolicyMakerUtils::dumpPolicyToFile(
         cereal::JSONOutputArchive ar(settings_ss);
         policy.getSettings().save(ar);
     }
+#if 0
     string settings_str = settings_ss.str();
     try {
         ofstream settings_file(settings_path);
@@ -179,10 +189,38 @@ PolicyMakerUtils::dumpPolicyToFile(
         dbgDebug(D_NGINX_POLICY) << "Error while writing settings to " << settings_path << ", Error: " << e.what();
     }
     dbgDebug(D_LOCAL_POLICY) << settings_path << " content: " << settings_str;
+#endif
 
     return policy_str;
 }
 
+template<class R>
+vector<string>
+extractExceptionAnnotationNames(
+    const R &parsed_rule,
+    const R &default_rule,
+    const string &policy_name)
+{
+    vector<string> annotation_names;
+
+    const R &rule = (!parsed_rule.getExceptions().empty() ? parsed_rule : default_rule);
+    for (const string &exception_name : rule.getExceptions()) {
+        if (exception_name.empty()) {
+            continue;
+        }
+
+        const auto policy_exception = policy_name + "/" + exception_name;
+
+        dbgTrace(D_NGINX_POLICY) << "Adding " << policy_exception << " to exception vector";
+
+        annotation_names.push_back(policy_exception);
+    }
+
+    dbgTrace(D_NGINX_POLICY) << "Number of exceptions related to rule: " << annotation_names.size();
+
+    return annotation_names;
+}
+
 template<class R>
 map<AnnotationTypes, string>
 extractAnnotationsNames(
@@ -215,18 +253,6 @@ extractAnnotationsNames(
         rule_annotation[AnnotationTypes::TRIGGER] = policy_name + "/" + trigger_annotation_name;
     }
 
-    string exception_annotation_name;
-    // TBD: support multiple exceptions
-    if (!parsed_rule.getExceptions().empty() && !parsed_rule.getExceptions()[0].empty()) {
-        exception_annotation_name = parsed_rule.getExceptions()[0];
-    } else if (!default_rule.getExceptions().empty() && !default_rule.getExceptions()[0].empty()) {
-        exception_annotation_name = default_rule.getExceptions()[0];
-    }
-
-    if (!exception_annotation_name.empty()) {
-        rule_annotation[AnnotationTypes::EXCEPTION] = policy_name + "/" + exception_annotation_name;
-    }
-
     string web_user_res_annotation_name =
         parsed_rule.getCustomResponse().empty() ?
         default_rule.getCustomResponse() :
@@ -442,6 +468,7 @@ template<class T, class R>
 R
 getAppsecExceptionSpec(const string &exception_annotation_name, const T &policy)
 {
+    dbgFlow(D_NGINX_POLICY) << "anotation name: " << exception_annotation_name;
     auto exceptions_vec = policy.getAppsecExceptions();
     auto exception_it = extractElement(exceptions_vec.begin(), exceptions_vec.end(), exception_annotation_name);
 
@@ -536,7 +563,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp
     bool webHeaders = trigger_spec.getAppsecTriggerExtendedLogging().isHttpHeaders();
     bool webBody = trigger_spec.getAppsecTriggerExtendedLogging().isRequestBody();
     bool logToCloud = trigger_spec.getAppsecTriggerLogDestination().getCloud();
-    bool logToK8sService = trigger_spec.getAppsecTriggerLogDestination().isK8SNeeded();
+    bool logTolocalTuning = trigger_spec.getAppsecTriggerLogDestination().isContainerNeeded();
     bool logToAgent = trigger_spec.getAppsecTriggerLogDestination().isAgentLocal();
     bool beautify_logs = trigger_spec.getAppsecTriggerLogDestination().shouldBeautifyLogs();
     bool logToCef = trigger_spec.getAppsecTriggerLogDestination().isCefNeeded();
@@ -563,7 +590,7 @@ extractLogTriggerData(const string &trigger_annotation_name, const T &trigger_sp
         logToAgent,
         logToCef,
         logToCloud,
-        logToK8sService,
+        logTolocalTuning,
         logToSyslog,
         responseBody,
         tpDetect,
@@ -774,6 +801,7 @@ createExceptionSection(
     const string &exception_annotation_name,
     const T &policy)
 {
+    dbgFlow(D_NGINX_POLICY) << "exception annotation name" << exception_annotation_name;
     AppsecException exception_spec =
         getAppsecExceptionSpec<T, AppsecException>(exception_annotation_name, policy);
     vector<InnerException> res;
@@ -782,6 +810,7 @@ createExceptionSection(
         ExceptionBehavior exception_behavior(exception.getAction());
         res.push_back(InnerException(exception_behavior, exception_match));
     }
+
     return res;
 }
 
@@ -894,13 +923,16 @@ createMultiRulesSections(
     const string &web_user_res_vec_id,
     const string &web_user_res_vec_type,
     const string &asset_name,
-    const string &exception_name,
-    const vector<InnerException> &exceptions)
+    const std::map<std::string, std::vector<InnerException>> &exceptions)
 {
     PracticeSection practice = PracticeSection(practice_id, practice_type, practice_name);
     vector<ParametersSection> exceptions_result;
     for (auto exception : exceptions) {
-        exceptions_result.push_back(ParametersSection(exception.getBehaviorId(), exception_name));
+
+        const auto &exception_name = exception.first;
+        for (const auto &inner_exception : exception.second) {
+            exceptions_result.push_back(ParametersSection(inner_exception.getBehaviorId(), exception_name));
+        }
     }
 
     vector<RulesTriggerSection> triggers;
@@ -1014,7 +1046,7 @@ PolicyMakerUtils::createIpsSections(
         practice_name,
         practice_id,
         source_identifier,
-        override_mode,
+        "Inactive",
         apssec_practice.getIntrusionPrevention().createIpsRules(override_mode)
     );
 
@@ -1024,8 +1056,7 @@ PolicyMakerUtils::createIpsSections(
 void
 PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_temporary)
 {
-    auto path = getFilesystemPathConfig() + "/conf/snort/" + file_name;
-    string in_file = is_temporary ? path + ".rule" : path;
+    auto path = is_temporary ? getFilesystemPathConfig() + "/conf/snort/" + file_name + ".rule" : file_name;
 
     if (snort_protections.find(path) != snort_protections.end()) {
         dbgTrace(D_LOCAL_POLICY) << "Snort protections section for file " << file_name << " already exists";
@@ -1036,7 +1067,9 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         << (is_temporary ? " temporary" : "") << " file " << path;
 
     auto snort_script_path = getFilesystemPathConfig() + "/scripts/snort_to_ips_local.py";
-    auto cmd = "python3 " + snort_script_path + " " + in_file + " " + path + ".out " + path + ".err";
+    auto tmp_out = "/tmp/" + file_name + ".out";
+    auto tmp_err = "/tmp/" + file_name + ".err";
+    auto cmd = "python3 " + snort_script_path + " " + path + " " + tmp_out + " " + tmp_err;
 
     auto res = Singleton::Consume<I_ShellCmd>::by<LocalPolicyMgmtGenerator>()->getExecOutput(cmd);
 
@@ -1045,16 +1078,16 @@ PolicyMakerUtils::createSnortProtecionsSection(const string &file_name, bool is_
         return;
     }
 
-    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(path + ".out");
+    Maybe<ProtectionsSectionWrapper> maybe_protections = openFileAsJson<ProtectionsSectionWrapper>(tmp_out);
     if (!maybe_protections.ok()){
         dbgWarning(D_LOCAL_POLICY) << maybe_protections.getErr();
         return;
     }
 
     auto i_orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<LocalPolicyMgmtGenerator>();
-    if (is_temporary) i_orchestration_tools->removeFile(in_file);
-    i_orchestration_tools->removeFile(path + ".out");
-    i_orchestration_tools->removeFile(path + ".err");
+    if (is_temporary) i_orchestration_tools->removeFile(path);
+    i_orchestration_tools->removeFile(tmp_out);
+    i_orchestration_tools->removeFile(tmp_err);
 
     snort_protections[path] = ProtectionsSection(
         maybe_protections.unpack().getProtections(),
@@ -1184,7 +1217,8 @@ void
 PolicyMakerUtils::createWebAppSection(
     const V1beta2AppsecLinuxPolicy &policy,
     const RulesConfigRulebase& rule_config,
-    const string &practice_id, const string &full_url,
+    const string &practice_id,
+    const string &full_url,
     const string &default_mode,
     map<AnnotationTypes, string> &rule_annotations)
 {
@@ -1201,6 +1235,7 @@ PolicyMakerUtils::createWebAppSection(
         apssec_practice.getWebAttacks().getMaxObjectDepth(),
         apssec_practice.getWebAttacks().getMaxUrlSizeBytes()
     );
+
     WebAppSection web_app = WebAppSection(
         full_url == "Any" ? default_appsec_url : full_url,
         rule_config.getAssetId(),
@@ -1212,6 +1247,10 @@ PolicyMakerUtils::createWebAppSection(
         rule_config.getContext(),
         apssec_practice.getWebAttacks().getMinimumConfidence(practice_mode),
         apssec_practice.getWebAttacks().getMode(practice_mode),
+        apssec_practice.getAntiBot().getMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getOverrideMode(practice_mode),
+        apssec_practice.getOpenSchemaValidation().getEnforceLevel(),
+        apssec_practice.getOpenSchemaValidation().getOas(),
         practice_advance_config,
         apssec_practice.getAntiBot(),
         log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
@@ -1265,7 +1304,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
     );
     rules_config[rule_config.getAssetName()] = rule_config;
 
-    string current_identifier;
+    string current_identifier, current_identifier_value;
     if (!rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS].empty()) {
         UsersIdentifiersRulebase user_identifiers = createUserIdentifiers<V1beta2AppsecLinuxPolicy>(
             rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS],
@@ -1274,6 +1313,15 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         );
         users_identifiers[rule_annotations[AnnotationTypes::SOURCE_IDENTIFIERS]] = user_identifiers;
         current_identifier = user_identifiers.getIdentifier();
+        current_identifier_value = user_identifiers.getIdentifierValue();
+    }
+
+    string ips_identifier, ips_identifier_value;
+    if(key_to_source_identefier_val.find(current_identifier) != key_to_source_identefier_val.end()) {
+        ips_identifier = key_to_source_identefier_val.at(current_identifier);
+    }
+    if (current_identifier == "cookie" || current_identifier == "headerkey") {
+        ips_identifier_value = current_identifier_value;
     }
 
     createIpsSections(
@@ -1281,7 +1329,7 @@ PolicyMakerUtils::createThreatPreventionPracticeSections(
         rule_config.getAssetName(),
         practice_id,
         rule_annotations[AnnotationTypes::PRACTICE],
-        current_identifier,
+        ips_identifier + ips_identifier_value,
         rule_config.getContext(),
         policy,
         rule_annotations,
@@ -1342,6 +1390,7 @@ PolicyMakerUtils::combineElementsToPolicy(const string &policy_version)
             convertMapToVector(log_triggers), convertMapToVector(web_user_res_triggers)
         )
     );
+
     ExceptionsWrapper exceptions_section({
         ExceptionsRulebase(convertExceptionsMapToVector(inner_exceptions))
     });
@@ -1379,6 +1428,7 @@ PolicyMakerUtils::createPolicyElementsByRule(
     const string &policy_name)
 {
     map<AnnotationTypes, string> rule_annotations = extractAnnotationsNames(rule, default_rule, policy_name);
+
     if (
         !rule_annotations[AnnotationTypes::TRIGGER].empty() &&
         !log_triggers.count(rule_annotations[AnnotationTypes::TRIGGER])
@@ -1401,15 +1451,27 @@ PolicyMakerUtils::createPolicyElementsByRule(
             );
     }
 
-    if (
-        !rule_annotations[AnnotationTypes::EXCEPTION].empty() &&
-        !inner_exceptions.count(rule_annotations[AnnotationTypes::EXCEPTION])
-    ) {
-        inner_exceptions[rule_annotations[AnnotationTypes::EXCEPTION]] =
-            createExceptionSection<T>(
-                rule_annotations[AnnotationTypes::EXCEPTION],
-                policy
-            );
+    const auto exceptions_annotations = extractExceptionAnnotationNames(rule, default_rule, policy_name);
+    std::map<std::string, std::vector<InnerException>> rule_inner_exceptions;
+    if (!exceptions_annotations.empty()) {
+        for (const auto &exception_name :exceptions_annotations) {
+            dbgWarning(D_LOCAL_POLICY) << "exceptions name: " << exception_name;
+
+            if (rule_inner_exceptions.count(exception_name)) {
+                dbgWarning(D_LOCAL_POLICY) << "exception name already exists for that rule: " << exception_name;
+                continue;
+            }
+
+            if (inner_exceptions.count(exception_name)) {
+                dbgWarning(D_LOCAL_POLICY) << "exception name already exists in inner exceptions: " << exception_name;
+                rule_inner_exceptions[exception_name] = inner_exceptions[exception_name];
+                continue;
+            }
+
+            auto exception_section = createExceptionSection<T>(exception_name, policy);
+            rule_inner_exceptions[exception_name] = exception_section;
+            inner_exceptions[exception_name] = exception_section;
+        }
     }
 
     if (
@@ -1468,8 +1530,7 @@ PolicyMakerUtils::createPolicyElementsByRule(
             web_user_res_triggers[rule_annotations[AnnotationTypes::WEB_USER_RES]].getTriggerId(),
             "WebUserResponse",
             full_url,
-            rule_annotations[AnnotationTypes::EXCEPTION],
-            inner_exceptions[rule_annotations[AnnotationTypes::EXCEPTION]]
+            rule_inner_exceptions
         );
         rules_config[rule_config.getAssetName()] = rule_config;
 
@@ -1496,7 +1557,7 @@ PolicyMakerUtils::createPolicyElementsByRule(
                 log_triggers[rule_annotations[AnnotationTypes::TRIGGER]],
                 rule.getMode(),
                 trusted_sources[rule_annotations[AnnotationTypes::TRUSTED_SOURCES]],
-                inner_exceptions[rule_annotations[AnnotationTypes::EXCEPTION]]
+                rule_inner_exceptions
             );
             web_apps[rule_config.getAssetName()] = web_app;
         }
@@ -1634,7 +1695,9 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy(const string &policy_name, c
     createPolicyElements<T, R>(specific_rules, default_rule, appsec_policy, policy_name);
 
     // add default rule to policy
-    createPolicyElementsByRule<T, R>(default_rule, default_rule, appsec_policy, policy_name);
+    if (Singleton::Consume<I_EnvDetails>::by<PolicyMakerUtils>()->getEnvType() != EnvType::K8S) {
+        createPolicyElementsByRule<T, R>(default_rule, default_rule, appsec_policy, policy_name);
+    }
 }
 
 // LCOV_EXCL_START Reason: no test exist
@@ -1657,11 +1720,13 @@ PolicyMakerUtils::createAgentPolicyFromAppsecPolicy<V1beta2AppsecLinuxPolicy, Ne
     );
 
     // add default rule to policy
-    createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsedRule>(
-        default_rule,
-        default_rule,
-        appsec_policy,
-        policy_name);
+    if (Singleton::Consume<I_EnvDetails>::by<PolicyMakerUtils>()->getEnvType() != EnvType::K8S) {
+        createPolicyElementsByRule<V1beta2AppsecLinuxPolicy, NewParsedRule>(
+            default_rule,
+            default_rule,
+            appsec_policy,
+            policy_name);
+    }
 }
 // LCOV_EXCL_STOP
 

components/security_apps/local_policy_mgmt_gen/rules_config_section.cc

@@ -17,6 +17,8 @@ using namespace std;
 
 USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
+static const string empty_string="";
+
 AssetUrlParser
 AssetUrlParser::parse(const string &uri)
 {
@@ -242,6 +244,13 @@ UsersIdentifier::getIdentifier() const
 {
     return source_identifier;
 }
+
+const string &
+UsersIdentifier::getIdentifierValue() const
+{
+    if (identifier_values.empty()) return empty_string;
+    return identifier_values[0];
+}
 // LCOV_EXCL_STOP
 
 void
@@ -272,6 +281,13 @@ UsersIdentifiersRulebase::getIdentifier() const
     if (source_identifiers.empty()) return source_identifier;
     return source_identifiers[0].getIdentifier();
 }
+
+const string &
+UsersIdentifiersRulebase::getIdentifierValue() const
+{
+    if (source_identifiers.empty()) return empty_string;
+    return source_identifiers[0].getIdentifierValue();
+}
 // LCOV_EXCL_STOP
 
 void

components/security_apps/local_policy_mgmt_gen/triggers_section.cc

@@ -30,7 +30,7 @@ LogTriggerSection::LogTriggerSection(
     bool _logToAgent,
     bool _logToCef,
     bool _logToCloud,
-    bool _logToK8sService,
+    bool _logTolocalTuning,
     bool _logToSyslog,
     bool _responseBody,
     bool _tpDetect,
@@ -55,7 +55,7 @@ LogTriggerSection::LogTriggerSection(
     logToAgent(_logToAgent),
     logToCef(_logToCef),
     logToCloud(_logToCloud),
-    logToK8sService(_logToK8sService),
+    logTolocalTuning(_logTolocalTuning),
     logToSyslog(_logToSyslog),
     responseBody(_responseBody),
     tpDetect(_tpDetect),
@@ -96,12 +96,12 @@ LogTriggerSection::save(cereal::JSONOutputArchive &out_ar) const
         cereal::make_nvp("acDrop",                   acDrop),
         cereal::make_nvp("complianceViolations",     false),
         cereal::make_nvp("complianceWarnings",       false),
-        cereal::make_nvp("extendloggingMinSeverity", extendloggingMinSeverity),
-        cereal::make_nvp("extendlogging",            extendlogging),
+        cereal::make_nvp("extendLoggingMinSeverity", extendloggingMinSeverity),
+        cereal::make_nvp("extendLogging",            extendlogging),
         cereal::make_nvp("logToAgent",               logToAgent),
         cereal::make_nvp("logToCef",                 logToCef),
         cereal::make_nvp("logToCloud",               logToCloud),
-        cereal::make_nvp("logToK8sService",          logToK8sService),
+        cereal::make_nvp("logTolocalTuning",    logTolocalTuning),
         cereal::make_nvp("logToSyslog",              logToSyslog),
         cereal::make_nvp("responseBody",             responseBody),
         cereal::make_nvp("responseCode",             false),
@@ -387,12 +387,22 @@ void
 AppsecTriggerLogDestination::load(cereal::JSONInputArchive &archive_in)
 {
     dbgTrace(D_LOCAL_POLICY) << "Loading AppSec Trigger LogDestination";
-    // TBD: support "file"
-    parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
-    auto mode = Singleton::Consume<I_AgentDetails>::by<AppsecTriggerLogDestination>()->getOrchestrationMode();
-    auto env_type = Singleton::Consume<I_EnvDetails>::by<AppsecTriggerLogDestination>()->getEnvType();
-    bool k8s_service_default = (mode == OrchestrationMode::HYBRID && env_type == EnvType::K8S);
-    parseAppsecJSONKey<bool>("k8s-service", k8s_service, archive_in, k8s_service_default);
+    if (getConfigurationFlag("orchestration-mode") != "hybrid_mode") {
+        // TBD: support "file"
+        parseAppsecJSONKey<bool>("cloud", cloud, archive_in, false);
+    } else {
+        cloud = false;
+    }
+    // check ENV VAR LOCAL_TUNING_ENABLED
+    char * tuning_enabled = getenv("LOCAL_TUNING_ENABLED");
+    if (tuning_enabled != NULL) {
+        for (unsigned int i = 0; i < strlen(tuning_enabled); i++) {
+            tuning_enabled[i] = tolower(tuning_enabled[i]);
+        }
+        container_service = string(tuning_enabled) == "true";
+    } else {
+        container_service = false;
+    }
 
     StdoutLogging stdout_log;
     parseAppsecJSONKey<StdoutLogging>("stdout", stdout_log, archive_in);
@@ -433,9 +443,9 @@ AppsecTriggerLogDestination::getCloud() const
 }
 
 bool
-AppsecTriggerLogDestination::isK8SNeeded() const
+AppsecTriggerLogDestination::isContainerNeeded() const
 {
-    return k8s_service;
+    return container_service;
 }
 
 bool

components/security_apps/orchestration/CMakeLists.txt

@@ -12,6 +12,9 @@ add_subdirectory(manifest_controller)
 add_subdirectory(update_communication)
 add_subdirectory(details_resolver)
 add_subdirectory(health_check)
+add_subdirectory(health_check_manager)
+add_subdirectory(updates_process_reporter)
 add_subdirectory(env_details)
+add_subdirectory(external_sdk_server)
 
 #add_subdirectory(orchestration_ut)

components/security_apps/orchestration/details_resolver/details_resolver.cc

@@ -44,6 +44,8 @@ public:
     bool isGwNotVsx() override;
     bool isVersionAboveR8110() override;
     bool isReverseProxy() override;
+    bool isCloudStorageEnabled() override;
+    Maybe<tuple<string, string, string, string, string>> readCloudMetadata() override;
     Maybe<tuple<string, string, string>> parseNginxMetadata() override;
 #if defined(gaia) || defined(smb)
     bool compareCheckpointVersion(int cp_version, std::function<bool(int, int)> compare_operator) const override;
@@ -135,9 +137,22 @@ DetailsResolver::Impl::isReverseProxy()
     return getenv("DOCKER_RPM_ENABLED") && getenv("DOCKER_RPM_ENABLED") == string("true");
 }
 
+bool
+DetailsResolver::Impl::isCloudStorageEnabled()
+{
+    auto cloud_storage_mode_override = getProfileAgentSetting<bool>("agent.cloudStorage.enabled");
+    if (cloud_storage_mode_override.ok()) {
+        dbgDebug(D_ORCHESTRATOR) << "Received cloud-storage mode override: " << *cloud_storage_mode_override;
+        return *cloud_storage_mode_override;
+    }
+
+    return getenv("CLOUD_STORAGE_ENABLED") && getenv("CLOUD_STORAGE_ENABLED") == string("true");
+}
+
 bool
 DetailsResolver::Impl::isKernelVersion3OrHigher()
 {
+#if defined(gaia) || defined(smb)
     static const string cmd =
         "clish -c 'show version os kernel' | awk '{print $4}' "
         "| cut -d '.' -f 1 | awk -F: '{ if ( $1 >= 3 ) {print 1} else {print 0}}'";
@@ -146,12 +161,14 @@ DetailsResolver::Impl::isKernelVersion3OrHigher()
     if (is_gogo.ok() && !is_gogo.unpack().empty()) {
         return is_gogo.unpack().front() == '1';
     }
+#endif
     return false;
 }
 
 bool
 DetailsResolver::Impl::isGwNotVsx()
 {
+#if defined(gaia) || defined(smb)
     static const string is_gw_cmd = "cpprod_util FwIsFirewallModule";
     static const string is_vsx_cmd = "cpprod_util FWisVSX";
     auto is_gw = DetailsResolvingHanlder::getCommandOutput(is_gw_cmd);
@@ -159,6 +176,7 @@ DetailsResolver::Impl::isGwNotVsx()
     if (is_gw.ok() && is_vsx.ok() && !is_gw.unpack().empty() && !is_vsx.unpack().empty()) {
         return is_gw.unpack().front() == '1' && is_vsx.unpack().front() == '0';
     }
+#endif
     return false;
 }
 
@@ -175,17 +193,16 @@ DetailsResolver::Impl::getCheckpointVersion() const
 {
 #ifdef gaia
     static const string cmd =
-        "echo $CPDIR | awk -F'-' '{print $NF}' | cut -c 2- |"
-        " awk -F'.' '{ if( NF == 1 ) {print $1\"00\"} else {print $1$2} }'";
+        "echo $CPDIR | awk '{sub(/.*-R/,\"\"); sub(/\\/.*/,\"\")}/^[0-9]*$/{$0=$0\".00\"}{sub(/\\./, \"\"); print}'";
 #else // smb
     static const string cmd = "sqlcmd 'select major,minor from cpver' |"
         "awk '{if ($1 == \"major\") v += (substr($3,2) * 100);"
         " if ($1 == \"minor\") v += $3; } END { print v}'";
-
 #endif // gaia
     auto version_out = DetailsResolvingHanlder::getCommandOutput(cmd);
     int cp_version = 0;
     if (version_out.ok()) {
+        dbgTrace(D_ORCHESTRATOR) << "Identified version " << version_out.unpack();
         stringstream version_stream(version_out.unpack());
         version_stream >> cp_version;
     }
@@ -287,6 +304,67 @@ DetailsResolver::Impl::parseNginxMetadata()
     return make_tuple(config_opt, cc_opt, nginx_version);
 }
 
+Maybe<tuple<string, string, string, string, string>>
+DetailsResolver::Impl::readCloudMetadata()
+{
+    auto env_read_cloud_metadata = []() -> Maybe<tuple<string, string, string, string, string>> {
+            string account_id = getenv("CLOUD_ACCOUNT_ID") ? getenv("CLOUD_ACCOUNT_ID") : "";
+            string vpc_id = getenv("CLOUD_VPC_ID") ? getenv("CLOUD_VPC_ID") : "";
+            string instance_id = getenv("CLOUD_INSTANCE_ID") ? getenv("CLOUD_INSTANCE_ID") : "";
+            string instance_local_ip = getenv("CLOUD_INSTANCE_LOCAL_IP") ? getenv("CLOUD_INSTANCE_LOCAL_IP") : "";
+            string region = getenv("CLOUD_REGION") ? getenv("CLOUD_REGION") : "";
+
+        if (
+            account_id.empty() ||
+            vpc_id.empty() ||
+            instance_id.empty() ||
+            instance_local_ip.empty() ||
+            region.empty()) {
+            return genError("Could not read cloud metadata");
+        }
+
+        return make_tuple(account_id, vpc_id, instance_id, instance_local_ip, region);
+    };
+
+    auto cloud_metadata = env_read_cloud_metadata();
+    if (!cloud_metadata.ok()) {
+        const string cmd = getFilesystemPathConfig() + "/scripts/get-cloud-metadata.sh";
+        dbgTrace(D_ORCHESTRATOR) << cloud_metadata.getErr() << ", trying to fetch it via cmd: " << cmd;
+
+        auto result = DetailsResolvingHanlder::getCommandOutput(cmd);
+        if (result.ok()) {
+            istringstream iss(result.unpack());
+            string line;
+            while (getline(iss, line)) {
+                size_t pos = line.find('=');
+                if (pos != string::npos) {
+                    string key = line.substr(0, pos);
+                    string value = line.substr(pos + 1);
+                    if (!key.empty() && !value.empty()) setenv(key.c_str(), value.c_str(), 1);
+                }
+            }
+            cloud_metadata = env_read_cloud_metadata();
+        } else {
+            dbgWarning(D_ORCHESTRATOR) << "Could not fetch cloud metadata from cmd: " << result.getErr();
+        }
+    }
+
+    if (!cloud_metadata.ok()) {
+        dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr();
+        return genError("Failed to fetch cloud metadata");
+    }
+
+    dbgTrace(D_ORCHESTRATOR)
+        << "Successfully fetched cloud metadata: "
+        << ::get<0>(cloud_metadata.unpack()) << ", "
+        << ::get<1>(cloud_metadata.unpack()) << ", "
+        << ::get<2>(cloud_metadata.unpack()) << ", "
+        << ::get<3>(cloud_metadata.unpack()) << ", "
+        << ::get<4>(cloud_metadata.unpack());
+
+    return cloud_metadata;
+}
+
 DetailsResolver::DetailsResolver() : Component("DetailsResolver"), pimpl(make_unique<Impl>()) {}
 
 DetailsResolver::~DetailsResolver() {}

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -15,21 +15,38 @@
 #define __CHECKPOINT_PRODUCT_HANDLERS_H__
 
 #include <algorithm>
+#include <regex>
 #include <boost/regex.hpp>
+#include <boost/algorithm/string.hpp>
+#include <cereal/external/rapidjson/document.h>
+#include <cereal/external/rapidjson/filereadstream.h>
 
 #if defined(gaia)
 
 Maybe<string>
 checkSAMLSupportedBlade(const string &command_output)
 {
-    string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
+    // uncomment when vpn will support SAML authentication
+    // string supportedBlades[3] = {"identityServer", "vpn", "cvpn"};
+    string supportedBlades[1] = {"identityServer"};
     for(const string &blade : supportedBlades) {
         if (command_output.find(blade) != string::npos) {
             return string("true");
         }
     }
 
-    return genError("Current host does not have SAML capability");
+    return string("false");
+}
+
+Maybe<string>
+checkIDABlade(const string &command_output)
+{
+    string idaBlade = "identityServer";
+    if (command_output.find(idaBlade) != string::npos) {
+        return string("true");
+    }
+
+    return string("false");
 }
 
 Maybe<string>
@@ -39,13 +56,22 @@ checkSAMLPortal(const string &command_output)
         return string("true");
     }
 
-    return genError("Current host does not have SAML Portal configured");
+    return string("false");
 }
 
 Maybe<string>
-getIDASSamlGaia(const string &command_output)
+checkPepIdaIdnStatus(const string &command_output)
 {
-    return string("idaSaml_gaia");
+    if (command_output.find("nac_pep_identity_next_enabled = 1") != string::npos) {
+        return string("true");
+    }
+    return string("false");
+}
+
+Maybe<string>
+getRequiredNanoServices(const string &command_output)
+{
+    return command_output;
 }
 
 Maybe<string>
@@ -61,7 +87,7 @@ checkIDP(shared_ptr<istream> file_stream)
         }
     }
 
-    return genError("Identity Provider was not found");
+    return string("false");
 }
 
 #endif // gaia
@@ -76,6 +102,33 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
     return command_output;
 }
 
+Maybe<string>
+getOtlpAgentGaiaOsRole(const string &command_output)
+{
+    if (command_output == "" ) return string("-1");
+
+    return command_output;
+}
+
+Maybe<string>
+getQUID(const string &command_output)
+{
+    if (command_output == "" ) return string("false");
+    //  validate valid QUID - contains exactly 4 '-'
+    if (std::count(command_output.begin(), command_output.end(), '-') != 4) {
+        return genError("not valid QUID");
+    }
+    return command_output;
+}
+
+Maybe<string>
+getIsAiopsRunning(const string &command_output)
+{
+    if (command_output == "" ) return string("false");
+    
+    return command_output;
+}
+
 Maybe<string>
 checkHasSDWan(const string &command_output)
 {
@@ -92,6 +145,26 @@ checkCanUpdateSDWanData(const string &command_output)
     return string("true");
 }
 
+Maybe<string>
+checkLsmProfileName(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    return genError("LSM profile name was not found");
+}
+
+Maybe<string>
+checkLsmProfileUuid(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    return genError("LSM profile uuid was not found");
+}
+
 Maybe<string>
 getMgmtObjType(const string &command_output)
 {
@@ -131,26 +204,67 @@ getMgmtObjAttr(shared_ptr<istream> file_stream, const string &attr)
 }
 
 Maybe<string>
-getMgmtObjUid(shared_ptr<istream> file_stream)
+getAttrFromCpsdwanGetDataJson(const string &attr)
 {
+    static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json";
+    std::ifstream ifs(get_data_json_path);
+    if (ifs.is_open()) {
+        rapidjson::IStreamWrapper isw(ifs);
+        rapidjson::Document document;
+        document.ParseStream(isw);
+
+        if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) {
+            return string(document[attr.c_str()].GetString());
+        }
+    }
+
+    return genError("Attribute " + attr + " was not found in " + get_data_json_path);
+}
+
+Maybe<string>
+getMgmtObjUid(const string &command_output)
+{
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    Maybe<string> obj_uuid = getAttrFromCpsdwanGetDataJson("uuid");
+    if (obj_uuid.ok()) {
+        return obj_uuid.unpack();
+    }
+
+    static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
+    auto file_stream = std::make_shared<std::ifstream>(obj_path);
+    if (!file_stream->is_open()) {
+        return genError("Failed to open the object file");
+    }
     return getMgmtObjAttr(file_stream, "uuid ");
 }
 
 Maybe<string>
-getMgmtObjName(shared_ptr<istream> file_stream)
+getMgmtObjName(const string &command_output)
 {
+    if (!command_output.empty()) {
+        return command_output;
+    }
+
+    static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
+    auto file_stream = std::make_shared<std::ifstream>(obj_path);
+    if (!file_stream->is_open()) {
+        return genError("Failed to open the object file");
+    }
     return getMgmtObjAttr(file_stream, "name ");
 }
 
 Maybe<string>
-getGWHardware(const string &command_output)
+getHardware(const string &command_output)
 {
     if (!command_output.empty()) {
         if (command_output == "software") return string("Open server");
         if (command_output == "Maestro Gateway") return string("Maestro");
         return string(command_output);
     }
-    return genError("GW Hardware was not found");
+    return genError("Hardware was not found");
 }
 
 Maybe<string>
@@ -236,7 +350,12 @@ getSmbObjectName(const string &command_output)
     if (command_output.empty() || command_output[0] != centrally_managed_comd_output) {
         return genError("Object name was not found");
     }
-    
+
+    Maybe<string> obj_name = getAttrFromCpsdwanGetDataJson("name");
+    if (obj_name.ok()) {
+        return obj_name.unpack();
+    }
+
     static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C";
     auto ifs = std::make_shared<std::ifstream>(obj_path);
     if (!ifs->is_open()) {
@@ -271,6 +390,34 @@ getSmbGWIPSecVPNBlade(const string &command_output)
 {
     return getSmbBlade(command_output, "IPSec VPN Blade was not found");
 }
+
+Maybe<string>
+extractManagements(const string &command_output)
+{
+    size_t start_pos = command_output.find(":masters(");
+    if (start_pos == string::npos) {
+        return genError("Starting pattern \":masters(\" not found.");
+    }
+    size_t end_pos = command_output.find("))):", start_pos);
+    if (end_pos == string::npos) {
+        return genError("Ending pattern \"))):\" not found.");
+    }
+    string input_string = command_output.substr(start_pos, end_pos - start_pos + 3);
+    string json_output = "[";
+    regex pattern("\\(ReferenceObject\\:Uid\\(\"\\{([\\w-]+)\\}\"\\)\\:Name\\(([^\\)]+)\\)\\:Table\\(([^\\)]+)\\)\\)");
+    smatch matches;
+    auto words_begin = sregex_iterator(input_string.begin(), input_string.end(), pattern);
+    auto words_end = sregex_iterator();
+    for (sregex_iterator i = words_begin; i != words_end; ++i) {
+        const smatch& match = *i;
+        string uid = boost::algorithm::to_lower_copy(match[1].str());
+        string name = match[2].str();
+        if (json_output.back() != '[') json_output += ",";
+        json_output += "{\"Uid\":\"" + uid + "\",\"Name\":\"" + name + "\"}";
+    }
+    json_output += "]";
+    return json_output;
+}
 #endif // gaia || smb
 
 #if defined(gaia)

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -42,10 +42,46 @@ SHELL_PRE_CMD("gunzip local.cfg", "gunzip -c $FWDIR/state/local/FW1/local.cfg.gz
 #ifdef SHELL_CMD_HANDLER
 #if defined(gaia) || defined(smb)
 SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
+SHELL_CMD_HANDLER(
+    "cpProductIntegrationMgmtObjectUid",
+    "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].uid'",
+    getMgmtObjUid
+)
 SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
-    "[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] "
-    "&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
+    "FS_PATH=<FILESYSTEM-PREFIX>; [ -f ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log ] "
+    "&& head -1 ${FS_PATH}/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
     checkIsInstallHorizonTelemetrySucceeded)
+SHELL_CMD_HANDLER("GLOBAL_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("QUID", "FS_PATH=<FILESYSTEM-PREFIX>;"
+    "VS_ID=$(echo \"${FS_PATH}\" | grep -o -E \"vs[0-9]+\" | grep -o -E \"[0-9]+\");"
+    "[ -z \"${VS_ID}\" ] && "
+    "(python3 /opt/CPquid/Quid_Api.py -i /opt/CPotelcol/quid_api/get_global_id.json | jq -r .message || echo '');"
+    "[ -n \"${VS_ID}\" ] && "
+    "(sed \"s|###VS_ID###|${VS_ID}|g\" /opt/CPotelcol/quid_api/get_vs_quid.json"
+    " > /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID}); "
+    "[ -n \"${VS_ID}\" ] && [ -f /opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} ] && "
+    "(python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_vs_quid.json.${VS_ID} | jq -r .message[0].QUID || echo '');",
+    getQUID)
+SHELL_CMD_HANDLER("SMO_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_smo_quid.json | jq -r .message[0].SMO_QUID || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] "
+    "&& python3 /opt/CPquid/Quid_Api.py -i "
+    "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''",
+    getQUID)
+SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] "
+    "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh",
+    getOtlpAgentGaiaOsRole)
+SHELL_CMD_HANDLER(
+    "IS_AIOPS_RUNNING",
+    "FS_PATH=<FILESYSTEM-PREFIX>; "
+    "PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); "
+    "[ -z \"{PID}\" ] && echo 'false' || echo 'true'",
+    getIsAiopsRunning)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(
     "canUpdateSDWanData",
@@ -56,6 +92,16 @@ SHELL_CMD_HANDLER(
     "isSdwanRunning",
     "[ -v $(pidof cp-nano-sdwan) ] && echo 'false' || echo 'true'",
     checkIfSdwanRunning)
+SHELL_CMD_HANDLER(
+    "lsmProfileName",
+    "jq -r .lsm_profile_name /tmp/cpsdwan_getdata_orch.json",
+    checkLsmProfileName
+)
+SHELL_CMD_HANDLER(
+    "lsmProfileUuid",
+    "jq -r .lsm_profile_uuid /tmp/cpsdwan_getdata_orch.json",
+    checkLsmProfileUuid
+)
 SHELL_CMD_HANDLER(
     "IP Address",
     "[ $(cpprod_util FWisDAG) -eq 1 ] && echo \"Dynamic Address\" "
@@ -84,8 +130,15 @@ SHELL_CMD_HANDLER(
 
 #if defined(gaia)
 SHELL_CMD_HANDLER("hasSAMLSupportedBlade", "enabled_blades", checkSAMLSupportedBlade)
-SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status saml-vpn", checkSAMLPortal)
-SHELL_CMD_HANDLER("requiredNanoServices", "ida_saml_gaia", getIDASSamlGaia)
+SHELL_CMD_HANDLER("hasIDABlade", "enabled_blades", checkIDABlade)
+SHELL_CMD_HANDLER("hasSAMLPortal", "mpclient status nac", checkSAMLPortal)
+SHELL_CMD_HANDLER("hasIdaIdnEnabled", "fw ctl get int nac_pep_identity_next_enabled", checkPepIdaIdnStatus)
+SHELL_CMD_HANDLER("requiredNanoServices", "echo 'idaSaml_gaia;idaIdn_gaia;'", getRequiredNanoServices)
+SHELL_CMD_HANDLER(
+    "cpProductIntegrationMgmtObjectName",
+    "mgmt_cli --format json -r true show-session | jq -r '.[\"connected-server\"].name'",
+    getMgmtObjName
+)
 SHELL_CMD_HANDLER(
     "cpProductIntegrationMgmtParentObjectName",
     "cat $FWDIR/database/myself_objects.C "
@@ -100,8 +153,8 @@ SHELL_CMD_HANDLER(
 )
 SHELL_CMD_HANDLER(
     "Hardware",
-    "cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:appliance_type/ {print $3}' | head -n 1",
-    getGWHardware
+    "cat $FWDIR/database/myself_objects.C | awk -F '[:()]' '/:appliance_type/ {print $3}' | head -n 1 | sed 's/\"//g'",
+    getHardware
 )
 SHELL_CMD_HANDLER(
     "Application Control",
@@ -134,6 +187,11 @@ SHELL_CMD_HANDLER(
     "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
     getSMCBasedMgmtName
 )
+SHELL_CMD_HANDLER(
+    "managements",
+    "echo 1",
+    extractManagements
+)
 #endif //gaia
 
 #if defined(smb)
@@ -184,10 +242,17 @@ SHELL_CMD_HANDLER(
     "| awk -F '[:()]' '/:masters/ {found=1; next} found && /:Name/ {print $3; exit}'",
     getSMCBasedMgmtName
 )
+
+SHELL_CMD_HANDLER(
+    "managements",
+    "echo 1",
+    extractManagements
+)
 #endif//smb
 
 SHELL_CMD_OUTPUT("kernel_version", "uname -r")
 SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null")
+SHELL_CMD_OUTPUT("report_timestamp", "date -u +\%s")
 #endif // SHELL_CMD_OUTPUT
 
 
@@ -196,17 +261,11 @@ SHELL_CMD_OUTPUT("helloWorld", "cat /tmp/agentHelloWorld 2>/dev/null")
 #ifdef FILE_CONTENT_HANDLER
 
 #if defined(gaia)
-
 FILE_CONTENT_HANDLER(
     "hasIdpConfigured",
     (getenv("SAMLPORTAL_HOME") ? string(getenv("SAMLPORTAL_HOME")) : "") + "/phpincs/spPortal/idpPolicy.xml",
     checkIDP
 )
-FILE_CONTENT_HANDLER(
-    "cpProductIntegrationMgmtObjectName",
-    (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
-    getMgmtObjName
-)
 #endif //gaia
 
 #if defined(alpine)
@@ -214,11 +273,6 @@ FILE_CONTENT_HANDLER("alpine_tag", "/usr/share/build/cp-alpine-tag", getCPAlpine
 #endif // alpine
 #if defined(gaia) || defined(smb)
 FILE_CONTENT_HANDLER("os_release", "/etc/cp-release", getOsRelease)
-FILE_CONTENT_HANDLER(
-    "cpProductIntegrationMgmtObjectUid",
-    (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C",
-    getMgmtObjUid
-)
 #else // !(gaia || smb)
 FILE_CONTENT_HANDLER("os_release", "/etc/os-release", getOsRelease)
 #endif // gaia || smb

components/security_apps/orchestration/details_resolver/details_resolving_handler.cc

@@ -77,20 +77,29 @@ void
 DetailsResolvingHanlder::Impl::init()
 {
     string actual_filesystem_prefix = getFilesystemPathConfig();
+    size_t place_holder_size = filesystem_place_holder.size();
 
     for (auto &file_handler : file_content_handlers) {
         string &path = file_handler.second.first;
-        size_t place_holder_size = filesystem_place_holder.size();
         if (path.substr(0, place_holder_size) == filesystem_place_holder) {
             path = actual_filesystem_prefix + path.substr(place_holder_size);
         }
     }
+
+    for (auto &cmd_handler_pair : shell_command_handlers) {
+        string &cmd_str = cmd_handler_pair.second.first;
+        size_t fs_pos = cmd_str.find(filesystem_place_holder);
+        if (fs_pos != string::npos) {
+            cmd_str.replace(fs_pos, place_holder_size, actual_filesystem_prefix);
+        }
+    }
 }
 
 map<string, string>
 DetailsResolvingHanlder::Impl::getResolvedDetails() const
 {
     I_ShellCmd *shell = Singleton::Consume<I_ShellCmd>::by<DetailsResolvingHanlder>();
+    I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
     uint32_t timeout = getConfigurationWithDefault<uint32_t>(5000, "orchestration", "Details resolver time out");
 
     for (auto &shell_pre_command : shell_pre_commands) {
@@ -114,7 +123,15 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
         Maybe<string> shell_command_output = getCommandOutput(command);
         if (!shell_command_output.ok()) continue;
         Maybe<string> handler_ret = handler(*shell_command_output);
-        if (handler_ret.ok()) resolved_details[attr] = *handler_ret;
+
+        if (handler_ret.ok()) {
+            resolved_details[attr] = *handler_ret;
+        } else {
+            if (reporter->isPersistantAttr(attr)) {
+                dbgTrace(D_AGENT_DETAILS)<< "Persistent attribute changed, removing old value";
+                reporter->deleteAttr(attr);
+            }
+        }
     }
 
     for (auto file_handler : file_content_handlers) {
@@ -125,7 +142,7 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
         shared_ptr<ifstream> in_file =
             Singleton::Consume<I_OrchestrationTools>::by<DetailsResolvingHanlder>()->fileStreamWrapper(path);
         if (!in_file->is_open()) {
-            dbgWarning(D_AGENT_DETAILS) << "Could not open file for processing. Path: " << path;
+            dbgDebug(D_AGENT_DETAILS) << "Could not open file for processing. Path: " << path;
             continue;
         }
 
@@ -149,7 +166,6 @@ DetailsResolvingHanlder::Impl::getResolvedDetails() const
         }
     }
 
-    I_AgentDetailsReporter *reporter = Singleton::Consume<I_AgentDetailsReporter>::by<DetailsResolvingHanlder>();
     reporter->addAttr(resolved_details, true);
 
     return resolved_details;

components/security_apps/orchestration/downloader/CMakeLists.txt

@@ -1,5 +1,5 @@
 ADD_DEFINITIONS(-Wno-deprecated-declarations -Dalpine)
 
-add_library(orchestration_downloader curl_client.cc downloader.cc http_client.cc https_client.cc https_client_helper.cc)
+add_library(orchestration_downloader curl_client.cc downloader.cc https_client.cc https_client_helper.cc)
 
 #add_subdirectory(downloader_ut)

components/security_apps/orchestration/downloader/downloader.cc

@@ -15,7 +15,7 @@
 
 #include "i_orchestration_tools.h"
 #include "singleton.h"
-#include "http_client.h"
+#include "https_client.h"
 #include "debug.h"
 #include "config.h"
 #include "rest.h"
@@ -63,7 +63,7 @@ class Downloader::Impl : Singleton::Provide<I_Downloader>::From<Downloader>
 public:
     void init();
 
-    Maybe<string> downloadFileFromFog(
+    Maybe<string> downloadFile(
         const string &checksum,
         Package::ChecksumTypes checksum_type,
         const GetResourceFile &resourse_file
@@ -87,7 +87,7 @@ public:
     string getProfileFromMap(const string &tenant_id) const override;
 
 private:
-    Maybe<string> downloadFileFromFogByHTTP(
+    Maybe<string> downloadAttributeFile(
         const GetResourceFile &resourse_file,
         const string &file_name
     ) const;
@@ -130,12 +130,12 @@ Downloader::Impl::init()
 }
 
 Maybe<string>
-Downloader::Impl::downloadFileFromFog(
+Downloader::Impl::downloadFile(
     const string &checksum,
     Package::ChecksumTypes checksum_type,
     const GetResourceFile &resourse_file) const
 {
-    auto file_path = downloadFileFromFogByHTTP(resourse_file, resourse_file.getFileName() + ".download");
+    auto file_path = downloadAttributeFile(resourse_file, resourse_file.getFileName() + ".download");
 
     if (!file_path.ok()) {
         return file_path;
@@ -378,11 +378,11 @@ Downloader::Impl::validateChecksum(
 }
 
 Maybe<string>
-Downloader::Impl::downloadFileFromFogByHTTP(const GetResourceFile &resourse_file, const string &file_name) const
+Downloader::Impl::downloadAttributeFile(const GetResourceFile &resourse_file, const string &file_name) const
 {
     string file_path = dir_path + "/" + file_name;
 
-    dbgInfo(D_ORCHESTRATOR) << "Downloading file from fog. File: " << resourse_file.getFileName();
+    dbgInfo(D_ORCHESTRATOR) << "Downloading file. File: " << resourse_file.getFileName();
 
     I_UpdateCommunication *update_communication = Singleton::Consume<I_UpdateCommunication>::by<Downloader>();
     auto downloaded_file = update_communication->downloadAttributeFile(resourse_file, file_path);
@@ -408,17 +408,14 @@ Downloader::Impl::getFileFromLocal(const string &local_file_path, const string &
 Maybe<string>
 Downloader::Impl::getFileFromURL(const URLParser &url, const string &file_path, bool auth_required) const
 {
-    ofstream outFile(file_path, ofstream::out | ofstream::binary);
-    HTTPClient http_client;
     dbgInfo(D_ORCHESTRATOR) << "Downloading file. URL: " << url;
-    auto get_file_response = http_client.getFile(url, outFile, auth_required);
+    auto get_file_response = HTTPSClient().getFile(url, file_path, auth_required);
     if (!get_file_response.ok()) {
         Maybe<string> error = genError("Failed to download file from " + url.getBaseURL().unpack() +
                     ". Error: " + get_file_response.getErr());
         dbgWarning(D_ORCHESTRATOR) << "Download failed";
         return error;
     }
-    outFile.close();
     dbgInfo(D_ORCHESTRATOR) << "Download completed. URL: " << url;
     return file_path;
 }

components/security_apps/orchestration/downloader/downloader_ut/downloader_ut.cc

@@ -1,3 +1,7 @@
+#include <sstream>
+class Package;
+static std::ostream & operator<<(std::ostream &os, const Package &) { return os; }
+
 #include "cptest.h"
 #include "config.h"
 #include "config_component.h"
@@ -38,7 +42,7 @@ TEST_F(DownloaderTest, doNothing)
 {
 }
 
-TEST_F(DownloaderTest, downloadFileFromFog)
+TEST_F(DownloaderTest, downloadFile)
 {
     string fog_response = "bla bla";
     string checksum = "123";
@@ -55,7 +59,7 @@ TEST_F(DownloaderTest, downloadFileFromFog)
 
     EXPECT_CALL(mock_orchestration_tools, isNonEmptyFile("/tmp/virtualSettings.download")).WillOnce(Return(true));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file
@@ -76,7 +80,7 @@ TEST_F(DownloaderTest, downloadFileFromFogFailure)
         downloadAttributeFile(resourse_file, "/tmp/settings.download")
     ).WillOnce(Return(fog_response));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file
@@ -241,7 +245,7 @@ TEST_F(DownloaderTest, downloadEmptyFileFromFog)
         calculateChecksum(Package::ChecksumTypes::SHA256, "/tmp/manifest.download")
     ).WillOnce(Return(checksum));
 
-    Maybe<string> downloaded_file = i_downloader->downloadFileFromFog(
+    Maybe<string> downloaded_file = i_downloader->downloadFile(
         checksum,
         Package::ChecksumTypes::SHA256,
         resourse_file

components/security_apps/orchestration/downloader/http_client.cc

@@ -1,268 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#include "http_client.h"
-
-#include "curl_client.h"
-#include "downloader.h"
-#include "debug.h"
-#include "i_encryptor.h"
-#include "url_parser.h"
-#include "config.h"
-#include "i_environment.h"
-#include "orchestration_comp.h"
-
-#include <fstream>
-#include <string>
-#include <iostream>
-#include <chrono>
-#include <boost/asio/ip/tcp.hpp>
-
-using boost::asio::ip::tcp;
-using namespace std;
-
-USE_DEBUG_FLAG(D_ORCHESTRATOR);
-USE_DEBUG_FLAG(D_HTTP_REQUEST);
-
-// LCOV_EXCL_START Reason: Depends on real download server.
-class ClientConnection
-{
-public:
-    ClientConnection(
-        const URLParser &_url,
-        const Maybe<string> &_proxy_url,
-        const Maybe<uint16_t> &_proxy_port,
-        const Maybe<string> &_proxy_auth,
-        const string &_token)
-            :
-        url(_url),
-        proxy_url(_proxy_url),
-        proxy_port(_proxy_port),
-        proxy_auth(_proxy_auth),
-        token(_token)
-    {
-    }
-
-    Maybe<void>
-    handleConnect()
-    {
-        if (!url.getBaseURL().ok()) {
-            return genError("Failed to handle connection. Error: " + url.getBaseURL().getErr());
-        }
-        string server_name = url.getBaseURL().unpack();
-        string port = url.getPort();
-        string query = url.getQuery();
-        string host = server_name;
-        try {
-            if (stoi(port) != 80) {
-                host = host + ":" + port;
-            }
-        } catch (const exception &err) {
-            return genError("Failed to parse port to a number. Port: " + port );
-        }
-
-        chrono::duration<unsigned int, ratio<1>> sleep_time(60);
-        io_stream.expires_from_now(sleep_time);
-
-        if (proxy_url.ok()) {
-            if (!proxy_port.ok()) {
-                return genError(
-                    "Failed to handle connection to server. proxy domain is defined with invalid port, Error: " +
-                    proxy_port.getErr()
-                );
-            }
-            io_stream.connect(proxy_url.unpack(), to_string(proxy_port.unpack()));
-        } else {
-            io_stream.connect(server_name, port);
-        }
-
-        if (!io_stream) {
-            return genError("Failed to handle connection to server. Error: " + io_stream.error().message());
-        }
-
-        string request_url = query;
-        if (proxy_url.ok()) {
-            request_url = host + query;
-        }
-
-        stringstream http_request;
-        http_request << "GET http://" << request_url << " HTTP/1.1\r\n";
-        http_request << "Host: " << host << "\r\n";
-        if (!token.empty()) {
-            http_request << "Authorization: " << "Bearer " << token << "\r\n";
-        }
-        http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n";
-
-        auto i_trace_env = Singleton::Consume<I_Environment>::by<OrchestrationComp>();
-        http_request << i_trace_env->getCurrentHeaders();
-        http_request << "Accept: */*\r\n";
-
-        if (proxy_url.ok()) {
-            http_request << "Accept-Encoding: identity";
-            http_request << "Connection: close\r\n";
-            http_request << "Proxy-Connection: Keep-Alive\r\n";
-
-            if (proxy_auth.ok()) {
-                I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<Downloader>();
-                http_request << "Proxy-Authorization: Basic " + encryptor->base64Encode(proxy_auth.unpack()) + "\r\n";
-            }
-            http_request << "\r\n";
-        } else {
-            http_request << "Connection: close\r\n\r\n";
-        }
-
-        dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str();
-        io_stream << http_request.str();
-        return Maybe<void>();
-    }
-
-    Maybe<void>
-    handleResponse(ofstream &out_file)
-    {
-        string response_http_version;
-        io_stream >> response_http_version;
-        unsigned int status_code;
-        io_stream >> status_code;
-        string status_message;
-        getline(io_stream, status_message);
-
-        if (!io_stream || response_http_version.substr(0, 5) != "HTTP/")  {
-            return genError("Invalid response");
-        }
-
-        if (status_code != 200) {
-            return genError("HTTP response returned with status code " + status_code);
-        }
-
-        string header;
-        vector<string> headers;
-        while (getline(io_stream, header) && header != "\r") {
-            headers.push_back(header);
-        }
-
-        out_file << io_stream.rdbuf();
-
-        dbgTrace(D_HTTP_REQUEST)
-            << "Received HTTP Response with the following data (downloaded file will not be printed):"
-            << endl
-            << response_http_version
-            << " "
-            << status_code
-            << " "
-            << status_message
-            << endl
-            << makeSeparatedStr(headers, "\n");
-
-
-        return Maybe<void>();
-    }
-
-private:
-    const URLParser url;
-    const Maybe<string> proxy_url;
-    const Maybe<uint16_t> proxy_port;
-    const Maybe<string> proxy_auth;
-    const string &token;
-    boost::asio::ip::tcp::iostream io_stream;
-};
-
-Maybe<void>
-HTTPClient::getFile(const URLParser &url, ofstream &out_file, bool auth_required)
-{
-    auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-    auto load_env_proxy = proxy_config->loadProxy();
-    if (!load_env_proxy.ok()) return load_env_proxy;
-
-    string token = "";
-    if (auth_required) {
-        token = Singleton::Consume<I_AgentDetails>::by<HTTPClient>()->getAccessToken();
-    }
-
-    if (url.isOverSSL()) {
-        if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
-        if (getFileSSL(url, out_file, token).ok()) return Maybe<void>();
-        //CURL fallback
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL).";
-        return curlGetFileOverSSL(url, out_file, token);
-    }
-    auto get_file_http_res = getFileHttp(url, out_file, token);
-    if (!get_file_http_res.ok())
-    {
-        //CURL fallback
-        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over HTTP. Trying via CURL (HTTP).";
-        return curlGetFileOverHttp(url, out_file, token);
-    }
-
-    return get_file_http_res;
-}
-
-Maybe<void>
-HTTPClient::curlGetFileOverHttp(const URLParser &url, ofstream &out_file, const string &token)
-{
-    try {
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-
-        HttpCurl http_curl_client(
-            url,
-            out_file,
-            token,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
-            proxy_config->getProxyPort(ProxyProtocol::HTTPS),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS));
-
-        http_curl_client.setCurlOpts();
-        bool connection_ok = http_curl_client.connect();
-        if (!connection_ok)
-        {
-            stringstream url_s;
-            url_s << url;
-            string err_msg = string("Failed to get file over HTTP. URL: ") + url_s.str();
-            return genError(err_msg);
-        }
-
-        // As this class is a temporal solution catch all exception types is enabled.
-    } catch (const exception &e) {
-        string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what());
-        return genError(err_msg);
-    }
-
-    return Maybe<void>();
-}
-
-Maybe<void>
-HTTPClient::getFileHttp(const URLParser &url, ofstream &out_file, const string &token)
-{
-    try {
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-        ClientConnection client_connection(
-            url,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTP),
-            proxy_config->getProxyPort(ProxyProtocol::HTTP),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTP),
-            token
-        );
-        auto handle_connect_res = client_connection.handleConnect();
-        if (!handle_connect_res.ok()) return handle_connect_res;
-
-        return client_connection.handleResponse(out_file);
-
-    // As this class is a temporal solution catch all exception types is enabled.
-    } catch (const exception &e) {
-        string err_msg = "Failed to get file over HTTP. Exception: " + string(e.what());
-        return genError(err_msg);
-    }
-
-    return Maybe<void>();
-}
-// LCOV_EXCL_STOP

components/security_apps/orchestration/downloader/http_client.h

@@ -1,44 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __HTTP_CLIENT_H__
-#define __HTTP_CLIENT_H__
-
-#include <string>
-#include "maybe_res.h"
-#include "url_parser.h"
-#include "i_agent_details.h"
-#include "i_proxy_configuration.h"
-
-// LCOV_EXCL_START Reason: Depends on real download server.
-class HTTPClient
-        :
-    public Singleton::Consume<I_AgentDetails>,
-    public Singleton::Consume<I_ProxyConfiguration>
-{
-public:
-    HTTPClient() = default;
-
-    Maybe<void> getFile(const URLParser &url, std::ofstream &out_file, bool auth_required);
-
-private:
-    std::string loadCAChainDir();
-    Maybe<void> getFileSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> getFileSSLDirect(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> getFileHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> curlGetFileOverHttp(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-    Maybe<void> curlGetFileOverSSL(const URLParser &url, std::ofstream &out_file, const std::string &_token);
-};
-// LCOV_EXCL_STOP
-
-#endif // __HTTP_CLIENT_H__

components/security_apps/orchestration/downloader/https_client.cc

@@ -11,498 +11,51 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "http_client.h"
-#include "curl_client.h"
-
-#include "debug.h"
-#include "i_agent_details.h"
-#include "i_encryptor.h"
-#include "downloader.h"
-#include "config.h"
-#include "boost/uuid/uuid.hpp"
-#include "boost/uuid/uuid_generators.hpp"
-#include <boost/asio/deadline_timer.hpp>
-#include "boost/uuid/uuid_io.hpp"
+#include "https_client.h"
 
+#include <fstream>
 #include <string>
 #include <iostream>
-#include <istream>
-#include <ostream>
-#include <fstream>
-#include <boost/asio.hpp>
-#include <boost/bind/bind.hpp>
-#include <boost/asio/ssl.hpp>
-#include <exception>
+#include <chrono>
+
+#include "config.h"
+#include "curl_client.h"
 
-using namespace boost::placeholders;
-using boost::asio::ip::tcp;
 using namespace std;
 
-USE_DEBUG_FLAG(D_COMMUNICATION);
-USE_DEBUG_FLAG(D_HTTP_REQUEST);
 USE_DEBUG_FLAG(D_ORCHESTRATOR);
+USE_DEBUG_FLAG(D_HTTP_REQUEST);
 
 // LCOV_EXCL_START Reason: Depends on real download server.
-class BadResponseFromServer : public exception
+Maybe<void>
+HTTPSClient::getFile(const URLParser &url, const string &out_file, bool auth_required)
 {
-public:
-    BadResponseFromServer() : message("Bad response returned from server") {}
-    BadResponseFromServer(const string &msg) : message(msg) {}
-    const char *
-    what() const throw()
-    {
-        return message.c_str();
+    auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<OrchestrationComp>();
+    auto load_env_proxy = proxy_config->loadProxy();
+    if (!load_env_proxy.ok()) return load_env_proxy;
+
+    string token = "";
+    if (auth_required) {
+        token = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>()->getAccessToken();
     }
 
-private:
-    string message;
-};
+    if (!url.isOverSSL()) return genError("URL is not over SSL.");
 
-class Client
-{
-public:
-    Client(
-        ofstream &out_file,
-        boost::asio::io_service &io_service,
-        boost::asio::ssl::context &context,
-        const URLParser &_url,
-        const Maybe<string> &_proxy_url,
-        const Maybe<uint16_t> &_proxy_port,
-        const Maybe<string> &_proxy_auth,
-        const string &_token)
-            :
-        out_file(out_file),
-        url(_url),
-        proxy_url(_proxy_url),
-        proxy_port(_proxy_port),
-        proxy_auth(_proxy_auth),
-        resolver_(io_service),
-        deadline(io_service),
-        socket_(io_service),
-        ssl_socket(socket_, context),
-        token(_token)
-    {
-    }
+    if (getFileSSLDirect(url, out_file, token).ok()) return Maybe<void>();
+    dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL directly. Trying indirectly.";
 
-    Maybe<void>
-    handleConnection()
-    {
-        ostream request_stream(&request_);
-        stringstream http_request;
-        http_request << "GET " << url.getQuery() << " HTTP/1.1\r\n";
-        string host = url.getHost();
-        string port = url.getPort();
-        int port_int;
-        try {
-            port_int = stoi(port);
-        } catch (const exception &err) {
-            dbgWarning(D_COMMUNICATION)
-                << "Failed to convert port number from string. Port: "
-                << port
-                << ", Error: "
-                << err.what();
-            return genError("Failed to parse port to a number. Port: " + port);
-        }
-        if (port_int != 443) {
-            host = host + ":" + port;
-        }
+    if (getFileSSL(url, out_file, token).ok()) return Maybe<void>();
+    dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Trying via CURL (SSL).";
 
-        http_request << "Host: " << host << "\r\n";
-
-        if (!token.empty()) {
-            http_request << "Authorization: " << "Bearer " << token << "\r\n";
-        }
-        http_request << "User-Agent: Infinity Next (a7030abf93a4c13)\r\n";
-        boost::uuids::uuid correlation_id;
-        try {
-            correlation_id = uuid_random_gen();
-        } catch (const boost::uuids::entropy_error &) {
-            dbgWarning(D_COMMUNICATION) << "Failed to generate random correlation id - entropy exception";
-        }
-        http_request << "X-Trace-Id: " + boost::uuids::to_string(correlation_id) + "\r\n";
-        http_request << "Accept: */*\r\n";
-        http_request << "Connection: close\r\n\r\n";
-
-        request_stream << http_request.str();
-
-        deadline.expires_from_now(boost::posix_time::minutes(5));
-        deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1));
-
-        if (proxy_url.ok()) {
-            if (!proxy_port.ok()) {
-                dbgWarning(D_COMMUNICATION)
-                    << "Failed to connect to proxy due to invalid port value, Error: "
-                    << proxy_port.getErr();
-
-                return genError(
-                    "Failed to handle connection to server. proxy port is invalid, Error: " +
-                    proxy_port.getErr()
-                );
-            }
-            if (port_int == 443) host = host + ":" + port;
-            ostream connect_request_stream(&connect_request);
-            stringstream proxy_request;
-            proxy_request << "CONNECT " << host << " HTTP/1.1\r\n";
-            proxy_request << "Host: " << host << "\r\n";
-            if (proxy_auth.ok()) {
-                I_Encryptor *encryptor = Singleton::Consume<I_Encryptor>::by<Downloader>();
-                proxy_request
-                    << "Proxy-Authorization: Basic "
-                    << encryptor->base64Encode(proxy_auth.unpack())
-                    << "\r\n";
-            }
-            proxy_request << "\r\n";
-
-            dbgTrace(D_HTTP_REQUEST) << "Connecting to proxy: " << endl << proxy_request.str();
-            connect_request_stream << proxy_request.str();
-
-            tcp::resolver::query query(proxy_url.unpack(), to_string(proxy_port.unpack()));
-            resolver_.async_resolve(
-                query,
-                boost::bind(
-                    &Client::overProxyResolver,
-                    this,
-                    boost::asio::placeholders::error,
-                    boost::asio::placeholders::iterator
-                )
-            );
-        } else {
-            tcp::resolver::query query(url.getBaseURL().unpack(), port);
-            resolver_.async_resolve(
-                query,
-                boost::bind(
-                    &Client::handleResolve,
-                    this,
-                    boost::asio::placeholders::error,
-                    boost::asio::placeholders::iterator
-                )
-            );
-        }
-
-        dbgTrace(D_HTTP_REQUEST) << "Sending the following HTTP Request: " << endl << http_request.str();
-        return Maybe<void>();
-    }
-
-private:
-    void
-    checkDeadline(const boost::system::error_code &err)
-    {
-        if (err) return;
-        if (deadline.expires_at() <= boost::asio::deadline_timer::traits_type::now()) {
-            boost::system::error_code ignored_ec = boost::asio::error::operation_aborted;
-            socket_.close(ignored_ec);
-            deadline.expires_at(boost::posix_time::pos_infin);
-            return;
-        }
-        deadline.async_wait(boost::bind(&Client::checkDeadline, this, _1));
-    }
-
-    void
-    overProxyResolver(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator)
-    {
-        if (!err) {
-            boost::asio::async_connect(socket_, endpoint_iterator,
-                boost::bind(&Client::overProxyHandleConnect, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_msg = "Failed to connect to proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleConnect(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_write(socket_, connect_request,
-                boost::bind(&Client::overProxyHandleWriteRequest, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_msg = "Failed to connect to proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleWriteRequest(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_read_until(
-                socket_,
-                response_,
-                "\r\n",
-                boost::bind(&Client::overProxyHandleReadStatusLine, this, boost::asio::placeholders::error)
-            );
-        } else {
-            string err_msg = "Failed to write over proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-    }
-
-    void
-    overProxyHandleReadStatusLine(const boost::system::error_code &err)
-    {
-        if (err) {
-            string err_msg = "Failed to read status line over proxy. Error: " + err.message();
-            throw BadResponseFromServer(err_msg);
-        }
-        // Check that response is OK.
-        istream response_stream(&response_);
-        string response_http_version;
-        response_stream >> response_http_version;
-        unsigned int status_code;
-        response_stream >> status_code;
-        string status_message;
-        getline(response_stream, status_message);
-        if (!response_stream || response_http_version.substr(0, 5) != "HTTP/") {
-            throw BadResponseFromServer("Invalid response");
-            return;
-        }
-
-        if (status_code != 200) {
-            string err_msg = "Response returned with status code " + status_code;
-            throw BadResponseFromServer(err_msg);
-        }
-
-        dbgTrace(D_HTTP_REQUEST)
-            << "Received HTTP Response over proxied connection with the following data:"
-            << endl
-            << response_http_version
-            << " "
-            << status_code
-            << " "
-            << status_message;
-
-        if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.message.ignoreSslValidation") == false) {
-            ssl_socket.set_verify_mode(boost::asio::ssl::verify_peer | boost::asio::ssl::verify_fail_if_no_peer_cert);
-            ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2));
-        } else {
-            dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation";
-        }
-
-        ssl_socket.async_handshake(
-            boost::asio::ssl::stream_base::client,
-            boost::bind(&Client::handleHandshake, this, boost::asio::placeholders::error)
-        );
-    }
-
-    void
-    handleResolve(const boost::system::error_code &err, tcp::resolver::iterator endpoint_iterator)
-    {
-        if (!err) {
-            boost::asio::async_connect(ssl_socket.lowest_layer(), endpoint_iterator,
-                boost::bind(&Client::handleConnect, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string message = "Failed to connect. Error: " + err.message();
-            throw BadResponseFromServer(message);
-        }
-    }
-
-    bool
-    verifyCertificate(bool preverified, boost::asio::ssl::verify_context &ctx)
-    {
-        if (!token.empty()) {
-            X509_STORE_CTX *cts = ctx.native_handle();
-
-            switch (X509_STORE_CTX_get_error(cts))
-            {
-                case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT:
-                    dbgWarning(D_ORCHESTRATOR) << "SSL verification error: X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT";
-                    break;
-                case X509_V_ERR_CERT_NOT_YET_VALID:
-                case X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD:
-                    dbgWarning(D_ORCHESTRATOR) << "SSL verification error: Certificate not yet valid";
-                    break;
-                case X509_V_ERR_CERT_HAS_EXPIRED:
-                case X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD:
-                    dbgWarning(D_ORCHESTRATOR) << "Certificate expired";
-                    break;
-                case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT:
-                case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN:
-                    dbgDebug(D_ORCHESTRATOR) << "Self signed certificate in chain";
-                    if (getConfigurationWithDefault(false, "orchestration", "Self signed certificates acceptable")) {
-                        preverified = true;
-                    }
-                    break;
-                default:
-                    if (!preverified) {
-                        dbgWarning(D_ORCHESTRATOR)
-                            << "Certificate verification error number: "
-                            << X509_STORE_CTX_get_error(cts);
-                    }
-                    break;
-            }
-            return preverified;
-        }
-        return true;
-    }
-
-    void
-    handleConnect(const boost::system::error_code &err)
-    {
-        if (!err) {
-            if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.message.ignoreSslValidation") == false) {
-                ssl_socket.set_verify_mode(
-                    boost::asio::ssl::verify_peer |
-                    boost::asio::ssl::verify_fail_if_no_peer_cert
-                );
-                ssl_socket.set_verify_callback(boost::bind(&Client::verifyCertificate, this, _1, _2));
-            } else {
-                dbgWarning(D_HTTP_REQUEST) << "Ignoring SSL validation";
-            }
-
-            ssl_socket.async_handshake(boost::asio::ssl::stream_base::client,
-                boost::bind(&Client::handleHandshake, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Failed to connect. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleHandshake(const boost::system::error_code &error)
-    {
-        if (!error) {
-            boost::asio::buffer_cast<const char*>(request_.data());
-
-            boost::asio::async_write(ssl_socket, request_,
-                boost::bind(&Client::handleWriteRequest, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Handshake failed. Error: " + error.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleWriteRequest(const boost::system::error_code &err)
-    {
-        if (!err) {
-            boost::asio::async_read_until(ssl_socket, resp, "\r\n",
-                boost::bind(&Client::handleReadStatusLine, this,
-                    boost::asio::placeholders::error));
-        } else {
-            string err_message = "Failed to handle write request. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadStatusLine(const boost::system::error_code &err)
-    {
-        if (!err) {
-            istream response_stream(&resp);
-            string http_version;
-            response_stream >> http_version;
-            unsigned int status_code;
-            response_stream >> status_code;
-            string status_message;
-            getline(response_stream, status_message);
-            dbgTrace(D_HTTP_REQUEST)
-                << "Received HTTP Response with the following data:"
-                << endl
-                << http_version
-                << " "
-                << status_code;
-
-            if (!response_stream || http_version.substr(0, 5) != "HTTP/") {
-                string err_message = "Invalid response";
-                throw BadResponseFromServer(err_message);
-            }
-            if (status_code != 200) {
-                string err_message = "HTTPS response returned with status code " + to_string(status_code)
-                    + ". URL: " + url.toString();
-                throw BadResponseFromServer(err_message);
-            }
-
-            boost::asio::async_read_until(ssl_socket, resp, "\r\n\r\n",
-                boost::bind(&Client::handleReadHeaders, this,
-                    boost::asio::placeholders::error));
-        } else {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response status. Error:" << err.message();
-            string err_message = "Failed to read status. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadHeaders(const boost::system::error_code &err)
-    {
-        if (!err) {
-            // Process the response headers.
-            istream response_stream(&resp);
-            string header;
-            vector<string> headers;
-            while (getline(response_stream, header) && header != "\r") {
-                headers.push_back(header);
-            }
-
-            dbgTrace(D_HTTP_REQUEST) << "Received Response headers:" << endl << makeSeparatedStr(headers, "\n");
-            // Write whatever content we already have to output.
-            if (resp.size() > 0)
-                out_file << &resp;
-
-            // Start reading remaining data until EOF.
-            boost::asio::async_read(ssl_socket, resp,
-                boost::asio::transfer_at_least(1),
-                boost::bind(&Client::handleReadContent, this,
-                    boost::asio::placeholders::error));
-        } else {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response headers. Error:" << err.message();
-            string err_message = "Failed to read headers. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        }
-    }
-
-    void
-    handleReadContent(const boost::system::error_code &err)
-    {
-        if (!err) {
-            // Write all of the data that has been read so far.
-            out_file << &resp;
-            // Continue reading remaining data until EOF.
-            boost::asio::async_read(
-                ssl_socket,
-                resp,
-                boost::asio::transfer_at_least(1),
-                boost::bind(&Client::handleReadContent, this, boost::asio::placeholders::error)
-            );
-        } else if (err != boost::asio::error::eof && err != boost::asio::ssl::error::stream_truncated) {
-            dbgWarning(D_COMMUNICATION) << "Failed to read response body. Error:" << err.message();
-            string err_message = "Failed to read content. Error: " + err.message();
-            throw BadResponseFromServer(err_message);
-        } else if (err == boost::asio::ssl::error::stream_truncated) {
-            dbgError(D_COMMUNICATION) << "Had SSL warning during reading response body stage. Error:" << err.message();
-            deadline.cancel();
-        } else {
-            deadline.cancel();
-        }
-    }
-
-    ofstream &out_file;
-    const URLParser &url;
-    const Maybe<string> proxy_url;
-    const Maybe<uint16_t> proxy_port;
-    const Maybe<string> proxy_auth;
-    tcp::resolver resolver_;
-    boost::asio::deadline_timer deadline;
-    boost::asio::ip::tcp::socket socket_;
-    boost::asio::ssl::stream<boost::asio::ip::tcp::socket&> ssl_socket;
-    boost::asio::streambuf request_;
-    boost::asio::streambuf connect_request;
-    boost::asio::streambuf response_;
-    boost::asio::streambuf resp;
-    const string &token;
-    boost::uuids::random_generator uuid_random_gen;
-};
+    //CURL fallback
+    return curlGetFileOverSSL(url, out_file, token);
+}
 
 string
-HTTPClient::loadCAChainDir()
+HTTPSClient::loadCAChainDir()
 {
     string ca_chain_dir;
-    auto agent_details = Singleton::Consume<I_AgentDetails>::by<Downloader>();
+    auto agent_details = Singleton::Consume<I_AgentDetails>::by<OrchestrationComp>();
     auto load_ca_chain_dir = agent_details->getOpenSSLDir();
     if (load_ca_chain_dir.ok()) {
         ca_chain_dir = load_ca_chain_dir.unpack();
@@ -511,69 +64,26 @@ HTTPClient::loadCAChainDir()
 }
 
 Maybe<void>
-HTTPClient::getFileSSL(const URLParser &url, ofstream &out_file, const string &token)
+HTTPSClient::getFileSSL(const URLParser &url, const string &out_file, const string &)
 {
-    try {
-        boost::asio::ssl::context ctx(boost::asio::ssl::context::sslv23);
-        if (!token.empty()) {
-            string cert_file_path = getConfigurationWithDefault<string>(
-                getFilesystemPathConfig() + "/certs/fog.pem",
-                "message",
-                "Certificate chain file path"
-            );
-            dbgTrace(D_ORCHESTRATOR) << "Http client, cert file path: " << cert_file_path;
-            auto trusted_ca_directory = getConfiguration<string>("message", "Trusted CA directory");
-            if (trusted_ca_directory.ok() && !trusted_ca_directory.unpack().empty()) {
-                ctx.add_verify_path(trusted_ca_directory.unpack());
-            } else {
-                string cert_file_path = getConfigurationWithDefault<string>(
-                    getFilesystemPathConfig() + "/certs/fog.pem",
-                    "message",
-                    "Certificate chain file path"
-                );
-                ctx.load_verify_file(cert_file_path);
-            }
-        }
-        boost::asio::io_service io_service;
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
-
-        Client client(
-            out_file,
-            io_service,
-            ctx,
-            url,
-            proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
-            proxy_config->getProxyPort(ProxyProtocol::HTTPS),
-            proxy_config->getProxyAuthentication(ProxyProtocol::HTTPS),
-            token
-        );
-
-        auto connection_result = client.handleConnection();
-        if (!connection_result.ok()) {
-            return connection_result;
-        };
-
-        auto mainloop = Singleton::Consume<I_MainLoop>::by<Downloader>();
-        while (!io_service.stopped()) {
-            io_service.poll_one();
-            mainloop->yield(true);
-        }
-    } catch (const exception &e) {
-        dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what());
-        string error_str = "Failed to get file over HTTPS, exception: " + string(e.what());
-        return genError(error_str);
+    auto downlaod_file = Singleton::Consume<I_Messaging>::by<OrchestrationComp>()->downloadFile(
+        HTTPMethod::GET,
+        url.getQuery(),
+        out_file
+    );
+    if (!downlaod_file.ok()) {
+        dbgWarning(D_ORCHESTRATOR) << "Failed to get file over SSL. Error: " << downlaod_file.getErr().toString();
+        return genError(downlaod_file.getErr().toString());
     }
-
     return Maybe<void>();
 }
 
 Maybe<void>
-HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const string &token)
+HTTPSClient::curlGetFileOverSSL(const URLParser &url, const string &out_file, const string &token)
 {
     try {
         string cert_file_path;
-        if (!token.empty())
-        {
+        if (!token.empty()) {
             cert_file_path = getConfigurationWithDefault<string>(
                 getFilesystemPathConfig() + "/certs/fog.pem",
                 "message",
@@ -581,11 +91,12 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
             );
         }
 
-        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<HTTPClient>();
+        auto proxy_config = Singleton::Consume<I_ProxyConfiguration>::by<OrchestrationComp>();
+        ofstream out_file_stream(out_file, ofstream::out | ofstream::binary);
 
         HttpsCurl ssl_curl_client(
             url,
-            out_file,
+            out_file_stream,
             token,
             proxy_config->getProxyDomain(ProxyProtocol::HTTPS),
             proxy_config->getProxyPort(ProxyProtocol::HTTPS),
@@ -594,8 +105,7 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
 
         ssl_curl_client.setCurlOpts();
         bool connection_ok = ssl_curl_client.connect();
-        if (!connection_ok)
-        {
+        if (!connection_ok) {
             stringstream url_s;
             url_s << url;
             string err_msg = string("Failed to get file over HTTPS. URL: ") + url_s.str();
@@ -603,12 +113,11 @@ HTTPClient::curlGetFileOverSSL(const URLParser &url, ofstream &out_file, const s
         }
 
     } catch (const exception &e) {
-        dbgWarning(D_COMMUNICATION) << "Failed to get file over HTTPS. Error:" << string(e.what());
+        dbgWarning(D_HTTP_REQUEST) << "Failed to get file over HTTPS. Error:" << string(e.what());
         string error_str = "Failed to get file over HTTPS, exception: " + string(e.what());
         return genError(error_str);
     }
 
     return Maybe<void>();
 }
-
 // LCOV_EXCL_STOP

components/security_apps/orchestration/downloader/https_client.h

@@ -0,0 +1,36 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __HTTPS_CLIENT_H__
+#define __HTTPS_CLIENT_H__
+
+#include <string>
+#include "maybe_res.h"
+#include "url_parser.h"
+#include "orchestration_comp.h"
+
+// LCOV_EXCL_START Reason: Depends on real download server.
+class HTTPSClient
+{
+public:
+    Maybe<void> getFile(const URLParser &url, const std::string &out_file, bool auth_required);
+
+private:
+    std::string loadCAChainDir();
+    Maybe<void> getFileSSL(const URLParser &url, const std::string &out_file, const std::string &_token);
+    Maybe<void> getFileSSLDirect(const URLParser &url, const std::string &out_file, const std::string &_token);
+    Maybe<void> curlGetFileOverSSL(const URLParser &url, const std::string &out_file, const std::string &_token);
+};
+// LCOV_EXCL_STOP
+
+#endif // __HTTPS_CLIENT_H__

components/security_apps/orchestration/downloader/https_client_helper.cc

@@ -11,10 +11,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "http_client.h"
+#include "https_client.h"
 
 Maybe<void>
-HTTPClient::getFileSSLDirect(const URLParser &, std::ofstream &, const std::string &)
+HTTPSClient::getFileSSLDirect(const URLParser &, const std::string &, const std::string &)
 {
     return genError("No direct downloading in open-source");
 }

components/security_apps/orchestration/env_details/env_details.cc

@@ -15,6 +15,7 @@
 
 #include "config.h"
 #include "debug.h"
+#include "orchestration_tools.h"
 
 using namespace std;
 
@@ -22,10 +23,16 @@ USE_DEBUG_FLAG(D_LOCAL_POLICY);
 
 static const string k8s_service_account = "/var/run/secrets/kubernetes.io/serviceaccount";
 // LCOV_EXCL_START Reason: can't use on the pipline environment
-EnvDetails::EnvDetails()
+EnvDetails::EnvDetails() : env_type(EnvType::LINUX)
 {
+    auto tools = Singleton::Consume<I_OrchestrationTools>::from<OrchestrationTools>();
+    if (tools->doesFileExist("/.dockerenv")) env_type = EnvType::DOCKER;
     token = retrieveToken();
-    token.empty() ? env_type = EnvType::LINUX : env_type = EnvType::K8S;
+    agent_namespace = retrieveNamespace();
+    if (!token.empty()) {
+        auto env_res = getenv("deployment_type");
+        env_type = env_res != nullptr && env_res == string("non_crd_k8s") ? EnvType::NON_CRD_K8S : EnvType::K8S;
+    }
 }
 
 EnvType
@@ -40,12 +47,24 @@ EnvDetails::getToken()
     return token;
 }
 
+string
+EnvDetails::getNameSpace()
+{
+    return agent_namespace;
+}
+
 string
 EnvDetails::retrieveToken()
 {
     return readFileContent(k8s_service_account + "/token");
 }
 
+string
+EnvDetails::retrieveNamespace()
+{
+    return readFileContent(k8s_service_account + "/namespace");
+}
+
 string
 EnvDetails::readFileContent(const string &file_path)
 {

components/security_apps/orchestration/external_sdk_server/CMakeLists.txt

@@ -0,0 +1,4 @@
+include_directories(${PROJECT_SOURCE_DIR}/core/external_sdk/)
+
+add_library(external_sdk_server external_sdk_server.cc)
+add_subdirectory(external_sdk_server_ut)

components/security_apps/orchestration/external_sdk_server/external_sdk_server.cc

@@ -0,0 +1,348 @@
+#include "external_sdk_server.h"
+
+#include "external_agent_sdk.h"
+#include "log_generator.h"
+#include "rest_server.h"
+#include "generic_metric.h"
+#include "customized_cereal_map.h"
+#include "report/log_rest.h"
+
+using namespace std;
+
+USE_DEBUG_FLAG(D_EXTERNAL_SDK_USER);
+USE_DEBUG_FLAG(D_EXTERNAL_SDK_SERVER);
+
+class ExternalSdkRest : public ServerRest
+{
+public:
+    void
+    doCall() override
+    {
+        dbgFlow(D_EXTERNAL_SDK_SERVER);
+        Maybe<SdkApiType> sdk_event_type = convertToEnum<SdkApiType>(event_type.get());
+        if (!sdk_event_type.ok()) {
+            dbgWarning(D_EXTERNAL_SDK_SERVER) << "Received illegal event type. Type : " << event_type.get();
+            throw JsonError("Illegal event type provided");
+        }
+        dbgDebug(D_EXTERNAL_SDK_SERVER)
+            << "Handling a new external sdk api call event. Type : "
+            << convertApiTypeToString(sdk_event_type.unpack());
+
+        I_ExternalSdkServer *sdk_server = Singleton::Consume<I_ExternalSdkServer>::from<ExternalSdkServer>();
+        switch(sdk_event_type.unpack()) {
+            case SdkApiType::SendCodeEvent: {
+                if (!file.isActive()) {
+                    throw JsonError("File was not provided for code event");
+                }
+                if (!func.isActive()) {
+                    throw JsonError("Function was not provided for code event");
+                }
+                if (!line.isActive()) {
+                    throw JsonError("Line path was not provided for code event");
+                }
+                if (!trace_id.isActive()) {
+                    throw JsonError("Trace ID was not provided for code event");
+                }
+                if (!span_id.isActive()) {
+                    throw JsonError("Span ID was not provided for code event");
+                }
+                if (!message.isActive()) {
+                    throw JsonError("Message was not provided for code event");
+                }
+                sdk_server->sendDebug(
+                    file.get(),
+                    func.get(),
+                    line.get(),
+                    getDebugLevel(),
+                    trace_id.get(),
+                    span_id.get(),
+                    message.get(),
+                    additional_fields.isActive() ? additional_fields.get() : map<string, string>()
+                );
+                return;
+            }
+            case SdkApiType::SendEventDrivenEvent: {
+                if (!event_name.isActive()) {
+                    throw JsonError("Event name was not provided for event");
+                }
+                sdk_server->sendLog(
+                    event_name.get(),
+                    getAudience(),
+                    getSeverity(),
+                    getPriority(),
+                    tag.get(),
+                    additional_fields.isActive() ? additional_fields.get() : map<string, string>()
+                );
+                return;
+            }
+            case SdkApiType::SendGetConfigRequest: {
+                if (!config_path.isActive()) {
+                    throw JsonError("Config path was not provided for get configuration event");
+                }
+                Maybe<string> config_val = sdk_server->getConfigValue(config_path.get());
+                config_value = config_val.ok() ? config_val.unpack() : "";
+                return;
+            }
+            case SdkApiType::SendPeriodicEvent: {
+                if (!event_name.isActive()) {
+                    throw JsonError("Event name was not provided for periodic event");
+                }
+                if (!service_name.isActive()) {
+                    throw JsonError("Service name was not provided for periodic event");
+                }
+                sdk_server->sendMetric(
+                    event_name,
+                    service_name,
+                    getAudienceTeam(),
+                    ReportIS::IssuingEngine::AGENT_CORE,
+                    additional_fields.isActive() ? additional_fields.get() : map<string, string>()
+                );
+                return;
+            }
+            default: {
+                dbgError(D_EXTERNAL_SDK_SERVER) << "Received illegal event type. Type : " << event_type.get();
+            }
+        }
+    }
+
+private:
+    static string
+    convertApiTypeToString(SdkApiType type)
+    {
+        static const EnumArray<SdkApiType, string> api_type_string {
+            "Code Event",
+            "Periodic Event",
+            "Event Driven",
+            "Get Configuration",
+        };
+        return api_type_string[type];
+    }
+
+    Debug::DebugLevel
+    getDebugLevel()
+    {
+        static const map<int, Debug::DebugLevel> debug_levels = {
+            {0, Debug::DebugLevel::TRACE},
+            {1, Debug::DebugLevel::DEBUG},
+            {2, Debug::DebugLevel::INFO},
+            {3, Debug::DebugLevel::WARNING},
+            {4, Debug::DebugLevel::ERROR}
+        };
+        if (!debug_level.isActive()) {
+            throw JsonError("Debug level was not provided for code event");
+        }
+        auto level = debug_levels.find(debug_level.get());
+        if(level == debug_levels.end()) {
+            throw JsonError("Illegal debug level provided");
+        }
+
+        return level->second;
+    }
+
+    ReportIS::Severity
+    getSeverity()
+    {
+        if (!severity.isActive()) {
+            throw JsonError("Event severity was not provided for periodic event");
+        }
+        switch (severity.get()) {
+            case EventSeverity::SeverityCritical: return ReportIS::Severity::CRITICAL;
+            case EventSeverity::SeverityHigh: return ReportIS::Severity::HIGH;
+            case EventSeverity::SeverityMedium: return ReportIS::Severity::MEDIUM;
+            case EventSeverity::SeverityLow: return ReportIS::Severity::LOW;
+            case EventSeverity::SeverityInfo: return ReportIS::Severity::INFO;
+            default:
+                throw JsonError("Illegal event severity provided");
+        }
+    }
+
+    ReportIS::Priority
+    getPriority()
+    {
+        if (!priority.isActive()) {
+            throw JsonError("Event priority was not provided");
+        }
+        switch (priority.get()) {
+            case EventPriority::PriorityUrgent: return ReportIS::Priority::URGENT;
+            case EventPriority::PriorityHigh: return ReportIS::Priority::HIGH;
+            case EventPriority::PriorityMedium: return ReportIS::Priority::MEDIUM;
+            case EventPriority::PriorityLow: return ReportIS::Priority::LOW;
+            default:
+                throw JsonError("Illegal event priority provided");
+        }
+    }
+
+    ReportIS::Audience
+    getAudience()
+    {
+        if (!audience.isActive()) {
+            throw JsonError("Event audience was not provided");
+        }
+        switch (audience.get()) {
+            case EventAudience::AudienceSecurity: return ReportIS::Audience::SECURITY;
+            case EventAudience::AudienceInternal: return ReportIS::Audience::INTERNAL;
+            default:
+                throw JsonError("Illegal event audience provided");
+        }
+    }
+
+    ReportIS::AudienceTeam
+    getAudienceTeam()
+    {
+        if (!team.isActive()) {
+            throw JsonError("Event audience team was not provided");
+        }
+        switch (team.get()) {
+            case EventAudienceTeam::AudienceTeamAgentCore: return ReportIS::AudienceTeam::AGENT_CORE;
+            case EventAudienceTeam::AudienceTeamIot: return ReportIS::AudienceTeam::IOT_NEXT;
+            case EventAudienceTeam::AudienceTeamWaap: return ReportIS::AudienceTeam::WAAP;
+            case EventAudienceTeam::AudienceTeamAgentIntelligence: return ReportIS::AudienceTeam::AGENT_INTELLIGENCE;
+            default:
+                throw JsonError("Illegal event audience team provided");
+        }
+    }
+
+    using additional_fields_map = map<string, string>;
+    C2S_LABEL_PARAM(int, event_type, "eventType");
+    C2S_LABEL_OPTIONAL_PARAM(additional_fields_map, additional_fields, "additionalFields");
+    C2S_LABEL_OPTIONAL_PARAM(string, event_name, "eventName");
+    C2S_LABEL_OPTIONAL_PARAM(string, service_name, "serviceName");
+    C2S_OPTIONAL_PARAM(int, team);
+    C2S_OPTIONAL_PARAM(int, audience);
+    C2S_OPTIONAL_PARAM(int, severity);
+    C2S_OPTIONAL_PARAM(int, priority);
+    C2S_OPTIONAL_PARAM(string, tag);
+    C2S_OPTIONAL_PARAM(string, file);
+    C2S_OPTIONAL_PARAM(string, func);
+    C2S_OPTIONAL_PARAM(int, line);
+    C2S_LABEL_OPTIONAL_PARAM(int, debug_level, "debugLevel");
+    C2S_LABEL_OPTIONAL_PARAM(string, trace_id, "traceId");
+    C2S_LABEL_OPTIONAL_PARAM(string, span_id, "spanId");
+    C2S_OPTIONAL_PARAM(string, message);
+    C2S_LABEL_OPTIONAL_PARAM(string, config_path, "configPath");
+    S2C_LABEL_OPTIONAL_PARAM(string, config_value, "configValue");
+};
+
+class ExternalSdkServer::Impl
+        :
+    public Singleton::Provide<I_ExternalSdkServer>::From<ExternalSdkServer>
+{
+public:
+    void
+    init()
+    {
+        auto rest = Singleton::Consume<I_RestApi>::by<ExternalSdkServer>();
+        rest->addRestCall<ExternalSdkRest>(RestAction::ADD, "sdk-call");
+    }
+
+    void
+    sendLog(
+        const string &event_name,
+        ReportIS::Audience audience,
+        ReportIS::Severity severity,
+        ReportIS::Priority priority,
+        const string &tag_string,
+        const map<string, string> &additional_fields)
+    {
+        Maybe<ReportIS::Tags> tag = TagAndEnumManagement::convertStringToTag(tag_string);
+        set<ReportIS::Tags> tags;
+        if (tag.ok()) tags.insert(tag.unpack());
+        LogGen log(event_name, audience, severity, priority, tags);
+        for (const auto &field : additional_fields) {
+            log << LogField(field.first, field.second);
+        }
+    }
+
+    void
+    sendDebug(
+        const string &file_name,
+        const string &function_name,
+        unsigned int line_number,
+        Debug::DebugLevel debug_level,
+        const string &trace_id,
+        const string &span_id,
+        const string &message,
+        const map<string, string> &additional_fields)
+    {
+        (void)trace_id;
+        (void)span_id;
+        Debug debug(file_name, function_name, line_number, debug_level, D_EXTERNAL_SDK_USER);
+        debug.getStreamAggr() << message;
+        bool is_first_key = true;
+        for (const auto &field : additional_fields) {
+            if (is_first_key) {
+                is_first_key = false;
+                debug.getStreamAggr() << ". ";
+            } else {
+                debug.getStreamAggr() << ", ";
+            }
+            debug.getStreamAggr() << "\"" << field.first << "\": \"" << field.second << "\"";
+        }
+    }
+
+    void
+    sendMetric(
+        const string &event_title,
+        const string &service_name,
+        ReportIS::AudienceTeam team,
+        ReportIS::IssuingEngine issuing_engine,
+        const map<string, string> &additional_fields)
+    {
+        ScopedContext ctx;
+        ctx.registerValue("Service Name", service_name);
+
+        set<ReportIS::Tags> tags;
+        Report metric_to_fog(
+            event_title,
+            Singleton::Consume<I_TimeGet>::by<GenericMetric>()->getWalltime(),
+            ReportIS::Type::PERIODIC,
+            ReportIS::Level::LOG,
+            ReportIS::LogLevel::INFO,
+            ReportIS::Audience::INTERNAL,
+            team,
+            ReportIS::Severity::INFO,
+            ReportIS::Priority::LOW,
+            chrono::seconds(0),
+            LogField("agentId", Singleton::Consume<I_AgentDetails>::by<GenericMetric>()->getAgentId()),
+            tags,
+            ReportIS::Tags::INFORMATIONAL,
+            issuing_engine
+        );
+
+        for (const auto &field : additional_fields) {
+            metric_to_fog << LogField(field.first, field.second);
+        }
+
+        LogRest metric_client_rest(metric_to_fog);
+
+        string fog_metric_uri = getConfigurationWithDefault<string>("/api/v1/agents/events", "metric", "fogMetricUri");
+        Singleton::Consume<I_Messaging>::by<ExternalSdkServer>()->sendAsyncMessage(
+                HTTPMethod::POST,
+                fog_metric_uri,
+                metric_client_rest,
+                MessageCategory::METRIC,
+                MessageMetadata(),
+                false
+        );
+    }
+
+    Maybe<string>
+    getConfigValue(const string &config_path)
+    {
+        auto config_val = getProfileAgentSetting<string>(config_path);
+        if (!config_val.ok()) {
+            stringstream error;
+            error << "Failed to get configuration. Config path: " << config_path << ", Error: " << config_val.getErr();
+            return genError(error.str());
+        }
+        return config_val.unpack();
+    }
+};
+
+ExternalSdkServer::ExternalSdkServer() : Component("ExternalSdkServer"), pimpl(make_unique<Impl>()) {}
+ExternalSdkServer::~ExternalSdkServer() {}
+
+void ExternalSdkServer::init() { pimpl->init(); }
+void ExternalSdkServer::fini() {}
+
+void ExternalSdkServer::preload() {}

components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/CMakeLists.txt

@@ -0,0 +1,7 @@
+link_directories(${BOOST_ROOT}/lib)
+
+add_unit_test(
+        external_sdk_server_ut
+        "external_sdk_server_ut.cc"
+        "external_sdk_server;mainloop;singleton;rest;environment;time_proxy;logging;event_is;metric;-lboost_context;agent_details;-lboost_regex;messaging;"
+)

components/security_apps/orchestration/external_sdk_server/external_sdk_server_ut/external_sdk_server_ut.cc

@@ -0,0 +1,349 @@
+#include <stdio.h>
+#include <stdarg.h>
+
+#include "external_sdk_server.h"
+
+#include "cptest.h"
+#include "mock/mock_rest_api.h"
+#include "mock/mock_messaging.h"
+#include "mock/mock_logging.h"
+#include "mock/mock_time_get.h"
+#include "config.h"
+#include "config_component.h"
+#include "agent_details.h"
+
+using namespace std;
+using namespace testing;
+
+class ExternalSdkServerTest : public Test
+{
+public:
+    ExternalSdkServerTest()
+    {
+        EXPECT_CALL(rest_mocker, mockRestCall(RestAction::ADD, "sdk-call", _)).WillOnce(
+            WithArg<2>(
+                Invoke(
+                    [this](const unique_ptr<RestInit> &rest_ptr)
+                    {
+                        mock_sdk_rest = rest_ptr->getRest();
+                        return true;
+                    }
+                )
+            )
+        );
+
+        sdk_server.preload();
+        sdk_server.init();
+        i_sdk = Singleton::Consume<I_ExternalSdkServer>::from(sdk_server);
+    }
+
+    ~ExternalSdkServerTest()
+    {
+        sdk_server.fini();
+    }
+
+    ExternalSdkServer sdk_server;
+    NiceMock<MockTimeGet> mock_timer;
+    StrictMock<MockMessaging> messaging_mocker;
+    StrictMock<MockRestApi> rest_mocker;
+    StrictMock<MockLogging> log_mocker;
+    unique_ptr<ServerRest> mock_sdk_rest;
+    I_ExternalSdkServer *i_sdk;
+    ConfigComponent conf;
+    AgentDetails agent_details;
+    ::Environment env;
+};
+
+TEST_F(ExternalSdkServerTest, initTest)
+{
+}
+
+TEST_F(ExternalSdkServerTest, configCall)
+{
+    Maybe<string> no_conf = i_sdk->getConfigValue("key1");
+    EXPECT_FALSE(no_conf.ok());
+    string config_json =
+        "{\n"
+            "\"agentSettings\": [\n"
+                "{\n"
+                    "\"id\": \"id1\",\n"
+                    "\"key\": \"key1\",\n"
+                    "\"value\": \"value1\"\n"
+                "},\n"
+                "{\n"
+                    "\"id\": \"id1\",\n"
+                    "\"key\": \"key2\",\n"
+                    "\"value\": \"value2\"\n"
+                "}\n"
+            "]\n"
+        "}\n";
+    conf.preload();
+    istringstream conf_stream(config_json);
+    ASSERT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(conf_stream));
+
+    Maybe<string> conf_found = i_sdk->getConfigValue("key1");
+    ASSERT_TRUE(conf_found.ok());
+    EXPECT_EQ(conf_found.unpack(), "value1");
+
+    conf_found = i_sdk->getConfigValue("key2");
+    ASSERT_TRUE(conf_found.ok());
+    EXPECT_EQ(conf_found.unpack(), "value2");
+
+    stringstream config_call_body;
+    config_call_body << "{ \"eventType\": 3, \"configPath\": \"key1\" }";
+
+    Maybe<string> sdk_conf = mock_sdk_rest->performRestCall(config_call_body);
+    ASSERT_TRUE(sdk_conf.ok());
+    EXPECT_EQ(
+        sdk_conf.unpack(),
+        "{\n"
+        "    \"configValue\": \"value1\"\n"
+        "}"
+    );
+}
+
+template <typename T>
+string
+toJson(const T &obj)
+{
+    stringstream ss;
+    {
+        cereal::JSONOutputArchive ar(ss);
+        obj.serialize(ar);
+    }
+    return ss.str();
+}
+
+TEST_F(ExternalSdkServerTest, eventDrivenCall)
+{
+    string generated_log;
+    EXPECT_CALL(log_mocker, getCurrentLogId()).Times(2).WillRepeatedly(Return(0));
+    EXPECT_CALL(log_mocker, sendLog(_)).Times(2).WillRepeatedly(
+        WithArg<0>(
+            Invoke(
+                [&] (const Report &msg)
+                {
+                    generated_log = toJson(msg);
+                }
+            )
+        )
+    );
+
+    i_sdk->sendLog(
+        "my log",
+        ReportIS::Audience::INTERNAL,
+        ReportIS::Severity::LOW,
+        ReportIS::Priority::HIGH,
+        "IPS",
+        {{"key1", "value1"}, {"key2", "value2"}}
+    );
+    static const string expected_log =
+        "{\n"
+        "    \"eventTime\": \"\",\n"
+        "    \"eventName\": \"my log\",\n"
+        "    \"eventSeverity\": \"Low\",\n"
+        "    \"eventPriority\": \"High\",\n"
+        "    \"eventType\": \"Event Driven\",\n"
+        "    \"eventLevel\": \"Log\",\n"
+        "    \"eventLogLevel\": \"info\",\n"
+        "    \"eventAudience\": \"Internal\",\n"
+        "    \"eventAudienceTeam\": \"\",\n"
+        "    \"eventFrequency\": 0,\n"
+        "    \"eventTags\": [\n"
+        "        \"IPS\"\n"
+        "    ],\n"
+        "    \"eventSource\": {\n"
+        "        \"agentId\": \"Unknown\",\n"
+        "        \"eventTraceId\": \"\",\n"
+        "        \"eventSpanId\": \"\",\n"
+        "        \"issuingEngineVersion\": \"\",\n"
+        "        \"serviceName\": \"Unnamed Nano Service\"\n"
+        "    },\n"
+        "    \"eventData\": {\n"
+        "        \"logIndex\": 0,\n"
+        "        \"key1\": \"value1\",\n"
+        "        \"key2\": \"value2\"\n"
+        "    }\n"
+        "}";
+
+    EXPECT_EQ(generated_log, expected_log);
+
+    string event_call_body =
+        "{\n"
+        "    \"eventType\": 2,\n"
+        "    \"eventName\": \"my log\",\n"
+        "    \"audience\": 1,\n"
+        "    \"severity\": 3,\n"
+        "    \"priority\": 1,\n"
+        "    \"tag\": \"IPS\",\n"
+        "    \"team\": 3,\n"
+        "    \"additionalFields\": {\n"
+        "        \"key1\": \"value1\",\n"
+        "        \"key2\": \"value2\"\n"
+        "    }\n"
+        "}";
+
+    generated_log = "";
+    stringstream event_call_stream;
+    event_call_stream << event_call_body;
+    EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok());
+    EXPECT_EQ(generated_log, expected_log);
+}
+
+TEST_F(ExternalSdkServerTest, periodicEventCall)
+{
+    string message_body;
+    EXPECT_CALL(
+        messaging_mocker,
+        sendAsyncMessage(
+            HTTPMethod::POST,
+            "/api/v1/agents/events",
+            _,
+            MessageCategory::METRIC,
+            _,
+            false
+        )
+    ).Times(2).WillRepeatedly(SaveArg<2>(&message_body));
+
+    i_sdk->sendMetric(
+        "my metric",
+        "matrix",
+        ReportIS::AudienceTeam::AGENT_INTELLIGENCE,
+        ReportIS::IssuingEngine::AGENT_CORE,
+        {{"key", "value"}}
+    );
+
+    static const string expected_message =
+        "{\n"
+        "    \"log\": {\n"
+        "        \"eventTime\": \"\",\n"
+        "        \"eventName\": \"my metric\",\n"
+        "        \"eventSeverity\": \"Info\",\n"
+        "        \"eventPriority\": \"Low\",\n"
+        "        \"eventType\": \"Periodic\",\n"
+        "        \"eventLevel\": \"Log\",\n"
+        "        \"eventLogLevel\": \"info\",\n"
+        "        \"eventAudience\": \"Internal\",\n"
+        "        \"eventAudienceTeam\": \"Agent Intelligence\",\n"
+        "        \"eventFrequency\": 0,\n"
+        "        \"eventTags\": [\n"
+        "            \"Informational\"\n"
+        "        ],\n"
+        "        \"eventSource\": {\n"
+        "            \"agentId\": \"Unknown\",\n"
+        "            \"issuingEngine\": \"Agent Core\",\n"
+        "            \"eventTraceId\": \"\",\n"
+        "            \"eventSpanId\": \"\",\n"
+        "            \"issuingEngineVersion\": \"\",\n"
+        "            \"serviceName\": \"matrix\"\n"
+        "        },\n"
+        "        \"eventData\": {\n"
+        "            \"key\": \"value\"\n"
+        "        }\n"
+        "    }\n"
+        "}";
+
+    EXPECT_EQ(message_body, expected_message);
+
+    string event_call_body =
+        "{\n"
+        "    \"eventType\": 1,\n"
+        "    \"eventName\": \"my metric\",\n"
+        "    \"serviceName\": \"matrix\",\n"
+        "    \"team\": 3,\n"
+        "    \"additionalFields\": {\n"
+        "        \"key\": \"value\"\n"
+        "    }\n"
+        "}";
+
+    stringstream event_call_stream;
+    event_call_stream << event_call_body;
+
+    message_body = "";
+    EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok());
+    EXPECT_EQ(message_body, expected_message);
+}
+
+USE_DEBUG_FLAG(D_EXTERNAL_SDK_USER);
+USE_DEBUG_FLAG(D_EXTERNAL_SDK_SERVER);
+
+TEST_F(ExternalSdkServerTest, codeEventCall)
+{
+    ostringstream capture_debug;
+    Debug::setUnitTestFlag(D_EXTERNAL_SDK_SERVER, Debug::DebugLevel::TRACE);
+    Debug::setUnitTestFlag(D_EXTERNAL_SDK_USER, Debug::DebugLevel::TRACE);
+    Debug::setNewDefaultStdout(&capture_debug);
+
+    i_sdk->sendDebug(
+        "file.cc",
+        "myFunc2",
+        42,
+        Debug::DebugLevel::TRACE,
+        "123",
+        "abc",
+        "h#l1ow w0r!d",
+        {{"hi", "universe"}}
+    );
+
+    EXPECT_THAT(
+        capture_debug.str(),
+        HasSubstr(
+            "[myFunc2@file.cc:42                                           | >>>] "
+            "h#l1ow w0r!d. \"hi\": \"universe\"\n"
+        )
+    );
+
+
+    string debug_event =
+        "{\n"
+        "    \"eventType\": 0,\n"
+        "    \"file\": \"my file\",\n"
+        "    \"func\": \"function_name\",\n"
+        "    \"line\": 42,\n"
+        "    \"debugLevel\": 0,\n"
+        "    \"traceId\": \"\",\n"
+        "    \"spanId\": \"span2323\",\n"
+        "    \"message\": \"some short debug\",\n"
+        "    \"team\": 1,\n"
+        "    \"additionalFields\": {\n"
+        "        \"name\": \"moshe\",\n"
+        "        \"food\": \"bamba\"\n"
+        "    }\n"
+        "}";
+
+    stringstream event_call_stream;
+    event_call_stream << debug_event;
+
+    EXPECT_TRUE(mock_sdk_rest->performRestCall(event_call_stream).ok());
+
+    EXPECT_THAT(
+        capture_debug.str(),
+        HasSubstr(
+            "[function_name@my file:42                                     | >>>] "
+            "some short debug. \"food\": \"bamba\", \"name\": \"moshe\"\n"
+        )
+    );
+
+    Debug::setNewDefaultStdout(&cout);
+}
+
+TEST_F(ExternalSdkServerTest, ilegalEventCall)
+{
+    string event_call_body =
+        "{\n"
+        "    \"eventType\": 7,\n"
+        "    \"eventName\": \"my metric\",\n"
+        "    \"serviceName\": \"matrix\",\n"
+        "    \"team\": 3,\n"
+        "    \"additionalFields\": {\n"
+        "        \"key\": \"value\"\n"
+        "    }\n"
+        "}";
+
+    stringstream event_call_stream;
+    event_call_stream << event_call_body;
+
+    Maybe<string> failed_respond = mock_sdk_rest->performRestCall(event_call_stream);
+    EXPECT_FALSE(failed_respond.ok());
+    EXPECT_EQ(failed_respond.getErr(), "Illegal event type provided");
+}

components/security_apps/orchestration/health_check/health_check.cc

@@ -40,6 +40,8 @@ public:
         i_mainloop = Singleton::Consume<I_MainLoop>::by<HealthChecker>();
         i_socket = Singleton::Consume<I_Socket>::by<HealthChecker>();
         i_orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<HealthChecker>();
+        i_service_controller = Singleton::Consume<I_ServiceController>::by<HealthChecker>();
+
         initConfig();
         initServerSocket();
 
@@ -246,7 +248,7 @@ private:
             }
         }
         routine_id = i_mainloop->addFileRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             server_sock,
             [this] () { handleConnection(); },
             "Health check probe server",
@@ -270,18 +272,17 @@ private:
         }
 
         if (NGEN::Filesystem::exists(rpm_full_load_path)) {
-            dbgTrace(D_HEALTH_CHECK) << rpm_full_load_path << " exists, returning healthy status";
-            return HealthCheckStatus::HEALTHY;
+            dbgTrace(D_HEALTH_CHECK) << "RPM is fully loaded";
+            return i_service_controller->getServicesPolicyStatus()
+                ? HealthCheckStatus::HEALTHY
+                : HealthCheckStatus::UNHEALTHY;
         }
 
-        if (NGEN::Filesystem::exists(rpm_partial_load_path)) {
-            dbgTrace(D_HEALTH_CHECK) << rpm_partial_load_path << " exists, returning degraded status";
-            return HealthCheckStatus::DEGRADED;
-        }
-
-        if (!NGEN::Filesystem::exists(first_rpm_policy_load_path)) {
-            dbgTrace(D_HEALTH_CHECK) << "Could not load latest RPM policy, returning degraded status";
-            return HealthCheckStatus::DEGRADED;
+        if (NGEN::Filesystem::exists(rpm_partial_load_path) || !NGEN::Filesystem::exists(first_rpm_policy_load_path)) {
+            dbgTrace(D_HEALTH_CHECK) << "RPM is partially loaded";
+            return i_service_controller->getServicesPolicyStatus()
+                ? HealthCheckStatus::DEGRADED
+                : HealthCheckStatus::UNHEALTHY;
         }
 
         dbgTrace(D_HEALTH_CHECK) << "RPM is not loaded, returning unhealthy status";
@@ -344,7 +345,7 @@ private:
         dbgDebug(D_HEALTH_CHECK) << "Successfully accepted client, client fd: " << new_client_socket;
         open_connections_counter++;
         auto curr_routine = i_mainloop->addOneTimeRoutine(
-            I_MainLoop::RoutineType::RealTime,
+            I_MainLoop::RoutineType::System,
             [this] ()
             {
                 auto curr_routine_id = i_mainloop->getCurrentRoutineId().unpack();
@@ -442,6 +443,7 @@ private:
     I_Socket *i_socket                              = nullptr;
     I_Health_Check_Manager *i_health_check_manager  = nullptr;
     I_OrchestrationStatus *i_orchestration_status  = nullptr;
+    I_ServiceController *i_service_controller = nullptr;
 };
 
 HealthChecker::HealthChecker() : Component("HealthChecker"), pimpl(make_unique<Impl>()) {}

components/security_apps/orchestration/health_check/health_check_ut/CMakeLists.txt

@@ -3,5 +3,5 @@ link_directories(${BOOST_ROOT}/lib)
 add_unit_test(
     health_check_ut
     "health_check_ut.cc"
-    "health_check;messaging;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system"
+    "health_check;updates_process_reporter;messaging;mainloop;singleton;agent_details;config;logging;metric;event_is;health_check_manager;-lboost_regex;-lboost_system"
 )

components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc

@@ -9,6 +9,7 @@
 #include "mock/mock_shell_cmd.h"
 #include "mock/mock_orchestration_status.h"
 #include "health_check_manager.h"
+#include "mock/mock_service_controller.h"
 
 #include "config.h"
 #include "config_component.h"
@@ -76,6 +77,7 @@ public:
     I_MainLoop::Routine                 handle_probe_routine;
     HealthCheckManager                  health_check_manager;
     I_Health_Check_Manager              *i_health_check_manager;
+    StrictMock<MockServiceController>   mock_service_controller;
 };
 
 TEST_F(HealthCheckerTest, empty)
@@ -111,12 +113,12 @@ TEST_F(HealthCheckerTest, clientConnection)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -145,7 +147,7 @@ TEST_F(HealthCheckerTest, loadFromDynamicConfiguration)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     health_checker.init();
@@ -183,7 +185,7 @@ TEST_F(HealthCheckerTest, connectionsLimit)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(mock_mainloop, doesRoutineExist(_)).WillRepeatedly(Return(false));
@@ -218,12 +220,12 @@ TEST_F(HealthCheckerTest, disablingAfterEnabled)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -273,12 +275,12 @@ TEST_F(HealthCheckerTest, changePortIpConfig)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -321,12 +323,12 @@ TEST_F(HealthCheckerTest, FailedHealthCheck)
 
     EXPECT_CALL(
         mock_mainloop,
-        addFileRoutine(I_MainLoop::RoutineType::RealTime, _, _, _, true)
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
     ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
 
     EXPECT_CALL(
         mock_mainloop,
-        addOneTimeRoutine(I_MainLoop::RoutineType::RealTime, _, "Health check probe connection handler", true)
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
     ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
 
     int socket = 1;
@@ -342,3 +344,58 @@ TEST_F(HealthCheckerTest, FailedHealthCheck)
     connection_handler_routine();
     setConfiguration(false, "Health Check", "Probe enabled");
 }
+
+TEST_F(HealthCheckerTest, StandaloneHealthCheck)
+{
+    setenv("DOCKER_RPM_ENABLED", "true", 1);
+
+    string ip = "1.2.3.4";
+    setConfiguration(ip, "Health Check", "Probe IP");
+    uint port = 11600;
+    setConfiguration(port, "Health Check", "Probe port");
+
+    NGEN::Filesystem::touchFile("/tmp/wd.all_running");
+    NGEN::Filesystem::touchFile("/tmp/rpm_full_load");
+
+    auto on_exit = make_scope_exit(
+        []() {
+            NGEN::Filesystem::deleteFile("/tmp/wd.all_running");
+            NGEN::Filesystem::deleteFile("/tmp/rpm_full_load");
+        }
+    );
+
+    const string policy_version = "1";
+    EXPECT_CALL(mock_orchestration_status, getPolicyVersion()).WillRepeatedly(ReturnRef(policy_version));
+    EXPECT_CALL(mock_service_controller, getServicesPolicyStatus()).WillRepeatedly(Return(true));
+
+    EXPECT_CALL(
+        mock_mainloop,
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, _, false)
+    ).WillOnce(DoAll(SaveArg<1>(&handle_probe_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_socket,
+        genSocket(I_Socket::SocketType::TCP, false, true, _)
+    ).WillRepeatedly(Return(1));
+
+    EXPECT_CALL(
+        mock_mainloop,
+        addFileRoutine(I_MainLoop::RoutineType::System, _, _, _, true)
+    ).WillRepeatedly(DoAll(SaveArg<2>(&connection_handler_routine), Return(0)));
+
+    EXPECT_CALL(
+        mock_mainloop,
+        addOneTimeRoutine(I_MainLoop::RoutineType::System, _, "Health check probe connection handler", true)
+    ).WillOnce(DoAll(SaveArg<1>(&connection_handler_routine), Return(0)));
+
+    int socket = 1;
+    EXPECT_CALL(mock_socket, acceptSocket(1, false, ip)).WillOnce(Return(socket));
+    EXPECT_CALL(mock_mainloop, getCurrentRoutineId()).WillRepeatedly(Return(0));
+    EXPECT_CALL(mock_socket, receiveData(_, 1, false)).WillOnce(Return(vector<char>()));
+    EXPECT_CALL(mock_socket, writeData(_, response_buffer)).WillOnce(Return(true));
+    EXPECT_CALL(mock_socket, closeSocket(socket)).Times(2);
+    health_checker.init();
+    handle_probe_routine();
+    connection_handler_routine();
+    connection_handler_routine();
+}

components/security_apps/orchestration/health_check_manager/CMakeLists.txt

@@ -1,3 +1 @@
 add_library(health_check_manager health_check_manager.cc)
-
-add_subdirectory(health_check_manager_ut)

components/security_apps/orchestration/health_check_manager/health_check_manager.cc

@@ -21,6 +21,7 @@
 #include "config.h"
 #include "cereal/archives/json.hpp"
 #include "customized_cereal_map.h"
+#include "updates_process_event.h"
 
 using namespace std;
 
@@ -79,19 +80,22 @@ class HealthCheckValue
 public:
     HealthCheckValue() = default;
 
-    HealthCheckValue(HealthCheckStatus raw_status, const map<string, HealthCheckStatusReply> &descriptions)
+    HealthCheckValue(HealthCheckStatus raw_status, const HealthCheckStatusReply &description)
             :
         status(raw_status)
     {
-        for (const auto &single_stat : descriptions) {
-            if (single_stat.second.getStatus() == HealthCheckStatus::HEALTHY) {
-                dbgTrace(D_HEALTH_CHECK_MANAGER) << "Ignoring healthy status reply. Comp name: " << single_stat.first;
-                continue;
-            }
+        if (description.getStatus() == HealthCheckStatus::HEALTHY) {
+            dbgTrace(D_HEALTH_CHECK_MANAGER)
+                << "Ignoring healthy status reply. Comp name: "
+                << description.getCompName();
+            return;
+        }
 
-            for (const auto &status : single_stat.second.getExtendedStatus()) {
-                errors.push_back(HealthCheckError(single_stat.first + " " + status.first, status.second));
-            }
+        for (const auto &extended_status : description.getExtendedStatus()) {
+            errors.push_back(
+                HealthCheckError(description.getCompName() + " " + extended_status.first,
+                extended_status.second
+            ));
         }
     }
 
@@ -113,9 +117,9 @@ private:
 class HealthCheckPatch : public ClientRest
 {
 public:
-    HealthCheckPatch(HealthCheckStatus raw_status, const map<string, HealthCheckStatusReply> &descriptions)
+    HealthCheckPatch(HealthCheckStatus raw_status, const HealthCheckStatusReply &description)
     {
-        health_check = HealthCheckValue(raw_status, descriptions);
+        health_check = HealthCheckValue(raw_status, description);
     }
 
     C2S_LABEL_PARAM(HealthCheckValue, health_check, "healthCheck");
@@ -123,7 +127,8 @@ public:
 
 class HealthCheckManager::Impl
         :
-    Singleton::Provide<I_Health_Check_Manager>::From<HealthCheckManager>
+    Singleton::Provide<I_Health_Check_Manager>::From<HealthCheckManager>,
+    public Listener<UpdatesProcessEvent>
 {
 public:
     void
@@ -132,6 +137,7 @@ public:
         auto rest = Singleton::Consume<I_RestApi>::by<HealthCheckManager>();
         rest->addRestCall<HealthCheckOnDemand>(RestAction::SHOW, "health-check-on-demand");
 
+        registerListener();
         int interval_in_seconds =
             getProfileAgentSettingWithDefault<int>(30, "agent.healthCheck.intervalInSeconds");
 
@@ -157,9 +163,62 @@ public:
     void
     printRepliesHealthStatus(ofstream &oputput_file)
     {
-        getRegisteredComponentsHealthStatus();
         cereal::JSONOutputArchive ar(oputput_file);
-        ar(cereal::make_nvp("allComponentsHealthCheckReplies", all_comps_health_status));
+        ar(cereal::make_nvp(health_check_reply.getCompName(), health_check_reply));
+    }
+
+    void
+    upon(const UpdatesProcessEvent &event)
+    {
+
+        OrchestrationStatusFieldType status_field_type = event.getStatusFieldType();
+        HealthCheckStatus _status = convertResultToHealthCheckStatus(event.getResult());
+        string status_field_type_str = convertOrchestrationStatusFieldTypeToStr(status_field_type);
+
+        extended_status[status_field_type_str] =
+            _status == HealthCheckStatus::HEALTHY ?
+            "Success" :
+            event.parseDescription();
+        field_types_status[status_field_type_str] = _status;
+
+        switch(_status) {
+            case HealthCheckStatus::UNHEALTHY: {
+                general_health_aggregated_status = HealthCheckStatus::UNHEALTHY;
+                break;
+            }
+            case HealthCheckStatus::DEGRADED: {
+                for (const auto &type_status : field_types_status) {
+                    if ((type_status.first != status_field_type_str)
+                            && (type_status.second == HealthCheckStatus::UNHEALTHY))
+                    {
+                        break;
+                    }
+                }
+                general_health_aggregated_status = HealthCheckStatus::DEGRADED;
+                break;
+            }
+            case HealthCheckStatus::HEALTHY: {
+                for (const auto &type_status : field_types_status) {
+                    if ((type_status.first != status_field_type_str)
+                            && (type_status.second == HealthCheckStatus::UNHEALTHY
+                                || type_status.second == HealthCheckStatus::DEGRADED)
+                        )
+                    {
+                        break;
+                    }
+                    general_health_aggregated_status = HealthCheckStatus::HEALTHY;
+                }
+                break;
+            }
+            case HealthCheckStatus::IGNORED: {
+                break;
+            }
+        }
+        health_check_reply = HealthCheckStatusReply(
+            "Orchestration",
+            general_health_aggregated_status,
+            extended_status
+        );
     }
 
 private:
@@ -168,9 +227,10 @@ private:
     {
         dbgFlow(D_HEALTH_CHECK_MANAGER) << "Sending a health check patch";
 
-        HealthCheckPatch patch_to_send(general_health_aggregated_status, all_comps_health_status);
-        auto messaging = Singleton::Consume<I_Messaging>::by<HealthCheckManager>();
-        return messaging->sendSyncMessageWithoutResponse(
+        HealthCheckPatch patch_to_send(general_health_aggregated_status, health_check_reply);
+        extended_status.clear();
+        field_types_status.clear();
+        return Singleton::Consume<I_Messaging>::by<HealthCheckManager>()->sendSyncMessageWithoutResponse(
             HTTPMethod::PATCH,
             "/agents",
             patch_to_send,
@@ -178,59 +238,11 @@ private:
         );
     }
 
-    void
-    getRegisteredComponentsHealthStatus()
-    {
-        vector<HealthCheckStatusReply> health_check_event_reply = HealthCheckStatusEvent().query();
-        all_comps_health_status.clear();
-        for (const auto &reply : health_check_event_reply) {
-            if (reply.getStatus() != HealthCheckStatus::IGNORED) {
-                all_comps_health_status.emplace(reply.getCompName(), reply);
-            }
-        }
-    }
-
-    void
-    calcGeneralHealthAggregatedStatus()
-    {
-        general_health_aggregated_status = HealthCheckStatus::HEALTHY;
-
-        for (const auto &reply : all_comps_health_status) {
-            HealthCheckStatus status = reply.second.getStatus();
-
-            dbgTrace(D_HEALTH_CHECK_MANAGER)
-                << "Current aggregated status is: "
-                << HealthCheckStatusReply::convertHealthCheckStatusToStr(
-                        general_health_aggregated_status
-                    )
-                << ". Got health status: "
-                << HealthCheckStatusReply::convertHealthCheckStatusToStr(status)
-                << "for component: "
-                << reply.first;
-
-            switch (status) {
-                case HealthCheckStatus::UNHEALTHY : {
-                    general_health_aggregated_status = HealthCheckStatus::UNHEALTHY;
-                    return;
-                }
-                case HealthCheckStatus::DEGRADED : {
-                    general_health_aggregated_status = HealthCheckStatus::DEGRADED;
-                    break;
-                }
-                case HealthCheckStatus::IGNORED : break;
-                case HealthCheckStatus::HEALTHY : break;
-            }
-        }
-    }
-
     void
     executeHealthCheck()
     {
         dbgFlow(D_HEALTH_CHECK_MANAGER) << "Collecting health status from all registered components.";
 
-        getRegisteredComponentsHealthStatus();
-        calcGeneralHealthAggregatedStatus();
-
         dbgTrace(D_HEALTH_CHECK_MANAGER)
             << "Aggregated status: "
             << HealthCheckStatusReply::convertHealthCheckStatusToStr(general_health_aggregated_status);
@@ -244,9 +256,47 @@ private:
         };
     }
 
-    HealthCheckStatus general_health_aggregated_status;
-    map<string, HealthCheckStatusReply> all_comps_health_status;
+    string
+    convertOrchestrationStatusFieldTypeToStr(OrchestrationStatusFieldType type)
+    {
+        switch (type) {
+            case OrchestrationStatusFieldType::REGISTRATION : return "Registration";
+            case OrchestrationStatusFieldType::MANIFEST : return "Manifest";
+            case OrchestrationStatusFieldType::LAST_UPDATE : return "Last Update";
+            case OrchestrationStatusFieldType::COUNT : return "Count";
+        }
+
+        dbgAssert(false)
+            << AlertInfo(AlertTeam::CORE, "orchestration health")
+            << "Trying to convert unknown orchestration status field to string.";
+        return "";
+    }
+
+    HealthCheckStatus
+    convertResultToHealthCheckStatus(UpdatesProcessResult result)
+    {
+        switch (result) {
+            case UpdatesProcessResult::SUCCESS : return HealthCheckStatus::HEALTHY;
+            case UpdatesProcessResult::UNSET : return HealthCheckStatus::IGNORED;
+            case UpdatesProcessResult::FAILED : return HealthCheckStatus::UNHEALTHY;
+            case UpdatesProcessResult::DEGRADED : return HealthCheckStatus::DEGRADED;
+        }
+
+        dbgAssert(false)
+            << AlertInfo(AlertTeam::CORE, "orchestration health")
+            << "Trying to convert unknown update process result field to health check status.";
+        return HealthCheckStatus::IGNORED;
+    }
+
+    HealthCheckStatus general_health_aggregated_status = HealthCheckStatus::HEALTHY;
+    HealthCheckStatusReply health_check_reply = HealthCheckStatusReply(
+        "Orchestration",
+        HealthCheckStatus::HEALTHY,
+        {}
+    );
     bool should_patch_report;
+    map<string, string> extended_status;
+    map<string, HealthCheckStatus> field_types_status;
 };
 
 HealthCheckManager::HealthCheckManager() : Component("HealthCheckManager"), pimpl(make_unique<Impl>()) {}

components/security_apps/orchestration/health_check_manager/health_check_manager_ut/health_check_manager_ut.cc

@@ -13,42 +13,13 @@
 #include "mock/mock_mainloop.h"
 #include "mock/mock_messaging.h"
 #include "mock/mock_rest_api.h"
+#include "updates_process_event.h"
 
 using namespace std;
 using namespace testing;
 
 USE_DEBUG_FLAG(D_HEALTH_CHECK);
 
-class TestHealthCheckStatusListener : public Listener<HealthCheckStatusEvent>
-{
-public:
-    void upon(const HealthCheckStatusEvent &) override {}
-
-    HealthCheckStatusReply
-    respond(const HealthCheckStatusEvent &) override
-    {
-        map<string, string> extended_status;
-        extended_status["team"] = team;
-        extended_status["city"] = city;
-        HealthCheckStatusReply reply(comp_name, status, extended_status);
-        return reply;
-    }
-
-    void setStatus(HealthCheckStatus new_status) { status = new_status; }
-
-    string getListenerName() const { return "TestHealthCheckStatusListener"; }
-
-private:
-    static const string comp_name;
-    HealthCheckStatus status = HealthCheckStatus::HEALTHY;
-    static const string team;
-    static const string city;
-};
-
-const string TestHealthCheckStatusListener::comp_name = "Test";
-const string TestHealthCheckStatusListener::team = "Hapoel";
-const string TestHealthCheckStatusListener::city = "Tel-Aviv";
-
 class TestEnd {};
 
 class HealthCheckManagerTest : public Test
@@ -56,8 +27,7 @@ class HealthCheckManagerTest : public Test
 public:
     HealthCheckManagerTest()
     {
-        Debug::setNewDefaultStdout(&debug_output);
-        Debug::setUnitTestFlag(D_HEALTH_CHECK, Debug::DebugLevel::INFO);
+        Debug::setUnitTestFlag(D_HEALTH_CHECK, Debug::DebugLevel::NOISE);
 
         EXPECT_CALL(mock_ml, addRecurringRoutine(_, _, _, _, _)).WillRepeatedly(
             DoAll(SaveArg<2>(&health_check_periodic_routine), Return(1))
@@ -70,7 +40,6 @@ public:
         );
 
         env.preload();
-        event_listener.registerListener();
 
         env.init();
 
@@ -98,14 +67,12 @@ public:
     StrictMock<MockMainLoop>        mock_ml;
     StrictMock<MockRestApi>         mock_rest;
     StrictMock<MockMessaging>       mock_message;
-    stringstream                    debug_output;
     ConfigComponent                 config;
     Config::I_Config                *i_config = nullptr;
     ::Environment                   env;
     HealthCheckManager              health_check_manager;
     I_Health_Check_Manager          *i_health_check_manager;
     unique_ptr<ServerRest>          health_check_server;
-    TestHealthCheckStatusListener   event_listener;
 };
 
 TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
@@ -142,7 +109,20 @@ TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
     EXPECT_EQ(actual_body, expected_healthy_body);
     EXPECT_EQ("Healthy", aggregated_status_str);
 
-    event_listener.setStatus(HealthCheckStatus::DEGRADED);
+    UpdatesProcessEvent(
+        UpdatesProcessResult::DEGRADED,
+        UpdatesConfigType::SETTINGS,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "setting.json",
+        "File not found"
+    ).notify();
+    UpdatesProcessEvent(
+        UpdatesProcessResult::DEGRADED,
+        UpdatesConfigType::MANIFEST,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "manifest.json",
+        "File not found"
+    ).notify();
     try {
         health_check_periodic_routine();
     } catch (const TestEnd &t) {}
@@ -156,16 +136,16 @@ TEST_F(HealthCheckManagerTest, runPeriodicHealthCheckTest)
         "        \"status\": \"Degraded\",\n"
         "        \"errors\": [\n"
         "            {\n"
-        "                \"code\": \"Test city\",\n"
+        "                \"code\": \"Orchestration Last Update\",\n"
         "                \"message\": [\n"
-        "                    \"Tel-Aviv\"\n"
+        "                    \"Failed to download the file setting.json. Error: File not found\"\n"
         "                ],\n"
         "                \"internal\": true\n"
         "            },\n"
         "            {\n"
-        "                \"code\": \"Test team\",\n"
+        "                \"code\": \"Orchestration Manifest\",\n"
         "                \"message\": [\n"
-        "                    \"Hapoel\"\n"
+        "                    \"Failed to download the file manifest.json. Error: File not found\"\n"
         "                ],\n"
         "                \"internal\": true\n"
         "            }\n"
@@ -196,19 +176,24 @@ TEST_F(HealthCheckManagerTest, runOnDemandHealthCheckTest)
     config.preload();
     Singleton::Consume<Config::I_Config>::from(config)->loadConfiguration(ss);
 
+    UpdatesProcessEvent(
+        UpdatesProcessResult::FAILED,
+        UpdatesConfigType::MANIFEST,
+        UpdatesFailureReason::DOWNLOAD_FILE,
+        "manifest.json",
+        "File not found"
+    ).notify();
+
     stringstream is;
     is << "{}";
     health_check_server->performRestCall(is);
 
     string expected_status =
         "{\n"
-        "    \"allComponentsHealthCheckReplies\": {\n"
-        "        \"Test\": {\n"
-        "            \"status\": \"Healthy\",\n"
-        "            \"extendedStatus\": {\n"
-        "                \"city\": \"Tel-Aviv\",\n"
-        "                \"team\": \"Hapoel\"\n"
-        "            }\n"
+        "    \"Orchestration\": {\n"
+        "        \"status\": \"Unhealthy\",\n"
+        "        \"extendedStatus\": {\n"
+        "            \"Manifest\": \"Failed to download the file manifest.json. Error: File not found\"\n"
         "        }\n"
         "    }\n"
         "}";

components/security_apps/orchestration/hybrid_mode_telemetry.cc

@@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &)
 {
     auto shell_cmd = Singleton::Consume<I_ShellCmd>::by<OrchestrationComp>();
     auto maybe_cmd_output = shell_cmd->getExecOutput(
-        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count"
+        getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count",
+        1000,
+        false
     );
 
     // get wd process restart count

components/security_apps/orchestration/include/declarative_policy_utils.h

@@ -79,8 +79,8 @@ public:
     ) override;
     std::string getUpdate(CheckUpdateRequest &request) override;
     bool shouldApplyPolicy() override;
-    void turnOffApplyPolicyFlag() override;
-    void turnOnApplyPolicyFlag() override;
+    void turnOffApplyLocalPolicyFlag() override;
+    void turnOnApplyLocalPolicyFlag() override;
 
     std::string getCurrPolicy() override { return curr_policy; }
 
@@ -94,7 +94,7 @@ private:
     std::string curr_version;
     std::string curr_policy;
     std::string curr_checksum;
-    bool should_apply_policy;
+    bool should_apply_local_policy;
 };
 
 #endif // __DECLARATIVE_POLICY_UTILS_H__