sync code

2025-11-16 09:21:54 +03:00 · 2025-08-08 11:06:28 +00:00
parent dd19bf6158
commit da20943c09
145 changed files with 4157 additions and 1016 deletions
--- a/core/CMakeLists.txt
+++ b/core/CMakeLists.txt
@@ -32,6 +32,7 @@ add_subdirectory(compression)
 add_subdirectory(attachments)
 add_subdirectory(report_messaging)
 add_subdirectory(env_details)
+add_subdirectory(curl_http_client)

 add_library(ngen_core SHARED ".")
 target_link_libraries(
--- a/core/agent_core_utilities/agent_core_utilities.cc
+++ b/core/agent_core_utilities/agent_core_utilities.cc
@@ -539,6 +539,14 @@ trim(string str)
    return removeLeadingWhitespaces(removeTrailingWhitespaces(str));
 }

+string
+toLower(string str)
+{
+    transform(str.begin(), str.end(), str.begin(), ::tolower);
+    return str;
+}
+
+
 } // namespace Strings

 } // namespace NGEN
--- a/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc
+++ b/core/agent_core_utilities/agent_core_utilities_ut/agent_core_utilities_ut.cc
@@ -197,6 +197,12 @@ TEST_F(AgentCoreUtilUT, trimTest)
    EXPECT_EQ(NGEN::Strings::trim(str_with_leading_and_trailing_whitespace), "str_with_whitespace");
 }

+TEST_F(AgentCoreUtilUT, toLowerTest)
+{
+    string str = "ThIS Is A 123 TEsT StRiNG";
+    EXPECT_EQ(NGEN::Strings::toLower(str), "this is a 123 test string");
+}
+
 TEST_F(AgentCoreUtilUT, resolveFullPathTest)
 {
    string working_dir = cptestFnameInExeDir("");
@@ -208,3 +214,43 @@ TEST_F(AgentCoreUtilUT, resolveFullPathTest)
    EXPECT_EQ(full_path, working_dir + "test.txt");
    ASSERT_TRUE(NGEN::Filesystem::deleteFile(working_dir + "test.txt"));
 }
+
+TEST_F(AgentCoreUtilUT, regexReplaceTest)
+{
+    struct TestCase {
+        std::string input;
+        std::string expected;
+    };
+
+    std::vector<TestCase> test_cases = {
+        {"my?invalid//:filename*test.txt", "my_invalid_filename_test.txt"},
+        {"hello///world", "hello_world"},
+        {"file@@name..txt", "file_name..txt"},
+        {"file--name", "file--name"},
+        {"some@@@file!!name.txt", "some_file_name.txt"},
+        {"https://some_file_name.txt", "https_some_file_name.txt"},
+        {"spaces in filename.txt", "spaces_in_filename.txt"},
+        {"trailing-dash-", "trailing-dash-"},
+        {"trailing.dot.", "trailing.dot."},
+        {"file name with (parens).txt", "file_name_with_parens_.txt"},
+        {"$pecial#Chars&here.txt", "_pecial_Chars_here.txt"},
+        {"___leading_underscores", "___leading_underscores"},
+        {"<<<<weird>>>filename", "_weird_filename"},
+        {"double..dots...txt", "double..dots...txt"},
+        {"a:b|c*d?e<f>g/h.txt", "a_b_c_d_e_f_g_h.txt"},
+        {"/leading/slash", "_leading_slash"},
+        {"back\\slash\\file", "back_slash_file"},
+        {"file.with..multiple.dots.txt", "file.with..multiple.dots.txt"},
+        {"CAPITAL&LETTERS^HERE", "CAPITAL_LETTERS_HERE"},
+        {"123_456-789.ok", "123_456-789.ok"},
+        {"__", "__"},
+        {"*.*", "_._"}
+    };
+
+    boost::regex regex("[^\\w.-]+"); // Matches one or more non-word, non-dot, non-hyphen characters
+
+    for (const auto& testCase : test_cases) {
+        std::string replaced = NGEN::Regex::regexReplace(__FILE__, __LINE__, testCase.input, regex, "_");
+        EXPECT_EQ(replaced, testCase.expected);
+    }
+}
--- a/core/agent_details/agent_details.cc
+++ b/core/agent_details/agent_details.cc
@@ -40,13 +40,19 @@ void
 AgentDetails::init()
 {
    registerMachineType();
-    loadAccessToken();
-    Singleton::Consume<I_MainLoop>::by<AgentDetails>()->addRecurringRoutine(
-        I_MainLoop::RoutineType::System,
-        chrono::seconds(60),
-        [this] () { loadAccessToken(); },
-        "Load access token"
-    );
+    bool load_access_token =
+        getConfigurationWithDefault<bool>(true, "Agent details", "Load Access Token");
+
+    if (load_access_token) {
+        loadAccessToken();
+        Singleton::Consume<I_MainLoop>::by<AgentDetails>()->addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::seconds(60),
+            [this] () { loadAccessToken(); },
+            "Load access token"
+        );
+    }
+
    proxies = {
        {ProxyProtocol::HTTP, ProxyData()},
        {ProxyProtocol::HTTPS, ProxyData()}
@@ -277,6 +283,7 @@ AgentDetails::preload()
 {
    registerExpectedConfiguration<string>("orchestration", "Agent details path");
    registerExpectedConfiguration<string>("Agent details", "File path");
+    registerExpectedConfiguration<bool>("Agent details", "Load Access Token");
    registerConfigLoadCb([this] () { readAgentDetails(); });
 }

--- a/core/buffers/buffer.cc
+++ b/core/buffers/buffer.cc
@@ -394,7 +394,7 @@ Buffer::isEqualLowerCase(const Buffer &buf) const
 {
    if (len != buf.size()) return false;
    for (uint i = 0; i < len; i++) {
-        if (tolower((*this)[i]) != buf[i]) return false;
+        if (tolower((*this)[i]) != tolower(buf[i])) return false;
    }
    return true;
 }
--- a/core/config/config.cc
+++ b/core/config/config.cc
@@ -179,6 +179,8 @@ private:
    bool
    sendOrchestatorConfMsg(int env_listening_port)
    {
+        dbgTrace(D_CONFIG) << "Sending configuration to orchestrator, listening port: " << env_listening_port;
+
        registerExpectedConfigUpdates config_updates;

        config_updates.service_name = executable_name;
@@ -204,6 +206,7 @@ private:
        service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
        service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
        service_config_req_md.setSuspension(false);
+        service_config_req_md.setShouldSendAccessToken(false);
        auto service_config_status = messaging->sendSyncMessage(
            HTTPMethod::POST,
            "/set-nano-service-config",
@@ -212,10 +215,17 @@ private:
            service_config_req_md
        );
        if (!service_config_status.ok()) {
+            dbgWarning(D_CONFIG)
+                << "Could not send configuration to orchestrator 7777, error: "
+                << service_config_status.getErr().getBody()
+                << ", error-code: "
+                << static_cast<int>(service_config_status.getErr().getHTTPStatusCode());
+
            MessageMetadata secondary_port_req_md("127.0.0.1", 7778);
            secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
            secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
            secondary_port_req_md.setSuspension(false);
+            secondary_port_req_md.setShouldSendAccessToken(false);
            service_config_status = messaging->sendSyncMessage(
                HTTPMethod::POST,
                "/set-nano-service-config",
@@ -224,8 +234,25 @@ private:
                secondary_port_req_md
            );
        }
+        
+        if (!service_config_status.ok()) {
+            dbgWarning(D_CONFIG)
+                << "Could not send configuration to orchestrator 7778, error: "
+                << service_config_status.getErr().getBody()
+                << ", error-code: "
+                << static_cast<int>(service_config_status.getErr().getHTTPStatusCode());

-        return service_config_status.ok() && config_updates.status.get();
+            return false;
+        }
+
+        if (is_force_reload && config_updates.status.get()) {
+            dbgWarning(D_CONFIG) << "Reloading configuration due to force-reload config";
+            if (reloadConfiguration("", false, 0) == I_Config::AsyncLoadConfigStatus::Success) is_force_reload = false;
+        }
+
+        dbgTrace(D_CONFIG) << "Configuration successfully sent to orchestrator";
+
+        return config_updates.status.get() && !is_force_reload;
    }

    void
@@ -254,6 +281,7 @@ private:
        service_config_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
        service_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
        service_config_req_md.setSuspension(false);
+        service_config_req_md.setShouldSendAccessToken(false);
        bool service_config_status = messaging->sendSyncMessageWithoutResponse(
            HTTPMethod::POST,
            "/set-reconf-status",
@@ -262,10 +290,12 @@ private:
            service_config_req_md
        );
        if (!service_config_status) {
+            dbgWarning(D_CONFIG) << "Could not send reconf-status to orchestrator 7777, retrying on 7778";
            MessageMetadata secondary_port_req_md("127.0.0.1", 7778);
            secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::ONE_TIME_CONN);
            secondary_port_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN);
            secondary_port_req_md.setSuspension(false);
+            secondary_port_req_md.setShouldSendAccessToken(false);
            service_config_status = messaging->sendSyncMessageWithoutResponse(
                HTTPMethod::POST,
                "/set-reconf-status",
@@ -275,9 +305,10 @@ private:
            );
        }
        if (!service_config_status) {
-            dbgWarning(D_CONFIG) << "Unsuccessful attempt to send configuration reload status";
+            dbgWarning(D_CONFIG) << "Could not send reconf-status to orchestrator 7778";
            return false;
        }
+
        return true;
    }

@@ -303,6 +334,7 @@ private:
    vector<ConfigCb> configuration_abort_cbs;

    bool is_continuous_report = false;
+    bool is_force_reload = getenv("FORCE_CONFIG_RELOAD") && string(getenv("FORCE_CONFIG_RELOAD")) == "TRUE";
    const string default_tenant_id = "";
    const string default_profile_id = "";
    string executable_name = "";
@@ -342,14 +374,19 @@ ConfigComponent::Impl::init()
    tenant_manager = Singleton::Consume<I_TenantManager>::by<ConfigComponent>();

    if (!Singleton::exists<I_MainLoop>()) return;
-    auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();

-    mainloop->addOneTimeRoutine(
-        I_MainLoop::RoutineType::System,
-        [this] () { periodicRegistrationRefresh(); },
-        "Configuration update registration",
-        false
-    );
+    bool periodic_registration_refresh =
+        getConfigurationWithDefault<bool>(true, "Config Component", "Periodic Registration Refresh");
+
+    if (periodic_registration_refresh) {
+        auto mainloop = Singleton::Consume<I_MainLoop>::by<ConfigComponent>();
+        mainloop->addOneTimeRoutine(
+            I_MainLoop::RoutineType::System,
+            [this] () { periodicRegistrationRefresh(); },
+            "Configuration update registration",
+            false
+        );
+    }
 }

 static bool
@@ -893,8 +930,9 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
            dbgTrace(D_CONFIG) << "Could not open configuration file. Path: " << file.first;
        }
    }
-
+    env->registerValue<bool>("Is Async Config Load", is_async);
    bool res = loadConfiguration(archives, is_async);
+    env->unregisterKey<bool>("Is Async Config Load");
    if (res) env->registerValue<string>("Current Policy Version", version);
    return res;
 }
@@ -976,6 +1014,7 @@ ConfigComponent::preload()
 {
    registerExpectedConfiguration<string>("Config Component", "configuration path");
    registerExpectedConfiguration<uint>("Config Component", "Refresh config update registration time interval");
+    registerExpectedConfiguration<bool>("Config Component", "Periodic Registration Refresh");
    registerExpectedResource<bool>("Config Component", "Config Load Test");
    registerExpectedSetting<AgentProfileSettings>("agentSettings");
    pimpl->preload();
--- a/core/include/attachments/compression_utils.h
+++ b/core/include/attachments/compression_utils.h
@@ -45,7 +45,8 @@ typedef enum CompressionType
 {
    NO_COMPRESSION,
    GZIP,
-    ZLIB
+    ZLIB,
+    BROTLI
 } CompressionType;

 typedef struct CompressionResult
--- a/core/include/attachments/nginx_attachment_common.h
+++ b/core/include/attachments/nginx_attachment_common.h
@@ -146,7 +146,8 @@ typedef enum ngx_http_cp_verdict
    TRAFFIC_VERDICT_INJECT,
    TRAFFIC_VERDICT_IRRELEVANT,
    TRAFFIC_VERDICT_RECONF,
-    TRAFFIC_VERDICT_WAIT
+    TRAFFIC_VERDICT_WAIT,
+    LIMIT_RESPONSE_HEADERS
 } ngx_http_cp_verdict_e;

 #ifdef __cplusplus
@@ -190,6 +191,8 @@ typedef enum ngx_http_meta_data
    PARSED_HOST_DATA,
    PARSED_URI_SIZE,
    PARSED_URI_DATA,
+    WAF_TAG_SIZE,
+    WAF_TAG_DATA,

    META_DATA_COUNT
 } ngx_http_meta_data_e;
--- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h
+++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h
@@ -29,6 +29,7 @@ namespace Intelligence {

 class Invalidation;
 class Response;
+class TimeRangeInvalidations;

 } // namespace Intelligence

@@ -39,7 +40,8 @@ public:
    virtual bool isIntelligenceHealthy() const = 0;
    virtual Maybe<uint> registerInvalidation(
        const Intelligence::Invalidation &invalidation,
-        const std::function<void(const Intelligence::Invalidation &)> &callback
+        const std::function<void(const Intelligence::Invalidation &)> &callback,
+        const std::string &AgentId = ""
    ) = 0;
    virtual void unregisterInvalidation(uint id) = 0;
    virtual Maybe<Intelligence::Response>
@@ -59,6 +61,10 @@ public:
        const MessageMetadata &req_md
    ) const = 0;

+    virtual Maybe<std::vector<Intelligence::Invalidation>> getInvalidations(
+        Intelligence::TimeRangeInvalidations request
+    ) const = 0;
+
    template<typename Data>
    Maybe<std::vector<AssetReply<Data>>>
    queryIntelligence(
--- a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h
+++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h
@@ -70,6 +70,31 @@ private:
    std::vector<SerializableAssetSource<UserSerializableReplyAttr>> sources;
 };

+class ExternalSourceError
+{
+public:
+    ExternalSourceError() {}
+
+    const std::string & getSourceID() const { return source_id; }
+    const std::string & getSourceName() const { return source_name; }
+    uint getStatusCode() const { return status_code; }
+    const std::string & getErrorMessage() const { return error_message; }
+
+    void setSourceID(const std::string &id) { source_id = id; }
+    void setSourceName(const std::string &name) { source_name = name; }
+    void setStatusCode(uint code) { status_code = code; }
+    void setErrorMessage(const std::string &message) { error_message = message; }
+
+    template<class Archive>
+    void serialize(Archive &ar);
+
+private:
+    std::string source_id = "";
+    std::string source_name = "";
+    uint status_code = 0;
+    std::string error_message = "";
+};
+
 class IntelligenceQueryResponse
 {
 public:
@@ -83,6 +108,7 @@ public:
    Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; }
    const std::string & getCursor() const { return cursor; }
    uint getAmountOfAssets() const { return total_num_assets; }
+    const std::vector<ExternalSourceError> & getExternalSourcesErrorStatus() const;
    bool isValidInBulk() const { return !partial_fail_in_bulk; }
    void setFailInBulk() { partial_fail_in_bulk = true; }

@@ -91,6 +117,7 @@ private:
    uint total_num_assets = 0;
    std::string cursor = "";
    bool partial_fail_in_bulk = false;
+    std::vector<ExternalSourceError> external_sources_errors;
 };

 template <typename UserSerializableReplyAttr>
--- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h
+++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h
@@ -21,6 +21,7 @@
 #include <maybe_res.h>
 #include "asset_reply.h"
 #include "bulk_query_response_v2.h"
+#include "intelligence_invalidation.h"

 USE_DEBUG_FLAG(D_INTELLIGENCE);

@@ -39,6 +40,11 @@ public:
    Maybe<void> load();
    Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
    const std::string getCursor() const { return single_response.getCursor(); }
+    const std::vector<ExternalSourceError> & getExternalSourcesErrorStatus() const
+    {
+        return single_response.getExternalSourcesErrorStatus();
+    }
+
    void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; }
    template <typename UserSerializableReplyAttr>
    IntelligenceQueryResponseT<UserSerializableReplyAttr> getSerializableResponse() const
@@ -98,10 +104,14 @@ public:
        return bulk_data;
    }

+    const std::vector<Invalidation>& getInvalidations() const { return invalidations; }
+    Maybe<void> loadInvalidations();
+
 private:
    std::string json_response;
    std::vector<IntelligenceQueryResponse> responses;
    IntelligenceQueryResponse single_response;
+    std::vector<Invalidation> invalidations;
    size_t size = 0;
    bool is_bulk = false;
 };
--- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
+++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
@@ -34,7 +34,8 @@ public:
        const std::string &key,
        const std::string &value,
        bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
    );

    QueryRequest(
@@ -42,7 +43,8 @@ public:
        const std::string &key,
        const int64_t &value,
        bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
    );

    QueryRequest(
@@ -50,7 +52,8 @@ public:
        const std::string &key,
        const std::vector<std::string> &value,
        bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
    );

    void saveToJson(cereal::JSONOutputArchive &ar) const;
@@ -115,6 +118,7 @@ public:
 private:
    uint assets_limit = default_assets_limit;
    bool full_response = false;
+    bool external_sources_error_status = false;
    Maybe<ObjectType> object_type = genError("uninitialized");
    Maybe<RequestCursor> cursor = genError("Cursor not initialized");
    SerializableQueryFilter query;
--- a/core/include/services_sdk/interfaces/messaging/interface_impl.h
+++ b/core/include/services_sdk/interfaces/messaging/interface_impl.h
@@ -19,8 +19,12 @@

 USE_DEBUG_FLAG(D_MESSAGING);

-MessageMetadata::MessageMetadata()
+MessageMetadata::MessageMetadata(bool immediate_tracing)
 {
+    if (immediate_tracing && Singleton::exists<I_Environment>()) {
+        insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+    }
+
    if (!Singleton::exists<I_AgentDetails>() || !Singleton::exists<I_ProxyConfiguration>()) return;
    auto i_agent_details = Singleton::Consume<I_AgentDetails>::by<I_Messaging>();
    auto i_proxy_configuration = Singleton::Consume<I_ProxyConfiguration>::by<I_Messaging>();
@@ -137,6 +141,8 @@ I_Messaging::sendAsyncMessage(
        return;
    }

+    dbgTrace(D_MESSAGING) << "Sending async message. URI: " << uri << ", Body: " << req_body.unpack();
+
    sendAsyncMessage(
        method,
        uri,
--- a/core/include/services_sdk/interfaces/messaging/messaging_enums.h
+++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h
@@ -30,6 +30,7 @@ enum class MessageConnectionConfig
    UNSECURE_CONN,
    ONE_TIME_CONN,
    IGNORE_SSL_VALIDATION,
+    ONE_TIME_FOG_CONN, // used for learning mechanism - one time connection sent by dedicated thread

    COUNT
 };
--- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h
+++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h
@@ -9,6 +9,7 @@
 #include "singleton.h"
 #include "i_agent_details.h"
 #include "i_time_get.h"
+#include "i_environment.h"

 class MessageProxySettings
 {
@@ -55,14 +56,27 @@ private:
    uint16_t proxy_port = 0;
 };

-class MessageMetadata : Singleton::Consume<I_TimeGet>
+class MessageMetadata : Singleton::Consume<I_TimeGet>, Singleton::Consume<I_Environment>
 {
 public:
-    inline MessageMetadata();
+    inline MessageMetadata(bool immediate_tracing = false);

-    MessageMetadata(const std::string &_host_name, uint16_t _port_num, bool _buffer = false, bool _fog = false) :
-        host_name(_host_name), port_num(_port_num), should_buffer(_buffer), is_to_fog(_fog)
-    {}
+    MessageMetadata(
+        const std::string &_host_name,
+        uint16_t _port_num,
+        bool _buffer = false,
+        bool _fog = false,
+        bool immediate_tracing = false
+    ) :
+        host_name(_host_name),
+        port_num(_port_num),
+        should_buffer(_buffer),
+        is_to_fog(_fog)
+    {
+        if (immediate_tracing && Singleton::exists<I_Environment>()) {
+            insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+        }
+    }

    MessageMetadata(
        std::string _host_name,
@@ -70,7 +84,8 @@ public:
        Flags<MessageConnectionConfig> _conn_flags,
        bool _should_buffer = false,
        bool _is_to_fog = false,
-        bool _should_suspend = true
+        bool _should_suspend = true,
+        bool immediate_tracing = false
    ) :
        host_name(_host_name),
        port_num(_port_num),
@@ -79,7 +94,11 @@ public:
        is_to_fog(_is_to_fog),
        should_send_access_token(true),
        should_suspend(_should_suspend)
-    {}
+    {
+        if (immediate_tracing && Singleton::exists<I_Environment>()) {
+            insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+        }
+    }

    const bool &
    shouldSendAccessToken() const
@@ -135,6 +154,14 @@ public:
        return headers;
    }

+    Maybe<std::string>
+    getTraceId() const
+    {
+        auto trace_id = headers.find("X-Trace-Id");
+        if (trace_id != headers.end()) return trace_id->second;
+        return genError("Trace ID not found");
+    }
+
    std::string
    getCaPath() const
    {
--- a/core/include/services_sdk/interfaces/mock/mock_intelligence.h
+++ b/core/include/services_sdk/interfaces/mock/mock_intelligence.h
@@ -16,16 +16,31 @@ operator<<(std::ostream &os, const Intelligence::Invalidation &)
    return os;
 }

+std::ostream &
+operator<<(std::ostream &os, const std::vector<Intelligence::Invalidation> &)
+{
+    return os;
+}
+
 class MockIntelligence : public Singleton::Provide<I_Intelligence_IS_V2>::From<MockProvider<I_Intelligence_IS_V2>>
 {
 public:
    using InvalidationCb = std::function<void(const Intelligence::Invalidation &)>;
    using Invalidation = Intelligence::Invalidation;
    using Response = Intelligence::Response;
+    using TimeRangeInvalidations = Intelligence::TimeRangeInvalidations;

    MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
+    MOCK_CONST_METHOD1(getInvalidations, Maybe<std::vector<Invalidation>>(TimeRangeInvalidations));
    MOCK_CONST_METHOD0(isIntelligenceHealthy, bool(void));
-    MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
+    MOCK_METHOD3(
+        registerInvalidation,
+        Maybe<uint>(
+            const Invalidation &invalidation,
+            const InvalidationCb &callback,
+            const std::string &AgentId
+        )
+    );
    MOCK_METHOD1(unregisterInvalidation, void(uint id));
    MOCK_CONST_METHOD5(
        getResponse,
--- a/core/include/services_sdk/resources/config/config_loader.h
+++ b/core/include/services_sdk/resources/config/config_loader.h
@@ -55,7 +55,7 @@ public:
    std::pair<std::unique_ptr<EnvironmentEvaluator<bool>>, TypeWrapper>
    getLoaderConfig()
    {
-        return std::move(std::make_pair(std::move(context), TypeWrapper(value)));
+        return std::make_pair(std::move(context), TypeWrapper(value));
    }

 private:
--- a/core/include/services_sdk/resources/debug_flags.h
+++ b/core/include/services_sdk/resources/debug_flags.h
@@ -35,6 +35,7 @@ DEFINE_FLAG(D_INFRA, D_ALL)
    DEFINE_FLAG(D_TENANT_MANAGER, D_INFRA)
    DEFINE_FLAG(D_MONITORING, D_INFRA)
    DEFINE_FLAG(D_SERVICE_HEALTH_STATUS, D_INFRA)
+    DEFINE_FLAG(D_LOGGING, D_INFRA)
    DEFINE_FLAG(D_REPORT, D_INFRA)
        DEFINE_FLAG(D_REPORT_BULK, D_REPORT)
    DEFINE_FLAG(D_TRACE, D_INFRA)
@@ -48,6 +49,9 @@ DEFINE_FLAG(D_INFRA, D_ALL)
            DEFINE_FLAG(D_CONNECTION, D_MESSAGING)
            DEFINE_FLAG(D_MESSAGING_BUFFER, D_MESSAGING)
            DEFINE_FLAG(D_HTTP_REQUEST, D_MESSAGING)
+            DEFINE_FLAG(D_TRACE_ID, D_MESSAGING)
+    DEFINE_FLAG(D_MEMORY, D_INFRA)
+        DEFINE_FLAG(D_WAAP_MEMORY, D_MEMORY)

 DEFINE_FLAG(D_COMPONENT, D_ALL)
    DEFINE_FLAG(D_PRELOAD, D_COMPONENT)
@@ -72,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
            DEFINE_FLAG(D_WAAP_SAMPLE_SCAN, D_WAAP)
            DEFINE_FLAG(D_WAAP_ASSET_STATE, D_WAAP)
            DEFINE_FLAG(D_WAAP_CONFIDENCE_CALCULATOR, D_WAAP)
+            DEFINE_FLAG(D_WAAP_SERIALIZE, D_WAAP)
            DEFINE_FLAG(D_WAAP_REPUTATION, D_WAAP)
            DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
            DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
@@ -153,6 +158,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
    DEFINE_FLAG(D_SDWAN, D_COMPONENT)
        DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN)
        DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN)
+            DEFINE_FLAG(D_SDWAN_DATA_SENDER, D_SDWAN_DATA)
        DEFINE_FLAG(D_SDWAN_FEATURE_FLAG, D_SDWAN)
        DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN)
        DEFINE_FLAG(D_SDWAN_API, D_SDWAN)
@@ -196,6 +202,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
    DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
    DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
    DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
+    DEFINE_FLAG(D_AIGUARD, D_COMPONENT)

 DEFINE_FLAG(D_FLOW, D_ALL)
    DEFINE_FLAG(D_DROP, D_FLOW)
--- a/core/include/services_sdk/resources/generic_metric.h
+++ b/core/include/services_sdk/resources/generic_metric.h
@@ -116,7 +116,7 @@ private:
    friend class MetricCalc;
    void addCalc(MetricCalc *calc);

-    std::vector<PrometheusData> getPromMetricsData();
+    std::vector<PrometheusData> getPromMetricsData(const std::vector<MetricCalc*> *allowed_calcs = nullptr);

    void handleMetricStreamSending();
    void generateLog();
--- a/core/include/services_sdk/resources/intelligence_invalidation.h
+++ b/core/include/services_sdk/resources/intelligence_invalidation.h
@@ -19,6 +19,9 @@
 #include <string>
 #include <set>
 #include <cereal/archives/json.hpp>
+#include "rest.h"
+#include "messaging/messaging_enums.h"
+#include "messaging/messaging_metadata.h"

 #include "maybe_res.h"
 #include "enum_array.h"
@@ -32,6 +35,56 @@ enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
 enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED };
 enum class InvalidationType { ADD, DELETE, UPDATE };

+static const std::map<std::string, ObjectType> stringToObjectTypeMap = {
+    {"asset", ObjectType::ASSET},
+    {"zone", ObjectType::ZONE},
+    {"policyPackage", ObjectType::POLICY_PACKAGE},
+    {"configuration", ObjectType::CONFIGURATION},
+    {"session", ObjectType::SESSION},
+    {"shortLived", ObjectType::SHORTLIVED}
+};
+
+static const std::map<std::string, InvalidationType> stringToInvalidationTypeMap = {
+    {"add", InvalidationType::ADD},
+    {"delete", InvalidationType::DELETE},
+    {"update", InvalidationType::UPDATE}
+};
+
+class TimeRangeInvalidations
+{
+public:
+    TimeRangeInvalidations(uint64_t start_time, uint64_t end_time) : time_range{start_time, end_time} {}
+
+    Maybe<std::string> genJson() const
+    {
+        try {
+            std::stringstream out;
+            {
+                cereal::JSONOutputArchive out_ar(out);
+                out_ar(cereal::make_nvp("timeRange", time_range));
+            }
+            return out.str();
+        } catch (const std::exception &e) {
+            return genError("Failed to generate JSON for TimeRangeInvalidations. Error: " + std::string(e.what()));
+        }
+    }
+
+private:
+    struct TimeRange
+    {
+        uint64_t start;
+        uint64_t end;
+
+        template <class Archive>
+        void serialize(Archive &ar)
+        {
+            ar(cereal::make_nvp("start", start), cereal::make_nvp("end", end));
+        }
+    };
+
+    TimeRange time_range;
+};
+
 class StrAttributes
 {
 public:
@@ -101,6 +154,7 @@ private:
 class Invalidation
 {
 public:
+    Invalidation();
    Invalidation(const std::string &class_value);

    Invalidation & setClassifier(ClassifierType type, const std::string &val);
@@ -113,14 +167,18 @@ public:
    std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
    std::vector<StrAttributes> getMainAttributes() const { return main_attributes; }
    std::vector<IpAttributes> getAttributes() const { return attributes; }
-    const Maybe<std::string, void> & getSourceId() const { return source_id; }
-    const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
-    const Maybe<InvalidationType, void> & getInvalidationType() const { return invalidation_type; }
-    Maybe<std::string, void> getRegistrationID() const;
+    const Maybe<std::string> & getSourceId() const { return source_id; }
+    const Maybe<ObjectType> & getObjectType() const { return object_type; }
+    const Maybe<InvalidationType> & getInvalidationType() const { return invalidation_type; }
+    Maybe<std::string> getRegistrationID() const;

    bool report(I_Intelligence_IS_V2 *interface) const;

-    Maybe<uint> startListening(I_Intelligence_IS_V2 *interface, const std::function<void(const Invalidation &)> &cb);
+    Maybe<uint> startListening(
+        I_Intelligence_IS_V2 *interface,
+        const std::function<void(const Invalidation &)> &cb,
+        const std::string &AgentId = ""
+    );
    void stopListening(I_Intelligence_IS_V2 *interface);

    Maybe<std::string> genJson() const;
@@ -128,6 +186,7 @@ public:
    bool isLegalInvalidation() const;

    bool matches(const Invalidation &other) const;
+    void serialize(cereal::JSONInputArchive &ar);

 private:
    bool attr_matches(const std::vector<StrAttributes> &current, const std::vector<StrAttributes> &other) const;
@@ -136,11 +195,11 @@ private:
    EnumArray<ClassifierType, std::string, 6> classifiers;
    std::vector<StrAttributes> main_attributes;
    std::vector<IpAttributes> attributes;
-    Maybe<std::string, void> source_id;
-    Maybe<ObjectType, void> object_type;
-    Maybe<InvalidationType, void> invalidation_type;
-    Maybe<uint, void> listening_id;
-    Maybe<std::string, void> registration_id;
+    Maybe<std::string> source_id;
+    Maybe<ObjectType> object_type;
+    Maybe<InvalidationType> invalidation_type;
+    Maybe<uint> listening_id;
+    Maybe<std::string> registration_id;
 };

 } // namespace Intelligence
--- a/core/include/services_sdk/resources/log_generator.h
+++ b/core/include/services_sdk/resources/log_generator.h
@@ -31,16 +31,6 @@ class LogGen
    Singleton::Consume<I_Environment>
 {
 public:
-    template <typename Trigger, typename ...Args>
-    LogGen(
-        const Trigger &trigger,
-        const std::string &title,
-        Args ...args)
-            :
-        LogGen(trigger(title, std::forward<Args>(args)...))
-    {
-    }
-
    template <typename ...Args>
    LogGen(
        const std::string &title,
--- a/core/include/services_sdk/resources/report/report_bulks.h
+++ b/core/include/services_sdk/resources/report/report_bulks.h
@@ -51,6 +51,9 @@ public:
            dbgDebug(D_REPORT_BULK) << "Adding a new bulk to queue";
            bulks.push(LogBulkRest(bulk_size));;
        }
+        dbgTrace(D_REPORT_BULK)
+        << "Adding report to bulk, for asset: "
+        << (report.getStringData("assetName").ok() ? *report.getStringData("assetName") : "unknown");
        bulks.back().push(report);
        ++elem_in_quque;
    }
--- a/core/include/services_sdk/resources/report/report_enums.h
+++ b/core/include/services_sdk/resources/report/report_enums.h
@@ -71,6 +71,7 @@ enum class Tags {
    DEPLOYMENT_DOCKER,
    WEB_SERVER_SWAG,
    WEB_SERVER_NGINX_UNIFIED,
+    AIGUARD,

    COUNT
 };
--- a/core/include/services_sdk/utilities/agent_core_utilities.h
+++ b/core/include/services_sdk/utilities/agent_core_utilities.h
@@ -85,6 +85,7 @@ namespace Strings
 std::string removeTrailingWhitespaces(std::string str);
 std::string removeLeadingWhitespaces(std::string str);
 std::string trim(std::string str);
+std::string toLower(std::string str);

 } // namespace Strings

--- a/core/intelligence_is_v2/asset_replay.cc
+++ b/core/intelligence_is_v2/asset_replay.cc
@@ -15,6 +15,18 @@

 using namespace std;

+template<class Archive>
+void
+ExternalSourceError::serialize(Archive &ar)
+{
+    ar(
+        cereal::make_nvp("sourceID", source_id),
+        cereal::make_nvp("sourceName", source_name),
+        cereal::make_nvp("statusCode", status_code),
+        cereal::make_nvp("errorMessage", error_message)
+    );
+}
+
 void
 IntelligenceQueryResponse::loadFromJson(const std::string &json_response)
 {
@@ -35,4 +47,14 @@ IntelligenceQueryResponse::serialize(Archive &ar)
    try {
        ar(cereal::make_nvp("cursor", cursor));
    } catch (...) {}
+
+    try {
+        ar(cereal::make_nvp("externalSourcesErrorStatus", external_sources_errors));
+    } catch (...) {}
+}
+
+const std::vector<ExternalSourceError> &
+IntelligenceQueryResponse::getExternalSourcesErrorStatus() const
+{
+    return external_sources_errors;
 }
--- a/core/intelligence_is_v2/intelligence_comp_v2.cc
+++ b/core/intelligence_is_v2/intelligence_comp_v2.cc
@@ -36,6 +36,7 @@ static const string query_uri = "/api/v2/intelligence/assets/query";
 static const string queries_uri = "/api/v2/intelligence/assets/queries";
 static const string fog_health_uri = "/access-manager/health/live";
 static const string intelligence_health_uri = "/show-health";
+static const string time_range_invalidation_uri = "/api/v2/intelligence/invalidation/get";

 class I_InvalidationCallBack
 {
@@ -83,6 +84,12 @@ public:
        first = false;
    }

+    void
+    setAgentId(const string &_agent_id)
+    {
+        agent_id = _agent_id;
+    }
+
    RestCall
    genJson() const
    {
@@ -90,7 +97,7 @@ public:

        res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", \"callbackType\": \"invalidation\", ";
        auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
-        res << "\"name\": \"" << details->getAgentId() << "\", ";
+        res << "\"name\": \"" << (agent_id.empty() ? details->getAgentId() : agent_id) << "\", ";
        auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
        res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
        res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, ";
@@ -106,6 +113,7 @@ public:
 private:
    bool first = true;
    stringstream stream;
+    string agent_id = "";
 };

 class InvalidationCallBack : Singleton::Provide<I_InvalidationCallBack>::SelfInterface
@@ -137,10 +145,12 @@ public:
    bool empty() const { return callbacks.empty(); }

    InvalidationRegistration::RestCall
-    getRegistration() const
+    getRegistration(const string& agent_id) const
    {
        InvalidationRegistration registration;

+        registration.setAgentId(agent_id);
+
        for (auto &registed_invalidation : callbacks) {
            registration.addInvalidation(registed_invalidation.second.first);
        }
@@ -394,11 +404,24 @@ public:
        return sendIntelligence(invalidation).ok();
    }

+    Maybe<vector<Invalidation>>
+    getInvalidations(TimeRangeInvalidations request) const override
+    {
+        auto res = sendIntelligence(request);
+        if (res.ok()) return res.unpack().getInvalidations();
+        return res.passErr();
+    }
+
    Maybe<uint>
-    registerInvalidation(const Invalidation &invalidation, const function<void(const Invalidation &)> &cb) override
+    registerInvalidation(
+        const Invalidation &invalidation,
+        const function<void(const Invalidation &)> &cb,
+        const string &AgentId
+    ) override
    {
        if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation");
        auto res = invalidations.emplace(invalidation, cb);
+        agent_id = AgentId;
        sendRecurringInvalidationRegistration();
        return res;
    }
@@ -504,7 +527,7 @@ private:
        const string &server,
        const string &port_setting,
        const bool should_send_access_token = false
-) const
+    ) const
    {
        auto port = getSetting<uint>("intelligence", port_setting);
        if (!port.ok()) {
@@ -548,6 +571,16 @@ private:
        return load_status.passErr();
    }

+    Maybe<Response>
+    createResponse(const string &response_body) const
+    {
+        Response response(response_body, 0, false);
+        auto load_status = response.loadInvalidations();
+        if (load_status.ok()) return response;
+        dbgWarning(D_INTELLIGENCE) << "Could not create intelligence response.";
+        return load_status.passErr();
+    }
+
    Maybe<Response>
    sendIntelligenceRequestImpl(const Invalidation &invalidation, const MessageMetadata &local_req_md) const
    {
@@ -642,6 +675,39 @@ private:
        return createResponse(req_data->getBody(), query_request);
    }

+    Maybe<Response>
+    sendIntelligenceRequestImpl(
+        const TimeRangeInvalidations &request,
+        const MessageMetadata &global_req_md) const
+    {
+        dbgFlow(D_INTELLIGENCE) << "Sending time range invalidations request";
+
+        auto json_body = request.genJson();
+        if (!json_body.ok()) return json_body.passErr();
+
+        auto req_data = message->sendSyncMessage(
+                HTTPMethod::POST,
+                time_range_invalidation_uri,
+                *json_body,
+                MessageCategory::INTELLIGENCE,
+                global_req_md
+        );
+        if (!req_data.ok()) {
+            dbgWarning(D_INTELLIGENCE)
+                << "Could not send time range invalidations request. "
+                << req_data.getErr().getBody()
+                << " "
+                << req_data.getErr().toString();
+            return genError("Could not send time range invalidations request.");
+        }
+        if (req_data->getHTTPStatusCode() != HTTPStatusCode::HTTP_OK) {
+            dbgWarning(D_INTELLIGENCE) << "Invalid intelligence response: " << req_data->toString();
+            return genError(req_data->toString());
+        }
+
+        return createResponse(req_data->getBody());
+    }
+
    map<string, string>
    getHTTPHeaders() const
    {
@@ -651,7 +717,7 @@ private:
        if (tenant == "") tenant = "Global";
        headers["X-Tenant-Id"] = tenant;
        auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
-        auto agent = details->getAgentId() + ":" + to_string(rest->getListeningPort());
+        auto agent = (agent_id.empty() ? details->getAgentId() : agent_id) + ":" + to_string(rest->getListeningPort());
        headers["X-Source-Id"] = agent;

        return headers;
@@ -662,7 +728,7 @@ private:
    {
        if (invalidations.empty()) return;

-        sendLocalIntelligenceToLocalServer(invalidations.getRegistration());
+        sendLocalIntelligenceToLocalServer(invalidations.getRegistration(agent_id));
    }

    Maybe<Response>
@@ -678,6 +744,7 @@ private:
    InvalidationCallBack         invalidations;
    I_Messaging               *message = nullptr;
    I_MainLoop                   *mainloop = nullptr;
+    string agent_id = "";
 };

 IntelligenceComponentV2::IntelligenceComponentV2()
--- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
+++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
@@ -1,8 +1,13 @@
 #include <sstream>
-namespace Intelligence { class Response; }
+#include <vector>
+namespace Intelligence
+{
+    class Response;
+    class Invalidation;
+}
 std::ostream & operator<<(std::ostream &os, const Intelligence::Response &);
-
-#include "intelligence_comp_v2.h"
+std::ostream & operator<<(std::ostream &os, const Intelligence::Invalidation &);
+std::ostream & operator<<(std::ostream &os, const std::vector<Intelligence::Invalidation> &);

 #include "config.h"
 #include "config_component.h"
@@ -15,6 +20,8 @@ std::ostream & operator<<(std::ostream &os, const Intelligence::Response &);
 #include "mock/mock_agent_details.h"
 #include "read_attribute_v2.h"
 #include "singleton.h"
+#include "intelligence_comp_v2.h"
+#include "intelligence_invalidation.h"

 using namespace std;
 using namespace testing;
@@ -1397,3 +1404,68 @@ TEST_F(IntelligenceComponentTestV2, localIntelligenceHealthy)

    EXPECT_TRUE(intell->isIntelligenceHealthy());
 }
+
+TEST_F(IntelligenceComponentTestV2, getInvalidations)
+{
+    Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
+    Debug::setNewDefaultStdout(&cout);
+
+    HTTPResponse response_str(
+        HTTPStatusCode::HTTP_OK,
+        string(
+            "{ \"invalidations\": [ { "
+            "\"class\": \"aaa\", "
+            "\"category\": \"bbb\", "
+            "\"family\": \"ccc\", "
+            "\"group\": \"ddd\", "
+            "\"order\": \"eee\", "
+            "\"kind\": \"fff\", "
+            "\"objectType\": \"asset\", "
+            "\"sourceId\": \"id\", "
+            "\"mainAttributes\": [ { \"attr\": \"2\" } ], "
+            "\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.2\" ], "
+            "\"ipv4AddressesRange\": [ { \"max\": \"1.1.1.5\", \"min\": \"1.1.1.1\" } ] } ]"
+            " } ] }"
+        )
+    );
+
+    Intelligence::TimeRangeInvalidations time_range_invalidations(1622505600, 1622592000);
+    auto json_body = time_range_invalidations.genJson();
+
+    EXPECT_TRUE(json_body.ok());
+    EXPECT_CALL(mock_rest, getListeningPort()).WillOnce(Return(8888));
+    EXPECT_CALL(
+        messaging_mock,
+        sendSyncMessage(
+            HTTPMethod::POST,
+            "/api/v2/intelligence/invalidation/get",
+            json_body.unpack(),
+            MessageCategory::INTELLIGENCE,
+            _
+        )
+    ).WillOnce(Return(response_str));
+
+    auto main_attr = Intelligence::StrAttributes()
+        .addStringAttr("attr", "2");
+    Intelligence::IpAddressRange range("1.1.1.1", "1.1.1.5");
+    Intelligence::IpAttributes attributes = Intelligence::IpAttributes()
+        .addIpv4Addresses("1.1.1.2")
+        .addIpv4AddressRanges(range);
+
+    auto invalidation = Intelligence::Invalidation("aaa")
+        .addMainAttr(main_attr)
+        .addAttr(attributes)
+        .setSourceId("id")
+        .setClassifier(Intelligence::ClassifierType::FAMILY, "ccc")
+        .setClassifier(Intelligence::ClassifierType::CATEGORY, "bbb")
+        .setClassifier(Intelligence::ClassifierType::GROUP, "ddd")
+        .setClassifier(Intelligence::ClassifierType::ORDER, "eee")
+        .setClassifier(Intelligence::ClassifierType::KIND, "fff")
+        .setObjectType(Intelligence::ObjectType::ASSET);
+
+    auto res = intell->getInvalidations(time_range_invalidations);
+
+    EXPECT_TRUE(res.ok());
+    EXPECT_TRUE(res.unpack().front().matches(invalidation));
+}
--- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
+++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
@@ -63,6 +63,28 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequestProxied) {
    EXPECT_EQ(*query.genJson(), expected);
 }

+TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequestExternalError) {
+    QueryRequest request(Condition::EQUALS, "phase", "testing", true, AttributeKeyType::MAIN, true);
+    vector<QueryRequest> requests = {request};
+    Intelligence::IntelligenceRequest query(requests, true, false, true, MessageMetadata("", 0));
+
+    std::string expected = "{\n"
+        "    \"queryTypes\": {\n"
+        "        \"proxyToCloud\": true\n"
+        "    },\n"
+        "    \"limit\": 20,\n"
+        "    \"fullResponse\": true,\n"
+        "    \"externalSourcesErrorStatus\": true,\n"
+        "    \"query\": {\n"
+        "        \"operator\": \"equals\",\n"
+        "        \"key\": \"mainAttributes.phase\",\n"
+        "        \"value\": \"testing\"\n"
+        "    }\n"
+        "}";
+
+    EXPECT_EQ(*query.genJson(), expected);
+}
+
 TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
    QueryRequest request(Condition::EQUALS, "phase", "testing", true);
    vector<QueryRequest> requests = {request};
--- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
+++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
@@ -87,13 +87,14 @@ TEST(QueryRequestTestV2, QueryTest)
    range2.push_back("224.11.10.16");
    range2.push_back("224.11.10.31");

-    QueryRequest request3(Condition::RANGE, "ipv4AddressesRange", range1, true);
+    QueryRequest request3(Condition::RANGE, "ipv4AddressesRange", range1, true, AttributeKeyType::MAIN, true);
    request3.addCondition(Condition::RANGE, "ipv4AddressesRange", range2);

    string output_json3=
    "{\n"
    "    \"limit\": 20,\n"
    "    \"fullResponse\": true,\n"
+    "    \"externalSourcesErrorStatus\": true,\n"
    "    \"query\": {\n"
    "        \"operator\": \"and\",\n"
    "        \"operands\": [\n"
--- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc
+++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc
@@ -145,7 +145,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
        "  ],\n"
        "  \"status\": \"done\",\n"
        "  \"totalNumAssets\": 2,\n"
-        "  \"cursor\": \"start\"\n"
+        "  \"cursor\": \"start\",\n"
+        "  \"externalSourcesErrorStatus\": []\n"
        "}\n"
    );

@@ -159,6 +160,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
    EXPECT_EQ(obj2.getAmountOfAssets(), 2u);
    EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE);
    EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE);
+    EXPECT_TRUE(obj.getExternalSourcesErrorStatus().empty());
+    EXPECT_TRUE(obj2.getExternalSourcesErrorStatus().empty());
    EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1u);
    EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip");
    EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1u);
@@ -217,6 +220,86 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
    EXPECT_EQ(asset_sources_it->getData1().toString(), "Max");
 }

+TEST(QueryResponseTestV2, ExternalSourcesErrorStatusTestV2)
+{
+    DataString data;
+    IntelligenceQueryResponseT<stringData1> obj;
+    string string_attribute(
+        "{\n"
+        "  \"assetCollections\": [\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-ip\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"some-group-id\",\n"
+        "      \"name\": \"[1.1.1.1]\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"ip\",\n"
+        "      \"group\": \"\",\n"
+        "      \"order\": \"\",\n"
+        "      \"kind\": \"\",\n"
+        "      \"mainAttributes\": {\n"
+        "        \"team\": \"hapoel\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
+        "          \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"color\": \"red\",\n"
+        "            \"user\": \"Omry\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    }\n"
+        "  ],\n"
+        "  \"status\": \"done\",\n"
+        "  \"totalNumAssets\": 1,\n"
+        "  \"cursor\": \"start\",\n"
+        "  \"externalSourcesErrorStatus\": [\n"
+        "    {\n"
+        "      \"sourceID\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+        "      \"sourceName\": \"test-source-1\",\n"
+        "      \"statusCode\": 500,\n"
+        "      \"errorMessage\": \"Internal server error\"\n"
+        "    },\n"
+        "    {\n"
+        "      \"sourceID\": \"a1b2c3d4-5678-9abc-def0-123456789abc\",\n"
+        "      \"sourceName\": \"test-source-2\",\n"
+        "      \"statusCode\": 404,\n"
+        "      \"errorMessage\": \"Not found\"\n"
+        "    }\n"
+        "  ]\n"
+        "}\n"
+    );
+
+    stringstream ss(string_attribute);
+    {
+        cereal::JSONInputArchive ar(ss);
+        obj.serialize(ar);
+    }
+
+    const auto& errors = obj.getExternalSourcesErrorStatus();
+    EXPECT_EQ(errors.size(), 2u);
+
+    EXPECT_EQ(errors[0].getSourceID(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00");
+    EXPECT_EQ(errors[0].getSourceName(), "test-source-1");
+    EXPECT_EQ(errors[0].getStatusCode(), 500u);
+    EXPECT_EQ(errors[0].getErrorMessage(), "Internal server error");
+
+    EXPECT_EQ(errors[1].getSourceID(), "a1b2c3d4-5678-9abc-def0-123456789abc");
+    EXPECT_EQ(errors[1].getSourceName(), "test-source-2");
+    EXPECT_EQ(errors[1].getStatusCode(), 404u);
+    EXPECT_EQ(errors[1].getErrorMessage(), "Not found");
+}
+
 TEST(QueryResponseTestV2, MainAttributesTestV2)
 {
    DataString data;
--- a/core/intelligence_is_v2/intelligence_response.cc
+++ b/core/intelligence_is_v2/intelligence_response.cc
@@ -58,6 +58,20 @@ Response::load()
    return {};
 }

+Maybe<void>
+Response::loadInvalidations()
+{
+    try {
+        stringstream in;
+        in.str(json_response);
+        cereal::JSONInputArchive in_ar(in);
+        in_ar(cereal::make_nvp("invalidations", invalidations));
+    } catch(const std::exception &e) {
+        return genError("Load invalidations failed. Error: " + string(e.what()));
+    }
+    return {};
+}
+
 Intelligence_IS_V2::ResponseStatus
 Response::getResponseStatus() const
 {
--- a/core/intelligence_is_v2/invalidation.cc
+++ b/core/intelligence_is_v2/invalidation.cc
@@ -25,13 +25,22 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
 using namespace Intelligence;
 using namespace std;

+Invalidation::Invalidation()
+        :
+    source_id(genError<string>("")),
+    object_type(genError<string>("")),
+    invalidation_type(genError<string>("")),
+    listening_id(genError<string>("")),
+    registration_id(genError<string>(""))
+{}
+
 Invalidation::Invalidation(const string &class_value)
        :
-    source_id(genError<void>()),
-    object_type(genError<void>()),
-    invalidation_type(genError<void>()),
-    listening_id(genError<void>()),
-    registration_id(genError<void>())
+    source_id(genError<string>("")),
+    object_type(genError<string>("")),
+    invalidation_type(genError<string>("")),
+    listening_id(genError<string>("")),
+    registration_id(genError<string>(""))
 {
    setClassifier(ClassifierType::CLASS, class_value);
 }
@@ -72,10 +81,14 @@ Invalidation::report(I_Intelligence_IS_V2 *interface) const
 }

 Maybe<uint>
-Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function<void(const Invalidation &)> &cb)
+Invalidation::startListening(
+    I_Intelligence_IS_V2 *interface,
+    const function<void(const Invalidation &)> &cb,
+    const string &AgentId
+)
 {
    registration_id = to_string(boost::uuids::random_generator()());
-    auto res = interface->registerInvalidation(*this, cb);
+    auto res = interface->registerInvalidation(*this, cb, AgentId);
    if (res.ok()) listening_id = *res;
    return res;
 }
@@ -243,6 +256,117 @@ Invalidation::matches(const Invalidation &other) const
    return true;
 }

+void
+Invalidation::serialize(cereal::JSONInputArchive &ar)
+{
+    std::string class_ = "";
+    std::string category = "";
+    std::string family = "";
+    std::string group = "";
+    std::string order = "";
+    std::string kind = "";
+    std::string object_type_;
+    std::string invalidation_type_;
+    std::string source_id_;
+    uint listening_id_;
+    std::string registration_id_;
+    
+    try {
+        ar(cereal::make_nvp("class", class_));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("category", category));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("family", family));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("group", group));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("order", order));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("kind", kind));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("mainAttributes", main_attributes));
+        ar(cereal::make_nvp("attributes", attributes));
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("objectType", object_type_));
+        auto it = stringToObjectTypeMap.find(object_type_);
+        if (it != stringToObjectTypeMap.end()) {
+            object_type = it->second;
+        } else {
+            throw std::invalid_argument("Invalid string for ObjectType: " + object_type_);
+        }
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("sourceId", source_id_));
+        source_id = source_id_;
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("invalidationRegistrationId", registration_id_));
+        registration_id = registration_id_;
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("invalidationType", invalidation_type_));
+        auto it = stringToInvalidationTypeMap.find(invalidation_type_);
+        if (it != stringToInvalidationTypeMap.end()) {
+            invalidation_type = it->second;
+        } else {
+            throw std::invalid_argument("Invalid string for InvalidationType: " + invalidation_type_);
+        }
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
+        ar(cereal::make_nvp("listeningId", listening_id_));
+        listening_id = listening_id_;
+    } catch (const cereal::Exception &e) {
+        dbgError(D_INTELLIGENCE) << e.what();
+    }
+
+    classifiers[ClassifierType::CLASS] = class_;
+    classifiers[ClassifierType::CATEGORY] = category;
+    classifiers[ClassifierType::FAMILY] = family;
+    classifiers[ClassifierType::GROUP] = group;
+    classifiers[ClassifierType::ORDER] = order;
+    classifiers[ClassifierType::KIND] = kind;
+}
+
 Invalidation &
 Invalidation::addAttr(const IpAttributes &attr)
 {
@@ -257,7 +381,7 @@ Invalidation::addMainAttr(const StrAttributes &attr)
    return *this;
 }

-Maybe<string, void>
+Maybe<string>
 Invalidation::getRegistrationID() const{
    return registration_id;
 }
--- a/core/intelligence_is_v2/query_request_v2.cc
+++ b/core/intelligence_is_v2/query_request_v2.cc
@@ -30,11 +30,13 @@ QueryRequest::QueryRequest(
    const string &key,
    const string &value,
    bool full_reponse,
-    AttributeKeyType attribute_type
+    AttributeKeyType attribute_type,
+    bool _external_sources_error_status
 ) {
    query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
    assets_limit = default_assets_limit;
    full_response = full_reponse;
+    external_sources_error_status = _external_sources_error_status;
 }

 QueryRequest::QueryRequest(
@@ -42,11 +44,13 @@ QueryRequest::QueryRequest(
    const string &key,
    const int64_t &value,
    bool full_reponse,
-    AttributeKeyType attribute_type
+    AttributeKeyType attribute_type,
+    bool _external_sources_error_status
 ) {
    query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
    assets_limit = default_assets_limit;
    full_response = full_reponse;
+    external_sources_error_status = _external_sources_error_status;
 }

 QueryRequest::QueryRequest(
@@ -54,11 +58,13 @@ QueryRequest::QueryRequest(
    const string &key,
    const vector<string> &value,
    bool full_reponse,
-    AttributeKeyType attribute_type
+    AttributeKeyType attribute_type,
+    bool _external_sources_error_status
 ) {
    query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
    assets_limit = default_assets_limit;
    full_response = full_reponse;
+    external_sources_error_status = _external_sources_error_status;
 }

 Maybe<string>
@@ -83,10 +89,15 @@ QueryRequest::save(cereal::JSONOutputArchive &ar) const
 {
    ar(
        cereal::make_nvp("limit", assets_limit),
-        cereal::make_nvp("fullResponse", full_response),
-        cereal::make_nvp("query", query)
+        cereal::make_nvp("fullResponse", full_response)
    );

+    if (external_sources_error_status) {
+        ar(cereal::make_nvp("externalSourcesErrorStatus", external_sources_error_status));
+    }
+
+    ar(cereal::make_nvp("query", query));
+
    auto objTypeString = convertObjectTypeToString();
    if (objTypeString.ok()) {
        ar(cereal::make_nvp("objectType", *objTypeString));
@@ -236,6 +247,7 @@ QueryRequest::calcQueryRequestOperator(const QueryRequest &other_query, const Op
    res_req_query.query = res_query_filter;
    res_req_query.assets_limit = this->assets_limit;
    res_req_query.full_response = this->full_response;
+    res_req_query.external_sources_error_status = this->external_sources_error_status;
    res_req_query.cursor = this->cursor;
    res_req_query.requested_attributes = this->requested_attributes;
    res_req_query.query_types = this->query_types;
--- a/core/logging/fog_stream.cc
+++ b/core/logging/fog_stream.cc
@@ -37,6 +37,7 @@ FogStream::sendLog(const Report &log)
    ScopedContext ctx;
    ctx.registerValue<bool>("Obfuscate log field", true);

+    dbgTrace(D_REPORT) << "Sending log to fog";
    LogRest rest(log);
    i_msg->sendAsyncMessage(HTTPMethod::POST, fog_log_uri, rest, MessageCategory::LOG);
 }
--- a/core/logging/log_generator.cc
+++ b/core/logging/log_generator.cc
@@ -14,15 +14,23 @@
 #include "log_generator.h"

 using namespace std;
+USE_DEBUG_FLAG(D_LOGGING);

 extern const string unnamed_service;

 LogGen::~LogGen()
 {
    try {
-        if (send_log) Singleton::Consume<I_Logging>::by<LogGen>()->sendLog(log);
+        if (send_log) {
+            dbgTrace(D_LOGGING) << "sending log";
+            Singleton::Consume<I_Logging>::by<LogGen>()->sendLog(log);
+        } else {
+            dbgTrace(D_LOGGING) << "not sending log";
+        }
    } catch (...) {
+        dbgWarning(D_LOGGING) << "Failed to send log";
    }
+
 }

 LogGen &
--- a/core/logging/log_streams.h
+++ b/core/logging/log_streams.h
@@ -157,7 +157,7 @@ protected:

 private:
    void init();
-    void sendLog(const std::vector<char> &data);
+    void sendLogData(const std::vector<char> &data);
    I_MainLoop::RoutineID log_send_routine = -1;
 };

--- a/core/logging/logging.cc
+++ b/core/logging/logging.cc
@@ -162,11 +162,17 @@ public:
    sendLog(const Report &log) override
    {
        if (getConf("agent.config.log.useBulkMode", "Enable bulk of logs", true)) {
+            dbgTrace(D_REPORT) << "Adding log to bulk";
            reports.setBulkSize(getConfigurationWithDefault<uint>(100, "Logging", "Sent log bulk size"));
            reports.push(log);
            if (reports.sizeQueue() >= 4) {
                auto persistence_only = getConf("agent.config.log.skip.enable", "Enable Log skipping", true);
-                sendBufferedLogsImpl(false, persistence_only);
+                dbgTrace(D_REPORT)
+                << "Sending buffered logs from queue size: "
+                << reports.sizeQueue()
+                << ", persistence_only: "
+                << persistence_only;
+            sendBufferedLogsImpl(false, persistence_only);
            }
        } else {
            LogEventLogsSent(true).notify();
@@ -230,10 +236,22 @@ private:
        for (auto &iter : local_streams) {
            LogBulkRest sub_batch;
            for (const auto &log : batch) {
-                if (log.isStreamActive(iter.first)) sub_batch.push(log);
+                if (log.isStreamActive(iter.first)) {
+                    dbgTrace(D_REPORT)
+                    << "stream: "
+                    << (int) iter.first
+                    << " is active. adding log to sub-batch";
+                    sub_batch.push(log);
+                }
            }

            if (sub_batch.size()) {
+                dbgTrace(D_REPORT)
+                << "Sending log to stream: "
+                << (int) iter.first
+                << ", batch size: "
+                << sub_batch.size();
+
                iter.second->sendLog(sub_batch, persistence_only);
                if (is_async) i_mainloop->yield();
            }
--- a/core/logging/syslog_stream.cc
+++ b/core/logging/syslog_stream.cc
@@ -55,13 +55,13 @@ SyslogStream::sendLog(const Report &log)
    vector<char> data(syslog_report.begin(), syslog_report.end());
    log_send_routine = mainloop->addOneTimeRoutine(
        I_MainLoop::RoutineType::Offline,
-        [this, data] () { sendLog(data); },
+        [this, data] () { sendLogData(data); },
        "Logging Syslog stream messaging"
    );
 }

 void
-SyslogStream::sendLog(const vector<char> &data)
+SyslogStream::sendLogData(const vector<char> &data)
 {
    dbgTrace(D_REPORT) << "Sending Syslog log." << " Max logs per send: " << max_logs_per_send;
    sendLogWithQueue(data);
--- a/core/mainloop/mainloop.cc
+++ b/core/mainloop/mainloop.cc
@@ -98,12 +98,17 @@ public:
    init()
    {
        fini_signal_flag = false;
-        addOneTimeRoutine(
-            RoutineType::Offline,
-            [this](){ reportStartupEvent(); },
-            "Nano service startup report",
-            false
-        );
+        bool report_startup_event =
+            getConfigurationWithDefault<bool>(true, "Mainloop", "Report Startup Event");
+
+        if (report_startup_event) {
+            addOneTimeRoutine(
+                RoutineType::Offline,
+                [this](){ reportStartupEvent(); },
+                "Nano service startup report",
+                false
+            );
+        }

        metric_report_interval = chrono::seconds(
            getConfigurationWithDefault<uint>(600, "Mainloop", "metric reporting interval")
@@ -319,7 +324,12 @@ MainloopComponent::Impl::run()
                ) {
                    dbgWarning(D_MAINLOOP)
                        << "Routine execution exceeded run time. Routine name: "
-                        << curr_iter->second.getRoutineName();
+                        << curr_iter->second.getRoutineName()
+                        << ", time slice: "
+                        << time_slice_to_use
+                        << ", exceeded time: "
+                        << (getTimer()->getMonotonicTime() - stop_time).count()
+                        << " microseconds";
                }
            }

@@ -642,5 +652,6 @@ MainloopComponent::preload()
    registerExpectedConfiguration<int>("Mainloop", "Busy routine time slice");
    registerExpectedConfiguration<uint>("Mainloop", "metric reporting interval");
    registerExpectedConfiguration<uint>("Mainloop", "Exceed Warning");
+    registerExpectedConfiguration<bool>("Mainloop", "Report Startup Event");
    registerConfigLoadCb([&] () { pimpl->reloadConfigurationCb(); });
 }
--- a/core/messaging/connection/connection.cc
+++ b/core/messaging/connection/connection.cc
@@ -21,6 +21,9 @@
 #include <fstream>
 #include <sstream>
 #include <string>
+#include <thread>
+#include <future>
+#include <atomic>

 #include "config.h"
 #include "http_request.h"
@@ -73,6 +76,7 @@ enum class ConnectionFlags
 {
    UNSECURE,
    ONE_TIME,
+    ASYNC_ONE_TIME,
    IGNORE_SSL_VALIDATION,
    PROXY,

@@ -87,6 +91,9 @@ public:
        auto metadata_flags = metadata.getConnectionFlags();
        if (metadata_flags.isSet(MessageConnectionConfig::UNSECURE_CONN)) flags.setFlag(ConnectionFlags::UNSECURE);
        if (metadata_flags.isSet(MessageConnectionConfig::ONE_TIME_CONN)) flags.setFlag(ConnectionFlags::ONE_TIME);
+        if (metadata_flags.isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) {
+            flags.setFlag(ConnectionFlags::ASYNC_ONE_TIME);
+        }
        if (metadata_flags.isSet(MessageConnectionConfig::IGNORE_SSL_VALIDATION)) {
            flags.setFlag(ConnectionFlags::IGNORE_SSL_VALIDATION);
        }
@@ -99,7 +106,6 @@ public:

        sni_hostname = metadata.getSniHostName();
        dn_host_name = metadata.getDnHostName();
-
    }

    void
@@ -511,12 +517,29 @@ private:
        BioConnectionStatus bio_connect = tryToBioConnect(full_address);
        uint attempts_count = 0;
        auto conn_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
+        auto maybe_is_orch = Singleton::Consume<I_Environment>::by<Messaging>()->get<bool>("Is Orchestrator");
+        auto is_orch = maybe_is_orch.ok() && *maybe_is_orch;
        while (i_time->getMonotonicTime() < conn_end_time && bio_connect == BioConnectionStatus::SHOULD_RETRY) {
+            if (is_orch) { // fixing code for orch case due to stability concerns - should be removed
+                if (isBioSocketReady()) {
+                    bio_connect = tryToBioConnect(full_address);
+                } else {
+                    i_mainloop->yield((attempts_count % 10) == 0);
+                }
+                continue;
+            }
+
            if (isBioSocketReady()) {
                bio_connect = tryToBioConnect(full_address);
-            } else {
-                i_mainloop->yield((attempts_count % 10) == 0);
            }
+
+            dbgTrace(D_CONNECTION)
+                << "Connection to: "
+                << full_address
+                << " should retry. number of made attempts: "
+                << ++attempts_count;
+
+            i_mainloop->yield(true);
        }

        if (bio_connect == BioConnectionStatus::SUCCESS) {
@@ -553,7 +576,8 @@ private:
        int data_sent_len = BIO_write(bio.get(), curr_data_to_send, data_left_to_send);

        if (data_sent_len >= 0) {
-            dbgTrace(D_CONNECTION) << "Sent " << data_sent_len << " bytes, out of: " << data_left_to_send << " bytes.";
+            dbgTrace(D_CONNECTION) << "Sent " << data_sent_len << " bytes, out of: " << data_left_to_send
+                << " bytes (total remaining: " << data_left_to_send - data_sent_len << " bytes).";
            return data_sent_len;
        }

@@ -637,13 +661,60 @@ private:
        I_TimeGet *i_time = Singleton::Consume<I_TimeGet>::by<Messaging>();
        auto sending_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
        size_t data_left_to_send = request.length();
+        atomic<bool> cancel_task{false};

-        while (data_left_to_send > 0) {
-            if (i_time->getMonotonicTime() > sending_end_time) return genError(sending_timeout);
-            auto send_size = sendData(request, data_left_to_send);
-            if (!send_size.ok()) return send_size.passErr();
-            data_left_to_send -= *send_size;
-            i_mainloop->yield(*send_size == 0);
+        // Use smaller chunks for one-time connections with yielding between chunks
+        if (flags.isSet(ConnectionFlags::ASYNC_ONE_TIME)) {
+            // Launch async task to send full request only
+            // note do not add debug logs inside the async task
+            // as it will cause a crash
+            auto task = async(launch::async, [this, request, &cancel_task]() -> Maybe<void, HTTPResponse> {
+                size_t remaining = request.length();
+                while (remaining > 0) {
+                    if (cancel_task.load()) {
+                        return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, "Async send task was canceled"));
+                    }
+                    auto sent = sendData(request, remaining);
+                    if (!sent.ok()) return genError(sent.getErr());
+                    remaining -= *sent;
+                    if (*sent == 0) {
+                        // sleep for 25ms to avoid busy waiting
+                        this_thread::sleep_for(chrono::milliseconds(25));
+                    }
+                }
+                return Maybe<void, HTTPResponse>();
+            });
+
+            // Set a timeout of 1 minute for the task
+            auto timeout = chrono::minutes(1);
+            auto start_time = i_time->getMonotonicTime();
+
+            // poll and yield until send completes or timeout occurs
+            if (!task.valid()) {
+                return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN,
+                    "Async send future is not valid (no_state)"));
+            }
+            // Modify the loop to yield after setting cancel_task to true
+            while (task.wait_for(chrono::milliseconds(0)) != future_status::ready) {
+                if (i_time->getMonotonicTime() - start_time > timeout) {
+                    cancel_task.store(true); // Signal the task to cancel
+                    i_mainloop->yield(chrono::milliseconds(50)); // Yield for 50ms after canceling
+                    return genError(HTTPResponse(HTTPStatusCode::HTTP_UNKNOWN, "Async send task timed out"));
+                }
+                i_mainloop->yield(chrono::milliseconds(30)); // Yield for 30ms
+                dbgTrace(D_CONNECTION) << "Waiting for async send to complete...";
+            }
+            dbgDebug(D_CONNECTION) << "Async send completed.";
+            auto send_res = task.get();
+            if (!send_res.ok()) return send_res.passErr();
+        } else {
+            while (data_left_to_send > 0) {
+                if (i_time->getMonotonicTime() > sending_end_time) return genError(sending_timeout);
+                auto send_size = sendData(request, data_left_to_send);
+                if (!send_size.ok()) return send_size.passErr();
+                data_left_to_send -= *send_size;
+                i_mainloop->yield(*send_size == 0);
+            }
        }

        auto receiving_end_time = i_time->getMonotonicTime() + getConnectionTimeout();
@@ -660,11 +731,11 @@ private:
                return receieved.passErr();
            }
            auto response = http_parser.parseData(*receieved, is_connect);
+            i_mainloop->yield(receieved.unpack().empty());
            if (response.ok()) {
                dbgTrace(D_MESSAGING) << printOut(response.unpack().toString());
                return response.unpack();
            }
-            i_mainloop->yield(receieved.unpack().empty());
        }
        return genError(parsing_error);
    }
@@ -708,7 +779,6 @@ private:
    bool is_dual_auth = false;
    Maybe<string> sni_hostname = genError<string>("Uninitialized");
    Maybe<string> dn_host_name = genError<string>("Uninitialized");
-
 };

 Connection::Connection(const MessageConnectionKey &key, const MessageMetadata &metadata)
--- a/core/messaging/connection/connection_comp.cc
+++ b/core/messaging/connection/connection_comp.cc
@@ -97,7 +97,9 @@ private:
        if (!external_certificate.empty()) conn.setExternalCertificate(external_certificate);

        auto connected = conn.establishConnection();
-        persistent_connections.emplace(conn_key, conn);
+        if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) {
+            persistent_connections.emplace(conn_key, conn);
+        }

        if (!connected.ok()) {
            string connection_err = "Failed to establish connection. Error: " + connected.getErr();
@@ -140,8 +142,9 @@ private:
        }

        dbgTrace(D_CONNECTION) << "Connection over proxy established succssesfuly";
-
-        persistent_connections.emplace(conn_key, conn);
+        if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) {
+            persistent_connections.emplace(conn_key, conn);
+        }
        return conn;
    }

--- a/core/messaging/connection/connection_ut/connection_comp_ut.cc
+++ b/core/messaging/connection/connection_ut/connection_comp_ut.cc
@@ -264,3 +264,51 @@ TEST_F(TestConnectionComp, testEstablishNewProxyConnection)

    auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
 }
+
+TEST_F(TestConnectionComp, testSendRequestWithOneTimeFogConnection)
+{
+    Flags<MessageConnectionConfig> conn_flags;
+    conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN);
+    conn_flags.setFlag(MessageConnectionConfig::ONE_TIME_FOG_CONN);
+    MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags);
+
+    auto maybe_connection = i_conn->establishConnection(conn_metadata, MessageCategory::LOG);
+    ASSERT_TRUE(maybe_connection.ok());
+    auto conn = maybe_connection.unpack();
+
+    auto req = HTTPRequest::prepareRequest(conn, HTTPMethod::POST, "/test", conn_metadata.getHeaders(), "test-body");
+    ASSERT_TRUE(req.ok());
+
+    EXPECT_CALL(mock_mainloop, yield(A<std::chrono::microseconds>()))
+        .WillOnce(
+            InvokeWithoutArgs(
+                [&]() {
+                    cerr << "accepting socket" << endl;
+                    dummy_socket.acceptSocket();
+                    dummy_socket.writeToSocket("HTTP/1.1 200 OK\r\nContent-Length: 7\r\n\r\nmy-test");
+                }
+            )
+        ).WillRepeatedly(Return());
+
+    EXPECT_CALL(mock_timer, getMonotonicTime())
+        .WillRepeatedly(Invoke([]() { static int j = 0; return chrono::microseconds(++j * 10); }));
+
+    auto maybe_response = i_conn->sendRequest(conn, *req);
+    if (!maybe_response.ok()) {
+        cout << "Error: " << maybe_response.getErr().toString() << endl;
+    }
+    ASSERT_TRUE(maybe_response.ok());
+    EXPECT_EQ((*maybe_response).getBody(), "my-test");
+
+    string expected_msg =
+        "POST /test HTTP/1.1\r\n"
+        "Accept-Encoding: identity\r\n"
+        "Authorization: Bearer accesstoken\r\n"
+        "Connection: keep-alive\r\n"
+        "Content-Length: 9\r\n"
+        "Content-type: application/json\r\n"
+        "Host: 127.0.0.1\r\n"
+        "\r\n"
+        "test-body";
+    EXPECT_EQ(dummy_socket.readFromSocket(), expected_msg);
+}
--- a/core/messaging/messaging_comp/messaging_comp.cc
+++ b/core/messaging/messaging_comp/messaging_comp.cc
@@ -25,6 +25,7 @@
 using namespace std;

 USE_DEBUG_FLAG(D_MESSAGING);
+USE_DEBUG_FLAG(D_TRACE_ID);

 class FogConnectionChecker : public ServerRest
 {
@@ -95,13 +96,14 @@ isMessageToFog(const MessageMetadata message_metadata)
 Maybe<Connection>
 MessagingComp::getConnection(MessageCategory category, const MessageMetadata &metadata)
 {
-    auto persistant_conn = getPersistentConnection(metadata, category);
-    if (persistant_conn.ok()) {
-        dbgTrace(D_MESSAGING) << "Found a persistant connection";
-        return persistant_conn;
+    if (!metadata.getConnectionFlags().isSet(MessageConnectionConfig::ONE_TIME_FOG_CONN)) {
+        auto persistant_conn = getPersistentConnection(metadata, category);
+        if (persistant_conn.ok()) {
+            dbgTrace(D_MESSAGING) << "Found a persistant connection";
+            return persistant_conn;
+        }
+        dbgDebug(D_MESSAGING) << persistant_conn.getErr();
    }
-    dbgDebug(D_MESSAGING) << persistant_conn.getErr();
-
    auto maybe_conn = i_conn->establishConnection(metadata, category);
    if (!maybe_conn.ok()) {
        dbgWarning(D_MESSAGING) << maybe_conn.getErr();
@@ -131,6 +133,7 @@ MessagingComp::sendMessage(

    bool is_to_fog = isMessageToFog(message_metadata);
    auto metadata = message_metadata;
+
    if (is_to_fog) {
        if (method == HTTPMethod::GET && fog_get_requests_cache.doesKeyExists(uri)) {
            HTTPResponse res = fog_get_requests_cache.getEntry(uri);
@@ -141,7 +144,16 @@ MessagingComp::sendMessage(

        auto i_env = Singleton::Consume<I_Environment>::by<Messaging>();
        metadata.insertHeader("User-Agent", "Infinity Next (a7030abf93a4c13)");
-        metadata.insertHeaders(i_env->getCurrentHeadersMap());
+        if (!metadata.getTraceId().ok()) {
+            metadata.insertHeaders(i_env->getCurrentHeadersMap());
+        }
+
+        Maybe<string> trace_id = metadata.getTraceId();
+        if (trace_id.ok()) {
+            dbgTrace(D_TRACE_ID) << "Sending message to fog (trace ID: " << trace_id.unpack() << ")";
+        } else {
+            dbgTrace(D_TRACE_ID) << "Could not retrieve trace ID for fog message. Error: " << trace_id.getErr();
+        }
    }

    auto req = HTTPRequest::prepareRequest(
--- a/core/metric/generic_metric.cc
+++ b/core/metric/generic_metric.cc
@@ -31,6 +31,36 @@ MetricMetadata::DotName operator"" _dot(const char *str, size_t) { return Metric
 MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricMetadata::Units{str}; }
 MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; }

+static const set<string> default_metrics = {
+    "watchdogProcessStartupEventsSum",
+    "reservedNgenA",
+    "reservedNgenB",
+    "reservedNgenC"
+    "reservedNgenD"
+    "reservedNgenE",
+    "reservedNgenF",
+    "reservedNgenG"
+    "reservedNgenH",
+    "reservedNgenI",
+    "reservedNgenJ",
+    "numberOfProtectedAssetsSample",
+    "preventEngineMatchesSample",
+    "detectEngineMatchesSample",
+    "ignoreEngineMatchesSample",
+    "cpuMaxSample",
+    "cpuAvgSample",
+    "cpuSample",
+    "serviceVirtualMemorySizeMaxSample",
+    "serviceVirtualMemorySizeMinSample",
+    "serviceVirtualMemorySizeAvgSample",
+    "serviceRssMemorySizeMaxSample",
+    "serviceRssMemorySizeMinSample",
+    "serviceRssMemorySizeAvgSample",
+    "generalTotalMemorySizeMaxSample",
+    "generalTotalMemorySizeMinSample",
+    "generalTotalMemorySizeAvgSample"
+};
+
 // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage
 static ostream & operator<<(ostream &os, const CompressAndEncodeAIOPSMetrics &metrics)
 {
@@ -165,6 +195,9 @@ GenericMetric::init(
    const string &_asset_id
 )
 {
+
+    if (!getConfigurationWithDefault<bool>(true, "metric", "genericMetricInitEnable")) return;
+
    turnOnStream(Stream::FOG);
    turnOnStream(Stream::DEBUG);

@@ -263,7 +296,22 @@ GenericMetric::respond(const AllMetricEvent &event)
 vector<PrometheusData>
 GenericMetric::respond(const MetricScrapeEvent &)
 {
-    return getPromMetricsData();
+    bool enable_all_metrics = getProfileAgentSettingWithDefault<bool>(false, "enable_all_metrics");
+    if (enable_all_metrics) {
+        dbgTrace(D_METRICS) << "Sensitive metrics enabled, returning all metrics";
+        return getPromMetricsData();
+    }
+
+    vector<MetricCalc *> allowed_calcs;
+    for (auto &calc : prometheus_calcs) {
+        auto metric_calc_name = calc->getMetricDotName() != "" ? calc->getMetricDotName() : calc->getMetricName();
+        if (default_metrics.find(metric_calc_name) != default_metrics.end()) {
+            allowed_calcs.push_back(calc);
+        }
+    }
+    if (allowed_calcs.empty()) return {};
+
+    return getPromMetricsData(&allowed_calcs);
 }

 string GenericMetric::getListenerName() const { return metric_name; }
@@ -329,7 +377,7 @@ GenericMetric::generateLog()
 }

 vector<PrometheusData>
-GenericMetric::getPromMetricsData()
+GenericMetric::getPromMetricsData(const vector<MetricCalc*> *allowed_calcs)
 {
    vector<PrometheusData> all_metrics;
    bool enable_prometheus = false;
@@ -345,7 +393,8 @@ GenericMetric::getPromMetricsData()
    if (!enable_prometheus) return all_metrics;
    dbgTrace(D_METRICS) << "Get prometheus metrics";

-    for (auto &calc : prometheus_calcs) {
+    const vector<MetricCalc*> &calcs_to_use = allowed_calcs ? *allowed_calcs : prometheus_calcs;
+    for (auto &calc : calcs_to_use) {
        const auto &calc_prom_metrics = calc->getPrometheusMetrics(metric_name, asset_id);
        all_metrics.insert(all_metrics.end(), calc_prom_metrics.begin(), calc_prom_metrics.end());
        calc->reset();
@@ -416,5 +465,6 @@ GenericMetric::preload()
    registerExpectedConfiguration<bool>("metric", "debugMetricSendEnable");
    registerExpectedConfiguration<bool>("metric", "aiopsMetricSendEnable");
    registerExpectedConfiguration<bool>("metric", "fogMetricUri");
+    registerExpectedConfiguration<bool>("metric", "genericMetricInitEnable");
    registerExpectedConfiguration<string>("metric", "metricsOutputTmpFile");
 }
--- a/core/metric/metric_ut/metric_ut.cc
+++ b/core/metric/metric_ut/metric_ut.cc
@@ -542,7 +542,8 @@ TEST_F(MetricTest, getPromeathusMetric)
    metric_scraper.init();

    stringstream configuration;
-    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
+    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},";
+    configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n";

    EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));

@@ -634,7 +635,8 @@ TEST_F(MetricTest, getPromeathusMultiMap)
    metric_scraper.init();

    stringstream configuration;
-    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
+    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},";
+    configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n";

    EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));

@@ -699,7 +701,8 @@ TEST_F(MetricTest, getPromeathusTwoMetrics)
    metric_scraper.init();

    stringstream configuration;
-    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"}]}\n";
+    configuration << "{\"agentSettings\":[{\"key\":\"prometheus\",\"id\":\"id1\",\"value\":\"true\"},";
+    configuration << "{\"key\":\"enable_all_metrics\",\"id\":\"id2\",\"value\":\"true\"}]}\n";

    EXPECT_TRUE(Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration));

--- a/core/report/tag_and_enum_management.cc
+++ b/core/report/tag_and_enum_management.cc
@@ -115,7 +115,8 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
        {"APISIX Server", ReportIS::Tags::WEB_SERVER_APISIX},
        {"Docker Deployment", ReportIS::Tags::DEPLOYMENT_DOCKER},
        {"SWAG Server", ReportIS::Tags::WEB_SERVER_SWAG},
-        {"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED}
+        {"NGINX Unified Server", ReportIS::Tags::WEB_SERVER_NGINX_UNIFIED},
+        {"AI Guard", ReportIS::Tags::AIGUARD}
    };
    
    auto report_is_tag = strings_to_tags.find(tag);
@@ -326,7 +327,8 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
    "APISIX Server",
    "Docker Deployment",
    "SWAG Server",
-    "NGINX Unified Server"
+    "NGINX Unified Server",
+    "AI Guard"
 };

 EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {
--- a/core/rest/rest_server.cc
+++ b/core/rest/rest_server.cc
@@ -46,8 +46,10 @@ public:

    void startNewConnection() const;

-    bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
-    bool bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range);
+    bool setupIpv4ServerSocket(bool accept_get_from_external_ip);
+    bool setupIpv6ServerSocket(bool &finish_port_range);
+    bool createIpv4Socket();
+    bool createIpv6Socket();
    bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
    bool addGetCall(const string &uri, const function<string()> &cb) override;
    bool addWildcardGetCall(const string &uri, const function<string(const string &)> &callback);
@@ -74,40 +76,141 @@ private:
 };

 bool
-RestServer::Impl::bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range)
+RestServer::Impl::createIpv4Socket()
+{
+    fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fd == -1) {
+        dbgAssert(fd >= 0) << alert << "Failed to open a socket";
+    }
+    int socket_enable = 1;
+    if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
+        dbgWarning(D_API) << "Could not set the socket options";
+    }
+    dbgDebug(D_API) << "IPv4 socket opened successfully";
+    return true;
+}
+
+bool
+RestServer::Impl::createIpv6Socket()
+{
+    fd = socket(AF_INET6, SOCK_STREAM, 0);
+    if (fd == -1) {
+        return false;
+    }
+    int socket_enable = 1;
+    if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
+        dbgWarning(D_API) << "Could not set the socket options";
+    }
+    dbgDebug(D_API) << "IPv6 socket opened successfully";
+    int option = 0;
+
+    if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) {
+        dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option";
+    }
+    return true;
+}
+
+bool
+RestServer::Impl::setupIpv4ServerSocket(bool accept_get_from_external_ip)
 {
    dbgFlow(D_API) << "Binding IPv4 socket";
+    struct sockaddr_in addr;
+    bzero(&addr, sizeof(addr));
+    addr.sin_family = AF_INET;
+    if (accept_get_from_external_ip) {
+        addr.sin_addr.s_addr = htonl(INADDR_ANY);
+        dbgDebug(D_API) << "Socket listening on any address";
+    } else {
+        addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+        dbgDebug(D_API) << "Socket listening on local address";
+    }
+
+    bool create_socket = true;
    for (uint16_t port : port_range) {
+        if(create_socket) {
+            if (!createIpv4Socket()) {
+                dbgDebug(D_API) << "Failed creating Ipv4 socket!";
+                return false;
+            }
+            create_socket = false;
+        }
+
        addr.sin_port = htons(port);

-        if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) == 0) return true;
+        if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in)) != 0) {
+            if (errno == EADDRINUSE) {
+                dbgDebug(D_API) << "Port " << port << " is already in use";
+            } else {
+                dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+            }
+            continue;
+        }
+
+        if (listen(fd, listen_limit) == 0) {
+            listening_port = ntohs(addr.sin_port);
+            return true;
+        }

        if (errno == EADDRINUSE) {
-            dbgDebug(D_API) << "Port " << port << " is already in use";
+            dbgDebug(D_API) << "Another socket is already listening on the port: " << port;
        } else {
-            dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+            dbgDebug(D_API) << "Failed to listen to socket with error: " << strerror(errno);
        }
+
+        create_socket = true;
+        close(fd);
+        fd = -1;
    }

    return false;
 }

 bool
-RestServer::Impl::bindRestServerSocket(struct sockaddr_in6 &addr, vector<uint16_t> port_range)
+RestServer::Impl::setupIpv6ServerSocket(bool &finish_port_range)
 {
    dbgFlow(D_API) << "Binding IPv6 socket";
+    struct sockaddr_in6 addr6;
+    bzero(&addr6, sizeof(addr6));
+    addr6.sin6_family = AF_INET6;
+    addr6.sin6_addr = in6addr_any;
+    dbgDebug(D_API) << "Socket listening on any address";
+    bool create_socket = true;
    for (uint16_t port : port_range) {
-        addr.sin6_port = htons(port);
+        if(create_socket) {
+            if (!createIpv6Socket()) {
+                dbgDebug(D_API) << "Failed creating Ipv6 socket!";
+                return false;
+            }
+            create_socket = false;
+        }

-        if (bind(fd, (struct sockaddr *)&addr, sizeof(struct sockaddr_in6)) == 0) return true;
+        addr6.sin6_port = htons(port);
+
+        if (bind(fd, (struct sockaddr *)&addr6, sizeof(struct sockaddr_in6)) != 0) {
+            if (errno == EADDRINUSE) {
+                dbgDebug(D_API) << "Port " << port << " is already in use";
+            } else {
+                dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+            }
+            continue;
+        }
+
+        if (listen(fd, listen_limit) == 0) {
+            listening_port = ntohs(addr6.sin6_port);
+            return true;
+        }

        if (errno == EADDRINUSE) {
-            dbgDebug(D_API) << "Port " << port << " is already in use";
+            dbgDebug(D_API) << "Another socket is already listening on the port: " << port;
        } else {
-            dbgDebug(D_API) << "Failed to bind to port " << port << " with error: " << strerror(errno);
+            dbgDebug(D_API) << "Failed to listen to socket with error: " << strerror(errno);
        }
+        create_socket = true;
+        close(fd);
+        fd = -1;
    }

+    finish_port_range = true;
    return false;
 }

@@ -163,60 +266,28 @@ RestServer::Impl::init()
            }
        }

-        bool is_ipv6 = false;
+        bool finish_port_range = false;
+        bool failed_to_listen = false;
        if (accept_get_from_external_ip) {
-            is_ipv6 = true;
-            fd = socket(AF_INET6, SOCK_STREAM, 0);
+            while (!setupIpv6ServerSocket(finish_port_range) && finish_port_range) {
+                dbgWarning(D_API) << "Failed to bind to any of the (IPv6) ports in the port range";
+                failed_to_listen = true;
+                finish_port_range = false;
+                mainloop->yield(bind_retry_interval_msec);
+            }
        }
        if (fd == -1) {
-            fd = socket(AF_INET, SOCK_STREAM, 0);
-            is_ipv6 = false;
+            while (!setupIpv4ServerSocket(accept_get_from_external_ip)) {
+                dbgWarning(D_API) << "Failed to bind to any of the (IPv4) ports in the port range";
+                failed_to_listen = true;
+                mainloop->yield(bind_retry_interval_msec);
+            }
+        }
+        if (failed_to_listen) {
+            dbgWarning(D_API) << "Manage to listen on port:" << listening_port << " after failure";
        }
        dbgAssert(fd >= 0) << alert << "Failed to open a socket";

-        int socket_enable = 1;
-        if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &socket_enable, sizeof(int)) < 0) {
-            dbgWarning(D_API) << "Could not set the socket options";
-        }
-
-        if (is_ipv6) {
-            dbgDebug(D_API) << "IPv6 socket opened successfully";
-            int option = 0;
-            if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &option, sizeof(option)) < 0) {
-                dbgWarning(D_API) << "Could not set the IPV6_V6ONLY option";
-            }
-
-            struct sockaddr_in6 addr6;
-            bzero(&addr6, sizeof(addr6));
-            addr6.sin6_family = AF_INET6;
-            addr6.sin6_addr = in6addr_any;
-            dbgDebug(D_API) << "Socket listening on any address";
-
-            while (!bindRestServerSocket(addr6, port_range)) {
-                mainloop->yield(bind_retry_interval_msec);
-            }
-            listening_port = ntohs(addr6.sin6_port);
-        } else {
-            dbgDebug(D_API) << "IPv4 socket opened successfully";
-            struct sockaddr_in addr;
-            bzero(&addr, sizeof(addr));
-            addr.sin_family = AF_INET;
-            if (accept_get_from_external_ip) {
-                addr.sin_addr.s_addr = htonl(INADDR_ANY);
-                dbgDebug(D_API) << "Socket listening on any address";
-            } else {
-                addr.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
-                dbgDebug(D_API) << "Socket listening on local address";
-            }
-
-            while (!bindRestServerSocket(addr, port_range)) {
-                mainloop->yield(bind_retry_interval_msec);
-            }
-            listening_port = ntohs(addr.sin_port);
-        }
-
-        listen(fd, listen_limit);
-
        auto is_primary = Singleton::Consume<I_Environment>::by<RestServer>()->get<bool>("Is Rest primary routine");
        id = mainloop->addFileRoutine(
            I_MainLoop::RoutineType::Offline,