Aug 08 2025 dev (#336)

* sync code

* sync code

* sync code

---------

Co-authored-by: Ned Wright <nedwright@proton.me>

core/include/services_sdk/interfaces/i_http_client.h

@@ -0,0 +1,50 @@
+#ifndef __I_HTTP_CLIENT_H__
+#define __I_HTTP_CLIENT_H__
+
+#include <string>
+#include <map>
+#include "messaging/http_response.h"
+
+class I_HttpClient
+{
+public:
+    virtual ~I_HttpClient() = default;
+    virtual void setProxy(const std::string& hosts) = 0;
+    virtual void setBasicAuth(const std::string& username, const std::string& password) = 0;
+    virtual void authEnabled(bool enabled) = 0;
+
+    virtual HTTPResponse
+    get(
+        const std::string& url,
+        const std::map<std::string, std::string>& headers = {}
+    ) = 0;
+
+    virtual HTTPResponse
+    post(
+        const std::string& url,
+        const std::string& data,
+        const std::map<std::string, std::string>& headers = {}
+    ) = 0;
+
+    virtual HTTPResponse
+    put(
+        const std::string& url,
+        const std::string& body,
+        const std::map<std::string, std::string>& headers = {}
+    ) = 0;
+
+    virtual HTTPResponse
+    patch(
+        const std::string& url,
+        const std::string& body,
+        const std::map<std::string, std::string>& headers = {}
+    ) = 0;
+
+    virtual HTTPResponse
+    del(
+        const std::string& url,
+        const std::map<std::string, std::string>& headers = {}
+    ) = 0;
+};
+
+#endif // __I_HTTP_CLIENT_H__

core/include/services_sdk/interfaces/i_intelligence_is_v2.h

@@ -29,6 +29,7 @@ namespace Intelligence {
 
 class Invalidation;
 class Response;
+class TimeRangeInvalidations;
 
 } // namespace Intelligence
 
@@ -39,7 +40,8 @@ public:
     virtual bool isIntelligenceHealthy() const = 0;
     virtual Maybe<uint> registerInvalidation(
         const Intelligence::Invalidation &invalidation,
-        const std::function<void(const Intelligence::Invalidation &)> &callback
+        const std::function<void(const Intelligence::Invalidation &)> &callback,
+        const std::string &AgentId = ""
     ) = 0;
     virtual void unregisterInvalidation(uint id) = 0;
     virtual Maybe<Intelligence::Response>
@@ -59,6 +61,10 @@ public:
         const MessageMetadata &req_md
     ) const = 0;
 
+    virtual Maybe<std::vector<Intelligence::Invalidation>> getInvalidations(
+        Intelligence::TimeRangeInvalidations request
+    ) const = 0;
+
     template<typename Data>
     Maybe<std::vector<AssetReply<Data>>>
     queryIntelligence(

core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h

@@ -70,6 +70,31 @@ private:
     std::vector<SerializableAssetSource<UserSerializableReplyAttr>> sources;
 };
 
+class ExternalSourceError
+{
+public:
+    ExternalSourceError() {}
+
+    const std::string & getSourceID() const { return source_id; }
+    const std::string & getSourceName() const { return source_name; }
+    uint getStatusCode() const { return status_code; }
+    const std::string & getErrorMessage() const { return error_message; }
+
+    void setSourceID(const std::string &id) { source_id = id; }
+    void setSourceName(const std::string &name) { source_name = name; }
+    void setStatusCode(uint code) { status_code = code; }
+    void setErrorMessage(const std::string &message) { error_message = message; }
+
+    template<class Archive>
+    void serialize(Archive &ar);
+
+private:
+    std::string source_id = "";
+    std::string source_name = "";
+    uint status_code = 0;
+    std::string error_message = "";
+};
+
 class IntelligenceQueryResponse
 {
 public:
@@ -83,6 +108,7 @@ public:
     Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; }
     const std::string & getCursor() const { return cursor; }
     uint getAmountOfAssets() const { return total_num_assets; }
+    const std::vector<ExternalSourceError> & getExternalSourcesErrorStatus() const;
     bool isValidInBulk() const { return !partial_fail_in_bulk; }
     void setFailInBulk() { partial_fail_in_bulk = true; }
 
@@ -91,6 +117,7 @@ private:
     uint total_num_assets = 0;
     std::string cursor = "";
     bool partial_fail_in_bulk = false;
+    std::vector<ExternalSourceError> external_sources_errors;
 };
 
 template <typename UserSerializableReplyAttr>

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h

@@ -21,6 +21,7 @@
 #include <maybe_res.h>
 #include "asset_reply.h"
 #include "bulk_query_response_v2.h"
+#include "intelligence_invalidation.h"
 
 USE_DEBUG_FLAG(D_INTELLIGENCE);
 
@@ -39,6 +40,11 @@ public:
     Maybe<void> load();
     Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
     const std::string getCursor() const { return single_response.getCursor(); }
+    const std::vector<ExternalSourceError> & getExternalSourcesErrorStatus() const
+    {
+        return single_response.getExternalSourcesErrorStatus();
+    }
+
     void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; }
     template <typename UserSerializableReplyAttr>
     IntelligenceQueryResponseT<UserSerializableReplyAttr> getSerializableResponse() const
@@ -98,10 +104,14 @@ public:
         return bulk_data;
     }
 
+    const std::vector<Invalidation>& getInvalidations() const { return invalidations; }
+    Maybe<void> loadInvalidations();
+
 private:
     std::string json_response;
     std::vector<IntelligenceQueryResponse> responses;
     IntelligenceQueryResponse single_response;
+    std::vector<Invalidation> invalidations;
     size_t size = 0;
     bool is_bulk = false;
 };

core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h

@@ -34,7 +34,8 @@ public:
         const std::string &key,
         const std::string &value,
         bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
     );
 
     QueryRequest(
@@ -42,7 +43,8 @@ public:
         const std::string &key,
         const int64_t &value,
         bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
     );
 
     QueryRequest(
@@ -50,7 +52,8 @@ public:
         const std::string &key,
         const std::vector<std::string> &value,
         bool full_response,
-        AttributeKeyType type = AttributeKeyType::MAIN
+        AttributeKeyType type = AttributeKeyType::MAIN,
+        bool _external_sources_error_status = false
     );
 
     void saveToJson(cereal::JSONOutputArchive &ar) const;
@@ -115,6 +118,7 @@ public:
 private:
     uint assets_limit = default_assets_limit;
     bool full_response = false;
+    bool external_sources_error_status = false;
     Maybe<ObjectType> object_type = genError("uninitialized");
     Maybe<RequestCursor> cursor = genError("Cursor not initialized");
     SerializableQueryFilter query;

core/include/services_sdk/interfaces/messaging/interface_impl.h

@@ -19,8 +19,12 @@
 
 USE_DEBUG_FLAG(D_MESSAGING);
 
-MessageMetadata::MessageMetadata()
+MessageMetadata::MessageMetadata(bool immediate_tracing)
 {
+    if (immediate_tracing && Singleton::exists<I_Environment>()) {
+        insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+    }
+
     if (!Singleton::exists<I_AgentDetails>() || !Singleton::exists<I_ProxyConfiguration>()) return;
     auto i_agent_details = Singleton::Consume<I_AgentDetails>::by<I_Messaging>();
     auto i_proxy_configuration = Singleton::Consume<I_ProxyConfiguration>::by<I_Messaging>();
@@ -137,6 +141,8 @@ I_Messaging::sendAsyncMessage(
         return;
     }
 
+    dbgTrace(D_MESSAGING) << "Sending async message. URI: " << uri << ", Body: " << req_body.unpack();
+
     sendAsyncMessage(
         method,
         uri,

core/include/services_sdk/interfaces/messaging/messaging_enums.h

@@ -30,6 +30,7 @@ enum class MessageConnectionConfig
     UNSECURE_CONN,
     ONE_TIME_CONN,
     IGNORE_SSL_VALIDATION,
+    ONE_TIME_FOG_CONN, // used for learning mechanism - one time connection sent by dedicated thread
 
     COUNT
 };

core/include/services_sdk/interfaces/messaging/messaging_metadata.h

@@ -9,6 +9,7 @@
 #include "singleton.h"
 #include "i_agent_details.h"
 #include "i_time_get.h"
+#include "i_environment.h"
 
 class MessageProxySettings
 {
@@ -55,14 +56,27 @@ private:
     uint16_t proxy_port = 0;
 };
 
-class MessageMetadata : Singleton::Consume<I_TimeGet>
+class MessageMetadata : Singleton::Consume<I_TimeGet>, Singleton::Consume<I_Environment>
 {
 public:
-    inline MessageMetadata();
+    inline MessageMetadata(bool immediate_tracing = false);
 
-    MessageMetadata(const std::string &_host_name, uint16_t _port_num, bool _buffer = false, bool _fog = false) :
-        host_name(_host_name), port_num(_port_num), should_buffer(_buffer), is_to_fog(_fog)
-    {}
+    MessageMetadata(
+        const std::string &_host_name,
+        uint16_t _port_num,
+        bool _buffer = false,
+        bool _fog = false,
+        bool immediate_tracing = false
+    ) :
+        host_name(_host_name),
+        port_num(_port_num),
+        should_buffer(_buffer),
+        is_to_fog(_fog)
+    {
+        if (immediate_tracing && Singleton::exists<I_Environment>()) {
+            insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+        }
+    }
 
     MessageMetadata(
         std::string _host_name,
@@ -70,7 +84,8 @@ public:
         Flags<MessageConnectionConfig> _conn_flags,
         bool _should_buffer = false,
         bool _is_to_fog = false,
-        bool _should_suspend = true
+        bool _should_suspend = true,
+        bool immediate_tracing = false
     ) :
         host_name(_host_name),
         port_num(_port_num),
@@ -79,7 +94,11 @@ public:
         is_to_fog(_is_to_fog),
         should_send_access_token(true),
         should_suspend(_should_suspend)
-    {}
+    {
+        if (immediate_tracing && Singleton::exists<I_Environment>()) {
+            insertHeaders(Singleton::Consume<I_Environment>::by<MessageMetadata>()->getCurrentHeadersMap());
+        }
+    }
 
     const bool &
     shouldSendAccessToken() const
@@ -135,6 +154,14 @@ public:
         return headers;
     }
 
+    Maybe<std::string>
+    getTraceId() const
+    {
+        auto trace_id = headers.find("X-Trace-Id");
+        if (trace_id != headers.end()) return trace_id->second;
+        return genError("Trace ID not found");
+    }
+
     std::string
     getCaPath() const
     {

core/include/services_sdk/interfaces/mock/mock_intelligence.h

@@ -16,16 +16,31 @@ operator<<(std::ostream &os, const Intelligence::Invalidation &)
     return os;
 }
 
+std::ostream &
+operator<<(std::ostream &os, const std::vector<Intelligence::Invalidation> &)
+{
+    return os;
+}
+
 class MockIntelligence : public Singleton::Provide<I_Intelligence_IS_V2>::From<MockProvider<I_Intelligence_IS_V2>>
 {
 public:
     using InvalidationCb = std::function<void(const Intelligence::Invalidation &)>;
     using Invalidation = Intelligence::Invalidation;
     using Response = Intelligence::Response;
+    using TimeRangeInvalidations = Intelligence::TimeRangeInvalidations;
 
     MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
+    MOCK_CONST_METHOD1(getInvalidations, Maybe<std::vector<Invalidation>>(TimeRangeInvalidations));
     MOCK_CONST_METHOD0(isIntelligenceHealthy, bool(void));
-    MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
+    MOCK_METHOD3(
+        registerInvalidation,
+        Maybe<uint>(
+            const Invalidation &invalidation,
+            const InvalidationCb &callback,
+            const std::string &AgentId
+        )
+    );
     MOCK_METHOD1(unregisterInvalidation, void(uint id));
     MOCK_CONST_METHOD5(
         getResponse,

core/include/services_sdk/resources/config/config_loader.h

@@ -55,7 +55,7 @@ public:
     std::pair<std::unique_ptr<EnvironmentEvaluator<bool>>, TypeWrapper>
     getLoaderConfig()
     {
-        return std::move(std::make_pair(std::move(context), TypeWrapper(value)));
+        return std::make_pair(std::move(context), TypeWrapper(value));
     }
 
 private:

core/include/services_sdk/resources/debug_flags.h

@@ -35,6 +35,7 @@ DEFINE_FLAG(D_INFRA, D_ALL)
     DEFINE_FLAG(D_TENANT_MANAGER, D_INFRA)
     DEFINE_FLAG(D_MONITORING, D_INFRA)
     DEFINE_FLAG(D_SERVICE_HEALTH_STATUS, D_INFRA)
+    DEFINE_FLAG(D_LOGGING, D_INFRA)
     DEFINE_FLAG(D_REPORT, D_INFRA)
         DEFINE_FLAG(D_REPORT_BULK, D_REPORT)
     DEFINE_FLAG(D_TRACE, D_INFRA)
@@ -48,6 +49,9 @@ DEFINE_FLAG(D_INFRA, D_ALL)
             DEFINE_FLAG(D_CONNECTION, D_MESSAGING)
             DEFINE_FLAG(D_MESSAGING_BUFFER, D_MESSAGING)
             DEFINE_FLAG(D_HTTP_REQUEST, D_MESSAGING)
+            DEFINE_FLAG(D_TRACE_ID, D_MESSAGING)
+    DEFINE_FLAG(D_MEMORY, D_INFRA)
+        DEFINE_FLAG(D_WAAP_MEMORY, D_MEMORY)
 
 DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_PRELOAD, D_COMPONENT)
@@ -72,6 +76,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_WAAP_SAMPLE_SCAN, D_WAAP)
             DEFINE_FLAG(D_WAAP_ASSET_STATE, D_WAAP)
             DEFINE_FLAG(D_WAAP_CONFIDENCE_CALCULATOR, D_WAAP)
+            DEFINE_FLAG(D_WAAP_SERIALIZE, D_WAAP)
             DEFINE_FLAG(D_WAAP_REPUTATION, D_WAAP)
             DEFINE_FLAG(D_WAAP_SCORE_BUILDER, D_WAAP)
             DEFINE_FLAG(D_WAAP_ULIMITS, D_WAAP)
@@ -153,6 +158,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_SDWAN, D_COMPONENT)
         DEFINE_FLAG(D_SDWAN_POLICY, D_SDWAN)
         DEFINE_FLAG(D_SDWAN_DATA, D_SDWAN)
+            DEFINE_FLAG(D_SDWAN_DATA_SENDER, D_SDWAN_DATA)
         DEFINE_FLAG(D_SDWAN_FEATURE_FLAG, D_SDWAN)
         DEFINE_FLAG(D_LOGGER_SDWAN, D_SDWAN)
         DEFINE_FLAG(D_SDWAN_API, D_SDWAN)
@@ -196,6 +202,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
     DEFINE_FLAG(D_IOT_ACCESS_CONTROL, D_COMPONENT)
     DEFINE_FLAG(D_HORIZON_TELEMETRY, D_COMPONENT)
     DEFINE_FLAG(D_PROMETHEUS, D_COMPONENT)
+    DEFINE_FLAG(D_AIGUARD, D_COMPONENT)
 
 DEFINE_FLAG(D_FLOW, D_ALL)
     DEFINE_FLAG(D_DROP, D_FLOW)

core/include/services_sdk/resources/generic_metric.h

@@ -116,7 +116,7 @@ private:
     friend class MetricCalc;
     void addCalc(MetricCalc *calc);
 
-    std::vector<PrometheusData> getPromMetricsData();
+    std::vector<PrometheusData> getPromMetricsData(const std::vector<MetricCalc*> *allowed_calcs = nullptr);
 
     void handleMetricStreamSending();
     void generateLog();

core/include/services_sdk/resources/intelligence_invalidation.h

@@ -19,6 +19,9 @@
 #include <string>
 #include <set>
 #include <cereal/archives/json.hpp>
+#include "rest.h"
+#include "messaging/messaging_enums.h"
+#include "messaging/messaging_metadata.h"
 
 #include "maybe_res.h"
 #include "enum_array.h"
@@ -32,6 +35,56 @@ enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
 enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION, SHORTLIVED };
 enum class InvalidationType { ADD, DELETE, UPDATE };
 
+static const std::map<std::string, ObjectType> stringToObjectTypeMap = {
+    {"asset", ObjectType::ASSET},
+    {"zone", ObjectType::ZONE},
+    {"policyPackage", ObjectType::POLICY_PACKAGE},
+    {"configuration", ObjectType::CONFIGURATION},
+    {"session", ObjectType::SESSION},
+    {"shortLived", ObjectType::SHORTLIVED}
+};
+
+static const std::map<std::string, InvalidationType> stringToInvalidationTypeMap = {
+    {"add", InvalidationType::ADD},
+    {"delete", InvalidationType::DELETE},
+    {"update", InvalidationType::UPDATE}
+};
+
+class TimeRangeInvalidations
+{
+public:
+    TimeRangeInvalidations(uint64_t start_time, uint64_t end_time) : time_range{start_time, end_time} {}
+
+    Maybe<std::string> genJson() const
+    {
+        try {
+            std::stringstream out;
+            {
+                cereal::JSONOutputArchive out_ar(out);
+                out_ar(cereal::make_nvp("timeRange", time_range));
+            }
+            return out.str();
+        } catch (const std::exception &e) {
+            return genError("Failed to generate JSON for TimeRangeInvalidations. Error: " + std::string(e.what()));
+        }
+    }
+
+private:
+    struct TimeRange
+    {
+        uint64_t start;
+        uint64_t end;
+
+        template <class Archive>
+        void serialize(Archive &ar)
+        {
+            ar(cereal::make_nvp("start", start), cereal::make_nvp("end", end));
+        }
+    };
+
+    TimeRange time_range;
+};
+
 class StrAttributes
 {
 public:
@@ -101,6 +154,7 @@ private:
 class Invalidation
 {
 public:
+    Invalidation();
     Invalidation(const std::string &class_value);
 
     Invalidation & setClassifier(ClassifierType type, const std::string &val);
@@ -113,14 +167,18 @@ public:
     std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
     std::vector<StrAttributes> getMainAttributes() const { return main_attributes; }
     std::vector<IpAttributes> getAttributes() const { return attributes; }
-    const Maybe<std::string, void> & getSourceId() const { return source_id; }
-    const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
-    const Maybe<InvalidationType, void> & getInvalidationType() const { return invalidation_type; }
-    Maybe<std::string, void> getRegistrationID() const;
+    const Maybe<std::string> & getSourceId() const { return source_id; }
+    const Maybe<ObjectType> & getObjectType() const { return object_type; }
+    const Maybe<InvalidationType> & getInvalidationType() const { return invalidation_type; }
+    Maybe<std::string> getRegistrationID() const;
 
     bool report(I_Intelligence_IS_V2 *interface) const;
 
-    Maybe<uint> startListening(I_Intelligence_IS_V2 *interface, const std::function<void(const Invalidation &)> &cb);
+    Maybe<uint> startListening(
+        I_Intelligence_IS_V2 *interface,
+        const std::function<void(const Invalidation &)> &cb,
+        const std::string &AgentId = ""
+    );
     void stopListening(I_Intelligence_IS_V2 *interface);
 
     Maybe<std::string> genJson() const;
@@ -128,6 +186,7 @@ public:
     bool isLegalInvalidation() const;
 
     bool matches(const Invalidation &other) const;
+    void serialize(cereal::JSONInputArchive &ar);
 
 private:
     bool attr_matches(const std::vector<StrAttributes> &current, const std::vector<StrAttributes> &other) const;
@@ -136,11 +195,11 @@ private:
     EnumArray<ClassifierType, std::string, 6> classifiers;
     std::vector<StrAttributes> main_attributes;
     std::vector<IpAttributes> attributes;
-    Maybe<std::string, void> source_id;
-    Maybe<ObjectType, void> object_type;
-    Maybe<InvalidationType, void> invalidation_type;
-    Maybe<uint, void> listening_id;
-    Maybe<std::string, void> registration_id;
+    Maybe<std::string> source_id;
+    Maybe<ObjectType> object_type;
+    Maybe<InvalidationType> invalidation_type;
+    Maybe<uint> listening_id;
+    Maybe<std::string> registration_id;
 };
 
 } // namespace Intelligence

core/include/services_sdk/resources/log_generator.h

@@ -31,16 +31,6 @@ class LogGen
     Singleton::Consume<I_Environment>
 {
 public:
-    template <typename Trigger, typename ...Args>
-    LogGen(
-        const Trigger &trigger,
-        const std::string &title,
-        Args ...args)
-            :
-        LogGen(trigger(title, std::forward<Args>(args)...))
-    {
-    }
-
     template <typename ...Args>
     LogGen(
         const std::string &title,

core/include/services_sdk/resources/report/report_bulks.h

@@ -51,6 +51,9 @@ public:
             dbgDebug(D_REPORT_BULK) << "Adding a new bulk to queue";
             bulks.push(LogBulkRest(bulk_size));;
         }
+        dbgTrace(D_REPORT_BULK)
+        << "Adding report to bulk, for asset: "
+        << (report.getStringData("assetName").ok() ? *report.getStringData("assetName") : "unknown");
         bulks.back().push(report);
         ++elem_in_quque;
     }

core/include/services_sdk/resources/report/report_enums.h

@@ -71,6 +71,7 @@ enum class Tags {
     DEPLOYMENT_DOCKER,
     WEB_SERVER_SWAG,
     WEB_SERVER_NGINX_UNIFIED,
+    AIGUARD,
 
     COUNT
 };

core/include/services_sdk/utilities/agent_core_utilities.h

@@ -85,6 +85,7 @@ namespace Strings
 std::string removeTrailingWhitespaces(std::string str);
 std::string removeLeadingWhitespaces(std::string str);
 std::string trim(std::string str);
+std::string toLower(std::string str);
 
 } // namespace Strings