sync code

core/intelligence_is_v2/intelligence_comp_v2.cc

@@ -17,9 +17,11 @@
 
 #include "cache.h"
 #include "config.h"
+#include "i_environment.h"
 #include "intelligence_invalidation.h"
 #include "intelligence_is_v2/intelligence_response.h"
 #include "intelligence_request.h"
+#include "intell_registration_event.h"
 
 using namespace std;
 using namespace chrono;
@@ -37,6 +39,8 @@ static const string queries_uri = "/api/v2/intelligence/assets/queries";
 static const string fog_health_uri = "/access-manager/health/live";
 static const string intelligence_health_uri = "/show-health";
 static const string time_range_invalidation_uri = "/api/v2/intelligence/invalidation/get";
+static const uint default_registration_interval_seconds = 720; // 12 minutes
+static const uint min_registration_interval_seconds = 30;
 
 class I_InvalidationCallBack
 {
@@ -100,7 +104,7 @@ public:
         res << "\"name\": \"" << (agent_id.empty() ? details->getAgentId() : agent_id) << "\", ";
         auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
         res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
-        res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, ";
+        res << "\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }, ";
         res << "\"dataMap\": [";
         res << stream.str();
         res << " ] }";
@@ -200,11 +204,17 @@ private:
 class SingleReceivedInvalidation : public ServerRest
 {
 public:
+
     void
     doCall() override
     {
         Invalidation invalidation(class_name);
 
+        for (const auto& header : request_headers) {
+            dbgTrace(D_INTELLIGENCE) << "Adding header: " << header.first << " = " << header.second;
+            invalidation.addHeader(header.first, header.second);
+        }
+
         if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get());
         if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get());
         if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get());
@@ -268,10 +278,10 @@ private:
     C2S_OPTIONAL_PARAM(string, invalidationType);
 };
 
-
 class ReceiveInvalidation : public ServerRest
 {
 public:
+    bool wantsHeaders() const override { return true; }
 
     void
     doCall() override
@@ -282,6 +292,8 @@ public:
             : "error in format, expected bulk invalidations, not single");
 
         for (SingleReceivedInvalidation &r : bulkArray.get()) {
+            // Copy headers from the bulk request to each individual invalidation
+            r.setRequestHeaders(request_headers);
             r.doCall();
         }
         return;
@@ -360,7 +372,7 @@ public:
 
         mainloop->addRecurringRoutine(
             I_MainLoop::RoutineType::System,
-            chrono::minutes(12),
+            chrono::seconds(getRegistrationIntervalSec()),
             [this] () { sendRecurringInvalidationRegistration(); },
             "Sending intelligence invalidation"
         );
@@ -467,6 +479,27 @@ public:
     }
 
 private:
+    uint
+    getRegistrationIntervalSec() const
+    {
+        uint interval_in_seconds = getConfigurationWithDefault(
+            default_registration_interval_seconds,
+            "intelligence",
+            "registration interval seconds"
+        );
+        if (interval_in_seconds < min_registration_interval_seconds) {
+            dbgWarning(D_INTELLIGENCE)
+                << "Registration interval is too low, setting to minimum: "
+                << min_registration_interval_seconds;
+            interval_in_seconds = min_registration_interval_seconds;
+        }
+        dbgInfo(D_INTELLIGENCE)
+            << "Using registration interval: "
+            << interval_in_seconds
+            << " seconds";
+        return interval_in_seconds;
+    }
+
     bool
     hasLocalIntelligenceSupport() const
     {
@@ -585,6 +618,7 @@ private:
     sendIntelligenceRequestImpl(const Invalidation &invalidation, const MessageMetadata &local_req_md) const
     {
         dbgFlow(D_INTELLIGENCE) << "Sending intelligence invalidation";
+
         auto res = message->sendSyncMessageWithoutResponse(
             HTTPMethod::POST,
             invalidation_uri,
@@ -634,15 +668,29 @@ private:
     ) const
     {
         dbgFlow(D_INTELLIGENCE) << "Sending intelligence invalidation registration";
-        auto res = message->sendSyncMessageWithoutResponse(
+        Maybe<string> registration_body = registration.genJson();
+        if (!registration_body.ok()) {
+            return genError("Could not generate intelligence invalidation registration body. Error: "
+                + registration_body.getErr());
+        }
+
+        auto res = message->sendSyncMessage(
             HTTPMethod::POST,
             registration_uri,
-            registration,
+            registration_body.unpack(),
             MessageCategory::INTELLIGENCE,
             registration_req_md
         );
-        if (res) return Response();
-        dbgWarning(D_INTELLIGENCE) << "Could not send intelligence invalidation registration.";
+        if (res.ok()){
+            string registration_response = res.unpack().getBody();
+            dbgInfo(D_INTELLIGENCE)
+                << "Intelligence invalidation registration sent successfully";
+            IntelligenceRegistrationEvent(true, registration_response).notify();
+            return Response();
+        }
+        IntelligenceRegistrationEvent(false).notify();
+        dbgWarning(D_INTELLIGENCE) << "Could not send intelligence invalidation registration. Error: "
+            << res.getErr().toString();
         return genError("Could not send intelligence invalidation registration");
     }
 
@@ -719,6 +767,13 @@ private:
         auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
         auto agent = (agent_id.empty() ? details->getAgentId() : agent_id) + ":" + to_string(rest->getListeningPort());
         headers["X-Source-Id"] = agent;
+        auto env = Singleton::Consume<I_Environment>::by<IntelligenceComponentV2>();
+        auto exec_name = env->get<string>("Base Executable Name");
+        if (exec_name.ok() && *exec_name != "") {
+            headers["X-Calling-Service"] = *exec_name;
+        } else {
+            dbgTrace(D_INTELLIGENCE) << "getHTTPHeaders: X-Calling-Service NOT added - exec_name not available";
+        }
 
         return headers;
     }
@@ -762,6 +817,7 @@ IntelligenceComponentV2::preload()
 {
     registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
     registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
+    registerExpectedConfiguration<uint>("intelligence", "registration interval seconds");
     registerExpectedConfiguration<bool>("intelligence", "support Invalidation");
     registerExpectedSetting<string>("intelligence", "local intelligence server ip");
     registerExpectedSetting<uint>("intelligence", primary_port_setting);

core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc

@@ -344,6 +344,7 @@ public:
         ON_CALL(mock_details, getFogDomain()).WillByDefault(Return(Maybe<string>(string("fog_domain.com"))));
         ON_CALL(mock_details, getFogPort()).WillByDefault(Return(Maybe<uint16_t>(443)));
 
+        env.preload();
         conf.preload();
         intelligence.preload();
         intelligence.init();
@@ -389,6 +390,8 @@ TEST_F(IntelligenceInvalidation, sending_incomplete_invalidation)
 
 TEST_F(IntelligenceInvalidation, sending_public_invalidation)
 {
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "idn");
+
     auto invalidation = Invalidation("aaa")
         .addMainAttr(main_attr)
         .addAttr(attr)
@@ -422,10 +425,16 @@ TEST_F(IntelligenceInvalidation, sending_public_invalidation)
         " } ] }";
     EXPECT_EQ(invalidation_json, expected_json);
     EXPECT_FALSE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
+
+    auto headers = md.getHeaders();
+    EXPECT_NE(headers.find("X-Calling-Service"), headers.end()) << "X-Calling-Service header should be present";
+    EXPECT_EQ(headers.at("X-Calling-Service"), "idn");
 }
 
 TEST_F(IntelligenceInvalidation, multiple_assets_invalidation)
 {
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "orchestration");
+
     auto main_attr_2 = StrAttributes()
         .addStringAttr("attr2", "22")
         .addStringSetAttr("attr3", {"33", "44"});
@@ -440,11 +449,13 @@ TEST_F(IntelligenceInvalidation, multiple_assets_invalidation)
         .setObjectType(Intelligence::ObjectType::ASSET);
 
     string invalidation_json;
+    MessageMetadata md;
     EXPECT_CALL(
         messaging_mock,
         sendSyncMessage(HTTPMethod::POST, invalidation_uri, _, MessageCategory::INTELLIGENCE, _)
     ).WillOnce(DoAll(
         SaveArg<2>(&invalidation_json),
+        SaveArg<4>(&md),
         Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))
     ));
 
@@ -461,10 +472,16 @@ TEST_F(IntelligenceInvalidation, multiple_assets_invalidation)
         "\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.1\" ] } ]"
         " } ] }";
     EXPECT_EQ(invalidation_json, expected_json);
+
+    auto headers = md.getHeaders();
+    EXPECT_NE(headers.find("X-Calling-Service"), headers.end());
+    EXPECT_EQ(headers.at("X-Calling-Service"), "orchestration");
 }
 
 TEST_F(IntelligenceInvalidation, sending_private_invalidation)
 {
+    Singleton::Consume<I_Environment>::from(env)->registerValue<string>("Base Executable Name", "idn");
+
     auto invalidation = Invalidation("aaa")
         .addMainAttr(main_attr)
         .addAttr(attr)
@@ -511,6 +528,10 @@ TEST_F(IntelligenceInvalidation, sending_private_invalidation)
         " } ] }";
     EXPECT_EQ(invalidation_json, expected_json);
     EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
+
+    auto headers = md.getHeaders();
+    EXPECT_NE(headers.find("X-Calling-Service"), headers.end());
+    EXPECT_EQ(headers.at("X-Calling-Service"), "idn");
 }
 
 TEST_F(IntelligenceInvalidation, register_for_invalidation)
@@ -554,7 +575,7 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation)
     EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]"));
     EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"ipv4Addresses\": [ \"1.1.1.1\" ] } ]"));
     EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN));
-    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));
+    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }"));
 
 }
 
@@ -599,7 +620,7 @@ TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation)
     ));
 
     EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0);
-    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }"));
+    EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true, \"returnRegistrationTTL\": true }"));
 
     EXPECT_THAT(
         body,

core/intelligence_is_v2/invalidation.cc

@@ -274,44 +274,44 @@ Invalidation::serialize(cereal::JSONInputArchive &ar)
     try {
         ar(cereal::make_nvp("class", class_));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgWarning(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("category", category));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("family", family));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("group", group));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("order", order));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("kind", kind));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("mainAttributes", main_attributes));
         ar(cereal::make_nvp("attributes", attributes));
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
@@ -323,21 +323,21 @@ Invalidation::serialize(cereal::JSONInputArchive &ar)
             throw std::invalid_argument("Invalid string for ObjectType: " + object_type_);
         }
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("sourceId", source_id_));
         source_id = source_id_;
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("invalidationRegistrationId", registration_id_));
         registration_id = registration_id_;
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     try {
@@ -349,14 +349,14 @@ Invalidation::serialize(cereal::JSONInputArchive &ar)
             throw std::invalid_argument("Invalid string for InvalidationType: " + invalidation_type_);
         }
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgWarning(D_INTELLIGENCE) << e.what();
     }
 
     try {
         ar(cereal::make_nvp("listeningId", listening_id_));
         listening_id = listening_id_;
     } catch (const cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 
     classifiers[ClassifierType::CLASS] = class_;
@@ -381,6 +381,35 @@ Invalidation::addMainAttr(const StrAttributes &attr)
     return *this;
 }
 
+Invalidation &
+Invalidation::addHeader(const string &name, const string &value)
+{
+    headers[name] = value;
+    return *this;
+}
+
+Maybe<string>
+Invalidation::getHeader(const string &name) const
+{
+    auto it = headers.find(name);
+    if (it != headers.end()) {
+        return it->second;
+    }
+    return genError("Header not found: " + name);
+}
+
+const map<string, string> &
+Invalidation::getHeaders() const
+{
+    return headers;
+}
+
+bool
+Invalidation::hasHeader(const string &name) const
+{
+    return headers.find(name) != headers.end();
+}
+
 Maybe<string>
 Invalidation::getRegistrationID() const{
     return registration_id;
@@ -660,11 +689,26 @@ IpAttributes::serialize(cereal::JSONInputArchive &ar)
 {
     try {
         ar(cereal::make_nvp("ipv4Addresses", ipv4_addresses));
+    } catch (cereal::Exception &e) {
+        dbgTrace(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
         ar(cereal::make_nvp("ipv4AddressesRange", ipv4_address_ranges));
+    } catch (cereal::Exception &e) {
+        dbgTrace(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
         ar(cereal::make_nvp("ipv6Addresses", ipv6_addresses));
+    } catch (cereal::Exception &e) {
+        dbgTrace(D_INTELLIGENCE) << e.what();
+    }
+
+    try {
         ar(cereal::make_nvp("ipv6AddressesRange", ipv6_address_ranges));
     } catch (cereal::Exception &e) {
-        dbgError(D_INTELLIGENCE) << e.what();
+        dbgTrace(D_INTELLIGENCE) << e.what();
     }
 }