Jul 5th update

core/config/config.cc

@@ -35,6 +35,8 @@ using namespace Config;
 
 USE_DEBUG_FLAG(D_CONFIG);
 
+static const string not_found = "";
+
 AgentProfileSettings AgentProfileSettings::default_profile_settings = AgentProfileSettings();
 
 class registerExpectedConfigUpdates : public ClientRest
@@ -107,7 +109,7 @@ public:
     PerContextValue getAllConfiguration(const std::vector<std::string> &paths) const;
     const TypeWrapper & getResource(const vector<string> &paths) const override;
     const TypeWrapper & getSetting(const vector<string> &paths) const override;
-    const string & getProfileAgentSetting(const string &setting_name) const override;
+    string getProfileAgentSetting(const string &setting_name) const override;
     vector<string> getProfileAgentSettings(const string &regex) const override;
 
     const string & getConfigurationFlag(const string &flag_name) const override;
@@ -139,9 +141,9 @@ public:
     void registerConfigPrepareCb(ConfigCb) override;
     void registerConfigLoadCb(ConfigCb) override;
     void registerConfigAbortCb(ConfigCb) override;
+    void clearOldTenants() override;
 
 private:
-    void clearOldTenants();
     bool areTenantAndProfileActive(const TenantProfilePair &tenant_profile) const;
     void periodicRegistrationRefresh();
 
@@ -152,6 +154,7 @@ private:
     void reloadConfigurationContinuesWrapper(const string &version, uint id);
     vector<string> fillMultiTenantConfigFiles(const map<string, set<string>> &tenants);
     vector<string> fillMultiTenantExpectedConfigFiles(const map<string, set<string>> &tenants);
+    map<string, string> getProfileAgentSetting() const;
 
     string
     getActiveTenant() const
@@ -262,7 +265,6 @@ private:
 
     unordered_map<TenantProfilePair, map<vector<string>, PerContextValue>> configuration_nodes;
     unordered_map<TenantProfilePair, map<vector<string>, TypeWrapper>> settings_nodes;
-    map<string, string> profile_settings;
     unordered_map<string, string> config_flags;
 
     map<vector<string>, TypeWrapper> new_resource_nodes;
@@ -337,13 +339,6 @@ ConfigComponent::Impl::init()
             false
         );
     }
-
-    mainloop->addRecurringRoutine(
-        I_MainLoop::RoutineType::System,
-        tenant_manager->getTimeoutVal(),
-        [this] () { clearOldTenants(); },
-        "Config comp old tenant cleanup"
-    );
 }
 
 static
@@ -428,13 +423,13 @@ ConfigComponent::Impl::getSetting(const vector<string> &paths) const
     return empty;
 }
 
-const string &
+string
 ConfigComponent::Impl::getProfileAgentSetting(const string &setting_name) const
 {
+    auto profile_settings = getProfileAgentSetting();
     auto setting_raw_val = profile_settings.find(setting_name);
     if (setting_raw_val != profile_settings.end()) return setting_raw_val->second;
 
-    const static string not_found = "";
     return not_found;
 }
 
@@ -447,7 +442,6 @@ ConfigComponent::Impl::getConfigurationFlag(const string &flag_name) const
     flag = config_flags.find(flag_name);
     if (flag != config_flags.end()) return flag->second;
 
-    const static string not_found = "";
     return not_found;
 }
 
@@ -535,6 +529,7 @@ ConfigComponent::Impl::getProfileAgentSettings(const string &regex) const
 {
     vector<string> setting_raw_values;
     boost::regex reg(regex);
+    auto profile_settings = getProfileAgentSetting();
     for (auto &setting : profile_settings) {
         if (NGEN::Regex::regexMatch(__FILE__, __LINE__, setting.first, reg)) {
             setting_raw_values.push_back(setting.second);
@@ -781,23 +776,6 @@ ConfigComponent::Impl::commitSuccess()
     configuration_nodes = move(new_configuration_nodes);
     settings_nodes = move(new_settings_nodes);
 
-    AgentProfileSettings profile_agent_settings = getSettingWithDefault<AgentProfileSettings>(
-        AgentProfileSettings::default_profile_settings,
-        "agentSettings"
-    );
-
-    AgentProfileSettings general_agent_settings = getSettingWithDefault<AgentProfileSettings>(
-        AgentProfileSettings::default_profile_settings,
-        "generalAgentSettings"
-    );
-
-    auto tmp_general_settings = general_agent_settings.getSettings();
-
-    for (const pair<string, string> &profile_setting : profile_agent_settings.getSettings()) {
-        tmp_general_settings.insert(profile_setting);
-    }
-
-    profile_settings = tmp_general_settings;
     reloadFileSystemPaths();
 
     for (auto &cb : configuration_commit_cbs) {
@@ -868,14 +846,16 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
 
     map<string, shared_ptr<ifstream>> files;
     for (const auto &path : config_file_paths) {
+        dbgTrace(D_CONFIG) << "Inserting " << path << " to the list of files to be handled";
         auto fullpath = config_directory_path + path;
         files.emplace(fullpath, make_shared<ifstream>(fullpath));
     }
 
     map<string, set<string>> active_tenants =
-        tenant_manager ? tenant_manager->fetchActiveTenantsAndProfiles() : map<string, set<string>>();
+        tenant_manager ? tenant_manager->fetchAndUpdateActiveTenantsAndProfiles(true) : map<string, set<string>>();
 
     dbgTrace(D_CONFIG) << "Number of active tenants found while reloading configuration: " << active_tenants.size();
+    clearOldTenants();
 
     const vector<string> &config_files = fillMultiTenantConfigFiles(active_tenants);
     const vector<string> &expected_config_files = fillMultiTenantExpectedConfigFiles(active_tenants);
@@ -903,6 +883,22 @@ ConfigComponent::Impl::reloadConfigurationImpl(const string &version, bool is_as
     return res;
 }
 
+map<string, string>
+ConfigComponent::Impl::getProfileAgentSetting() const
+{
+    auto general_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "generalAgentSettings");
+
+    auto settings = general_sets.getSettings();
+
+    auto profile_sets = getSettingWithDefault(AgentProfileSettings::default_profile_settings, "agentSettings");
+    auto profile_settings = profile_sets.getSettings();
+    for (const auto &profile_setting : profile_settings) {
+        settings.insert(profile_setting);
+    }
+
+    return settings;
+}
+
 void
 ConfigComponent::Impl::reloadConfigurationContinuesWrapper(const string &version, uint id)
 {

core/config/config_globals.cc

@@ -77,6 +77,12 @@ getFilesystemPathConfig()
     return Singleton::Consume<I_Config>::from<MockConfigProvider>()->getFilesystemPathConfig();
 }
 
+void
+clearOldTenants()
+{
+    Singleton::Consume<I_Config>::from<MockConfigProvider>()->clearOldTenants();
+}
+
 const string &
 getLogFilesPathConfig()
 {

core/include/attachments/nginx_attachment_common.h

@@ -18,6 +18,7 @@
 #include <stddef.h>
 #include <stdint.h>
 #include <sys/types.h>
+#include <assert.h>
 
 #define MAX_NGINX_UID_LEN 32
 #define NUM_OF_NGINX_IPC_ELEMENTS 200
@@ -183,6 +184,10 @@ typedef enum ngx_http_meta_data
     CLIENT_ADDR_SIZE,
     CLIENT_ADDR_DATA,
     CLIENT_PORT,
+    PARSED_HOST_SIZE,
+    PARSED_HOST_DATA,
+    PARSED_URI_SIZE,
+    PARSED_URI_DATA,
 
     META_DATA_COUNT
 } ngx_http_meta_data_e;
@@ -242,6 +247,7 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data {
         } custom_response_data;
 
         struct __attribute__((__packed__)) ngx_http_cp_redirect_data {
+            uint8_t unused_dummy;
             uint8_t add_event_id;
             uint16_t redirect_location_size;
             char redirect_location[0];
@@ -249,6 +255,12 @@ typedef struct __attribute__((__packed__)) ngx_http_cp_web_response_data {
     } response_data;
 } ngx_http_cp_web_response_data_t;
 
+static_assert(
+    sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.custom_response_data) ==
+    sizeof(((ngx_http_cp_web_response_data_t*)0)->response_data.redirect_data),
+    "custom_response_data must be equal to redirect_data in size"
+);
+
 typedef union __attribute__((__packed__)) ngx_http_cp_modify_data {
     ngx_http_cp_inject_data_t inject_data[0];
     ngx_http_cp_web_response_data_t web_response_data[0];

core/include/general/intelligence_comp_v2.h

@@ -20,6 +20,8 @@
 #include "i_messaging.h"
 #include "i_mainloop.h"
 #include "i_time_get.h"
+#include "i_agent_details.h"
+#include "i_rest_api.h"
 #include "component.h"
 
 class IntelligenceComponentV2
@@ -28,6 +30,8 @@ class IntelligenceComponentV2
     Singleton::Provide<I_Intelligence_IS_V2>,
     Singleton::Consume<I_Messaging>,
     Singleton::Consume<I_MainLoop>,
+    Singleton::Consume<I_AgentDetails>,
+    Singleton::Consume<I_RestApi>,
     Singleton::Consume<I_TimeGet>
 {
 public:

core/include/general/tenant_manager.h

@@ -21,6 +21,7 @@
 #include "i_time_get.h"
 #include "i_instance_awareness.h"
 #include "component.h"
+#include "i_shell_cmd.h"
 
 enum class TenantManagerType { CLIENT, SERVER };
 
@@ -33,7 +34,8 @@ class TenantManager
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_TimeGet>,
     Singleton::Consume<I_Environment>,
-    Singleton::Consume<I_RestApi>
+    Singleton::Consume<I_RestApi>,
+    Singleton::Consume<I_ShellCmd>
 {
 public:
     TenantManager();

core/include/services_sdk/interfaces/i_intelligence_is_v2.h

@@ -25,12 +25,25 @@
 #include "intelligence_is_v2/intelligence_query_v2.h"
 #include "config.h"
 
+namespace Intelligence {
+
+class Invalidation;
+
+} // namespace Intelligence
+
 class I_Intelligence_IS_V2
 {
 public:
+    virtual bool sendInvalidation(const Intelligence::Invalidation &invalidation) const = 0;
+    virtual Maybe<uint> registerInvalidation(
+        const Intelligence::Invalidation &invalidation,
+        const std::function<void(const Intelligence::Invalidation &)> &callback
+    ) = 0;
+    virtual void unregisterInvalidation(uint id) = 0;
+
     template<typename Data>
     Maybe<std::vector<AssetReply<Data>>>
-    queryIntelligence(QueryRequest &query_request)
+    queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true)
     {
         uint assets_limit = query_request.getAssetsLimit();
         static const uint upper_assets_limit = 50;
@@ -50,7 +63,7 @@ public:
             return genError("Paging is activated and already finished. No need for more queries.");
         }
 
-        IntelligenceQuery<Data> intelligence_query(query_request);
+        IntelligenceQuery<Data> intelligence_query(query_request, is_pretty);
         static const std::string query_uri = "/api/v2/intelligence/assets/query";
 
         bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit);
@@ -67,12 +80,15 @@ public:
             }
         }
 
+        if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
+            return genError("Query intelligence response with InProgress status");
+        }
         return intelligence_query.getData();
     }
 
     template<typename Data>
     Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
-    queryIntelligence(std::vector<QueryRequest> &query_requests)
+    queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true)
     {
         static const uint upper_assets_limit = 50;
         static const uint upper_confidence_limit = 1000;
@@ -95,7 +111,7 @@ public:
                 );
             }
         }
-        IntelligenceQuery<Data> intelligence_query(query_requests);
+        IntelligenceQuery<Data> intelligence_query(query_requests, is_pretty);
         static const std::string query_uri = "/api/v2/intelligence/assets/queries";
 
         dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items";

core/include/services_sdk/interfaces/i_messaging_downloader.h

@@ -1,38 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __I_MESSAGING_DOWNLOADER_H__
-#define __I_MESSAGING_DOWNLOADER_H__
-
-#include <string>
-#include <functional>
-
-#include "maybe_res.h"
-
-class I_MessagingDownloader
-{
-public:
-    using OnCompleteCB = std::function<void(const Maybe<std::string> &)>;
-
-    virtual bool downloadFile(
-        const std::string &file_name,
-        const std::string &url,
-        OnCompleteCB cb = nullptr,
-        const unsigned int port = 0
-    ) = 0;
-
-protected:
-    virtual ~I_MessagingDownloader() {}
-};
-
-#endif // __I_MESSAGING_DOWNLOADER_H__

core/include/services_sdk/interfaces/i_rest_api.h

@@ -48,6 +48,8 @@ public:
         return addRestCall(oper, uri, std::make_unique<SpecificRestInit<T>>());
     }
 
+    virtual uint16_t getListeningPort() const = 0;
+
 protected:
     ~I_RestApi() {}
     virtual bool addRestCall(RestAction oper, const std::string &uri, std::unique_ptr<RestInit> &&init) = 0;

core/include/services_sdk/interfaces/i_tenant_manager.h

@@ -21,12 +21,11 @@
 #include <chrono>
 #include <functional>
 
+#include "tenant_profile_pair.h"
+
 class I_TenantManager
 {
 public:
-    using newTenantCB = std::function<void(const std::vector<std::string> &)>;
-
-    virtual void uponNewTenants(const newTenantCB &cb) = 0;
     virtual bool areTenantAndProfileActive(const std::string &tenant_id, const std::string &profile_id) const = 0;
 
     virtual std::set<std::string> fetchAllActiveTenants() const = 0;
@@ -36,14 +35,13 @@ public:
         const std::string &profile_id
     ) const = 0;
     virtual std::map<std::string, std::set<std::string>> fetchActiveTenantsAndProfiles() const = 0;
+    virtual std::map<std::string, std::set<std::string>> fetchAndUpdateActiveTenantsAndProfiles(bool update) = 0;
     virtual std::set<std::string> fetchProfileIds(const std::string &tenant_id) const = 0;
 
     virtual void deactivateTenant(const std::string &tenant_id, const std::string &profile_id) = 0;
 
     virtual void addActiveTenantAndProfile(const std::string &tenant_id, const std::string &profile_id) = 0;
 
-    virtual std::chrono::microseconds getTimeoutVal() const = 0;
-
     virtual std::set<std::string> getProfileIdsForRegionAccount(
         const std::string &tenant_id,
         const std::string &region,

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h

@@ -27,20 +27,22 @@ template <typename UserSerializableReplyAttr>
 class IntelligenceQuery
 {
 public:
-    IntelligenceQuery(QueryRequest &filter)
+    IntelligenceQuery(QueryRequest &filter, bool is_pretty)
             :
         request(filter),
         response(),
         responses(),
-        is_bulk(false)
+        is_bulk(false),
+        is_pretty(is_pretty)
     {}
 
-    IntelligenceQuery(std::vector<QueryRequest> &filters)
+    IntelligenceQuery(std::vector<QueryRequest> &filters, bool is_pretty)
             :
         requests(filters),
         response(),
         responses(),
-        is_bulk(true)
+        is_bulk(true),
+        is_pretty(is_pretty)
     {}
 
     Maybe<std::string> genJson() const;
@@ -67,6 +69,7 @@ private:
     IntelligenceQueryResponse<UserSerializableReplyAttr> response;
     std::vector<IntelligenceQueryResponse<UserSerializableReplyAttr>> responses;
     bool is_bulk;
+    bool is_pretty;
 };
 
 #include "intelligence_query_v2_impl.h"

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h

@@ -18,6 +18,9 @@
 #error intelligence_query_impl_v2.h should not be included directly!
 #endif // __INTELLIGENCE_QUERY_V2_H__
 
+#include <sstream>
+#include "json_stream.h"
+
 USE_DEBUG_FLAG(D_INTELLIGENCE);
 
 template <typename UserSerializableReplyAttr>
@@ -32,9 +35,10 @@ Maybe<std::string>
 IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
 {
     {
-        std::stringstream out;
+        std::stringstream str_stream;
+        JsonStream json_stream(&str_stream, is_pretty);
         {
-            cereal::JSONOutputArchive out_ar(out);
+            cereal::JSONOutputArchive out_ar(json_stream);
             if (is_bulk) {
                 std::vector<BulkQueryRequest> bulk_requests;
                 int index = 0;
@@ -46,7 +50,8 @@ IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
                 request.saveToJson(out_ar);
             }
         }
-        return out.str();
+
+        return str_stream.str();
     }
 }
 

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_types_v2.h

@@ -43,7 +43,9 @@ enum class Condition
     STARTS_WITH,
     CONTAINS,
     IN,
-    NOT_IN
+    NOT_IN,
+    GREATER_THAN,
+    LESS_THAN
 };
 
 enum class CursorState {

core/include/services_sdk/interfaces/intelligence_is_v2/json_stream.h

@@ -0,0 +1,35 @@
+// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __JSON_STREAM_H__
+#define __JSON_STREAM_H__
+
+#include <ostream>
+
+class JsonStream : public std::streambuf, public std::ostream
+{
+public:
+    JsonStream(std::ostream *os, bool is_pretty);
+
+private:
+    int overflow(int c) override;
+    void emplace(char c);
+    void add(char c);
+
+    std::ostream *os;
+    bool is_prev_single_backslash = false;
+    bool is_pretty;
+    bool in_string = false;
+};
+
+#endif // __JSON_STREAM_H__

core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h

@@ -14,6 +14,7 @@
 #ifndef __QUERY_FILTER_V2_H__
 #define __QUERY_FILTER_V2_H__
 
+#include <boost/variant.hpp>
 #include <string>
 #include <chrono>
 #include <boost/functional/hash.hpp>
@@ -29,6 +30,8 @@ using namespace Intelligence_IS_V2;
 class SerializableQueryCondition
 {
 public:
+    typedef boost::variant<int64_t, std::string> ValueVariant;
+
     SerializableQueryCondition() {}
 
     SerializableQueryCondition(Condition _condition_type, std::string _key, std::string _value)
@@ -36,18 +39,25 @@ public:
         condition_type(_condition_type),
         key(_key),
         value(_value)
-    {};
+        {}
+
+    SerializableQueryCondition(Condition _condition_type, std::string _key, int64_t _value)
+            :
+        condition_type(_condition_type),
+        key(_key),
+        value(_value)
+        {}
 
     void save(cereal::JSONOutputArchive &ar) const;
 
     Condition getConditionType() const { return condition_type; }
     const std::string & getKey() const { return key; }
-    const std::string & getValue() const { return value; }
+    const ValueVariant & getValue() const { return value; }
 
 private:
     Condition condition_type = Condition::EQUALS;
     std::string key = "";
-    std::string value = "";
+    ValueVariant value = "";
 };
 
 class SerializableQueryFilter
@@ -55,22 +65,18 @@ class SerializableQueryFilter
 public:
     SerializableQueryFilter() {}
     SerializableQueryFilter(Condition condition_type, const std::string &key, const std::string &value);
-    SerializableQueryFilter(
-        Operator operator_type,
-        Condition condition_type,
-        const std::string &key,
-        const std::string &value
-    );
+    SerializableQueryFilter(Condition condition_type, const std::string &key, const int64_t &value);
 
     void save(cereal::JSONOutputArchive &ar) const;
 
     void addCondition(Condition condition_type, const std::string &key, const std::string &value);
+    void addCondition(Condition condition_type, const std::string &key, const int64_t &value);
 
     Operator getOperator() const { return operator_type; }
     const std::vector<SerializableQueryCondition> & getConditionOperands() const { return condition_operands; }
     const std::vector<SerializableQueryFilter> & getQueriesOperands() const { return queries_operands; }
 
-    const std::string & getConditionValueByKey(const std::string &key) const;
+    Maybe<SerializableQueryCondition::ValueVariant> getConditionValueByKey(const std::string &key) const;
 
     bool empty() const { return condition_operands.empty() && queries_operands.empty(); }
 
@@ -80,7 +86,8 @@ public:
 private:
     void saveCondition(cereal::JSONOutputArchive &ar) const;
     void saveOperation(cereal::JSONOutputArchive &ar) const;
-    SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type);
+    bool isOperatorComp(const Operator &oper) const;
+    SerializableQueryFilter calcOperator(const SerializableQueryFilter &other_query, const Operator &oper);
 
     Operator operator_type = Operator::NONE;
     std::vector<SerializableQueryFilter> queries_operands = {};

core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h

@@ -37,6 +37,14 @@ public:
         AttributeKeyType type = AttributeKeyType::MAIN
     );
 
+    QueryRequest(
+        Condition condition_type,
+        const std::string &key,
+        const int64_t &value,
+        bool full_response,
+        AttributeKeyType type = AttributeKeyType::MAIN
+    );
+
     void saveToJson(cereal::JSONOutputArchive &ar) const;
     void save(cereal::JSONOutputArchive &ar) const;
 
@@ -51,6 +59,13 @@ public:
         AttributeKeyType attribute_type = AttributeKeyType::MAIN
     );
 
+    void addCondition(
+        Condition condition_type,
+        const std::string &key,
+        const int64_t &value,
+        AttributeKeyType attribute_type = AttributeKeyType::MAIN
+    );
+
     void setRequestedAttr(
         const std::string &attr,
         AttributeKeyType attribute_type = AttributeKeyType::REGULAR

core/include/services_sdk/interfaces/mock/mock_rest_api.h

@@ -8,6 +8,7 @@
 class MockRestApi : public Singleton::Provide<I_RestApi>::From<MockProvider<I_RestApi>>
 {
 public:
+    MOCK_CONST_METHOD0(getListeningPort, uint16_t());
     // You can't mock a function with an R-value reference. So mock a slightly different one
     MOCK_METHOD3(mockRestCall, bool(RestAction, const std::string &, const std::unique_ptr<RestInit> &));
 

core/include/services_sdk/interfaces/mock/mock_tenant_manager.h

@@ -13,8 +13,6 @@
 class MockTenantManager : public Singleton::Provide<I_TenantManager>::From<MockProvider<I_TenantManager>>
 {
 public:
-    MOCK_METHOD1(uponNewTenants,                    void(const I_TenantManager::newTenantCB &cb));
-
     MOCK_CONST_METHOD0(fetchActiveTenantsAndProfiles,     std::map<std::string, std::set<std::string>>());
     MOCK_CONST_METHOD0(fetchActiveTenants,          std::set<std::string>());
     MOCK_CONST_METHOD0(fetchAllActiveTenants,       std::set<std::string>());
@@ -26,13 +24,11 @@ public:
     MOCK_CONST_METHOD2(areTenantAndProfileActive,   bool(const std::string &, const std::string &));
     MOCK_METHOD2(addActiveTenantAndProfile,         void(const std::string &, const std::string &));
     MOCK_METHOD2(deactivateTenant,                  void(const std::string &, const std::string &));
+    MOCK_METHOD1(fetchAndUpdateActiveTenantsAndProfiles, std::map<std::string, std::set<std::string>>(bool));
     MOCK_CONST_METHOD3(
         getProfileIdsForRegionAccount,
         std::set<std::string>(const std::string &, const std::string &, const std::string &)
     );
-
-    MOCK_CONST_METHOD0(getTimeoutVal,               std::chrono::microseconds());
-
 private:
     MOCK_METHOD3(
         addInstance,

core/include/services_sdk/resources/config.h

@@ -90,6 +90,7 @@ std::string getConfigurationFlagWithDefault(const std::string &default_val, cons
 
 const std::string & getFilesystemPathConfig();
 const std::string & getLogFilesPathConfig();
+void clearOldTenants();
 
 std::string getPolicyConfigPath(
     const std::string &name,

core/include/services_sdk/resources/config/config_impl.h

@@ -155,7 +155,7 @@ Maybe<SettingType, Config::Errors>
 getProfileAgentSetting(const std::string &setting)
 {
     auto i_config = Singleton::Consume<Config::I_Config>::from<Config::MockConfigProvider>();
-    const std::string &value = i_config->getProfileAgentSetting(setting);
+    std::string value = i_config->getProfileAgentSetting(setting);
 
     if (value.empty()) return TypeWrapper::failMissing<SettingType>();
 

core/include/services_sdk/resources/config/i_config.h

@@ -39,7 +39,7 @@ public:
     virtual PerContextValue getAllConfiguration(const std::vector<std::string> &paths) const = 0;
     virtual const TypeWrapper & getResource(const std::vector<std::string> &paths) const = 0;
     virtual const TypeWrapper & getSetting(const std::vector<std::string> &paths) const = 0;
-    virtual const string & getProfileAgentSetting(const string &setting_name) const = 0;
+    virtual string getProfileAgentSetting(const string &setting_name) const = 0;
     virtual vector<string> getProfileAgentSettings(const string &setting_name_regex) const = 0;
 
     virtual const string & getConfigurationFlag(const string &flag_name) const = 0;
@@ -104,6 +104,8 @@ public:
     virtual void registerConfigLoadCb(ConfigCb) = 0;
     virtual void registerConfigAbortCb(ConfigCb) = 0;
 
+    virtual void clearOldTenants() = 0;
+
 protected:
     virtual ~I_Config() {}
 };

core/include/services_sdk/resources/debug_flags.h

@@ -148,6 +148,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_IOT_QUERY_INTELLIGENCE, D_IOT_AUXILIARY)
             DEFINE_FLAG(D_IOT_SAVE_PERSISTENT, D_IOT_AUXILIARY)
             DEFINE_FLAG(D_IOT_DOCKER, D_IOT_AUXILIARY)
+            DEFINE_FLAG(D_IOT_REST_AUTHENTICATION, D_IOT_AUXILIARY)
         DEFINE_FLAG(D_IOT_ENFORCE, D_IOT_NEXT)
             DEFINE_FLAG(D_IOT_ENFORCE_POLICY, D_IOT_ENFORCE)
             DEFINE_FLAG(D_IOT_ENFORCE_ASSETS, D_IOT_ENFORCE)
@@ -156,6 +157,7 @@ DEFINE_FLAG(D_COMPONENT, D_ALL)
             DEFINE_FLAG(D_IOT_QUERY_ASSETS, D_IOT_RISK)
             DEFINE_FLAG(D_IOT_INDICATOR_DATA, D_IOT_RISK)
             DEFINE_FLAG(D_IOT_INDICATORS, D_IOT_RISK)
+            DEFINE_FLAG(D_IOT_DOCKER_WATCHDOG, D_IOT_RISK)
         DEFINE_FLAG(D_IOT_DISCOVERY, D_IOT_NEXT)
             DEFINE_FLAG(D_IOT_PROBE, D_IOT_DISCOVERY)
             DEFINE_FLAG(D_IOT_ASSETS_DATA, D_IOT_DISCOVERY)

core/include/services_sdk/resources/intelligence_invalidation.h

@@ -0,0 +1,74 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __INTELLIGENCE_INVALIDATION_H__
+#define __INTELLIGENCE_INVALIDATION_H__
+
+#include <functional>
+#include <map>
+#include <string>
+#include <set>
+
+#include "maybe_res.h"
+#include "enum_array.h"
+
+class I_Intelligence_IS_V2;
+
+namespace Intelligence
+{
+
+enum class ClassifierType { CLASS, CATEGORY, FAMILY, GROUP, ORDER, KIND };
+enum class ObjectType { ASSET, ZONE, POLICY_PACKAGE, CONFIGURATION, SESSION };
+
+class Invalidation
+{
+public:
+    Invalidation(const std::string &class_value);
+
+    Invalidation & setClassifier(ClassifierType type, const std::string &val);
+    Invalidation & setStringAttr(const std::string &attr, const std::string &val);
+    Invalidation & setStringSetAttr(const std::string &attr, const std::set<std::string> &val);
+    Invalidation & setSourceId(const std::string &id);
+    Invalidation & setObjectType(ObjectType type);
+
+    std::string getClassifier(ClassifierType type) const { return classifiers[type]; }
+    Maybe<std::string, void> getStringAttr(const std::string &attr) const;
+    Maybe<std::set<std::string>, void> getStringSetAttr(const std::string &attr) const;
+    const Maybe<std::string, void> & getSourceId() const { return source_id; }
+    const Maybe<ObjectType, void> & getObjectType() const { return object_type; }
+
+    bool report(I_Intelligence_IS_V2 *interface) const;
+
+    Maybe<uint> startListening(I_Intelligence_IS_V2 *interface, const std::function<void(const Invalidation &)> &cb);
+    void stopListening(I_Intelligence_IS_V2 *interface);
+
+    Maybe<std::string> genJson() const;
+    std::string genObject() const;
+    bool isLegalInvalidation() const;
+
+    bool matches(const Invalidation &other) const;
+
+private:
+    bool hasAttr(const std::string &key, const std::string &value) const;
+
+    EnumArray<ClassifierType, std::string, 6> classifiers;
+    std::map<std::string, std::string> string_main_attr;
+    std::map<std::string, std::set<std::string>> set_string_main_attr;
+    Maybe<std::string, void> source_id;
+    Maybe<ObjectType, void> object_type;
+    Maybe<uint, void> listening_id;
+};
+
+} // namespace Intelligence
+
+#endif // __INTELLIGENCE_INVALIDATION_H__

core/include/services_sdk/resources/report/base_field.h

@@ -25,8 +25,6 @@
 #include "debug.h"
 #include "flags.h"
 #include "config.h"
-#include "virtual_container.h"
-#include "Log_modifiers.h"
 
 enum class LogFieldOption { XORANDB64, COUNT };
 
@@ -111,14 +109,53 @@ class LogField : Singleton::Consume<I_Environment>
         getSyslogAndCef() const override
         {
             std::string value(Details::getValueAsString(getValue()));
-            auto modifier1 = makeVirtualContainer<LogModifiers::ReplaceBackslash>(value);
-            auto modifier2 = makeVirtualContainer<LogModifiers::ReplaceCR>(modifier1);
-            auto modifier3 = makeVirtualContainer<LogModifiers::ReplaceLF>(modifier2);
-            auto modifier4 = makeVirtualContainer<LogModifiers::ReplaceDoubleOuotes>(modifier3);
-            auto modifier5 = makeVirtualContainer<LogModifiers::ReplaceQuote>(modifier4);
-            auto modifier6 = makeVirtualContainer<LogModifiers::ReplaceClosingBrace>(modifier5);
-            auto modifier7 = makeVirtualContainer<LogModifiers::ReplaceEqualSign>(modifier6);
-            return name + "=\"" + std::string(modifier7.begin(), modifier7.end()) + "\"";
+
+            std::string encoded_value;
+            encoded_value.reserve(value.size() + 6);
+            for (char ch : value) {
+                switch (ch) {
+                    case '\\': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('\\');
+                        break;
+                    }
+                    case '\n': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('n');
+                        break;
+                    }
+                    case '\r': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('r');
+                        break;
+                    }
+                    case '"': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('"');
+                        break;
+                    }
+                    case '\'': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('\'');
+                        break;
+                    }
+                    case ']': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back(']');
+                        break;
+                    }
+                    case '=': {
+                        encoded_value.push_back('\\');
+                        encoded_value.push_back('=');
+                        break;
+                    }
+                    default: {
+                        encoded_value.push_back(ch);
+                    }
+                }
+            }
+
+            return name + "=\"" + encoded_value + "\"";
         }
 
         // LCOV_EXCL_START Reason: seems that assert prevent the LCOV from identifying that method was tested

core/include/services_sdk/resources/report/report_enums.h

@@ -63,6 +63,8 @@ enum class Tags {
     DEPLOYMENT_K8S,
     LAYER_7_ACCESS_CONTROL,
     HORIZON_TELEMETRY_METRICS,
+    CROWDSEC,
+    PLAYGROUND,
 
     COUNT
 };

core/include/services_sdk/utilities/customized_cereal_multimap.h

@@ -20,6 +20,8 @@
 #include "cereal/types/vector.hpp"
 #include "cereal/archives/json.hpp"
 
+#include "rest/schema_printer.h"
+
 template <typename ... Types>
 class SerializableMultiMap : std::map<std::string, Types> ...
 {
@@ -40,6 +42,27 @@ public:
         }
     }
 
+    void clear() { clear(Types()...); }
+
+    template <typename T>
+    std::map<std::string, T> &
+    getMap()
+    {
+        return static_cast<std::map<std::string, T> &>(*this);
+    }
+
+    void
+    performOutputingSchema(std::ostream &out, int level)
+    {
+        RestHelper::printIndent(out, level) << "\"additionalProperties\": {\n";
+        RestHelper::printIndent(out, level + 1) << "\"anyOf\": [";
+        printTypes<Types...>(out, level +2, 0);
+        out << '\n';
+        RestHelper::printIndent(out, level + 1) << "]\n";
+        RestHelper::printIndent(out, level) << "}";
+    }
+
+private:
     template <typename Archive, typename T, typename ... More>
     void
     load(const std::string &key, Archive &archive, T t, More ... more)
@@ -59,20 +82,29 @@ public:
         std::map<std::string, T>::operator[](key) = t;
     }
 
-
-    void clear() { clear(Types()...); }
-
     template <typename T, typename ... More>
     void clear(const T &t, const More & ... more) { clear(t); clear(more...); }
 
     template <typename T>
     void clear(const T &) { std::map<std::string, T>::clear(); }
 
+    template <typename T, typename ... More>
+    void
+    printTypes(std::ostream &out, int level, uint)
+    {
+        printTypes<T>(out, level, 0);
+        out << ",";
+        printTypes<More...>(out, level, 0);
+    }
 
     template <typename T>
-    std::map<std::string, T> &
-    getMap()
+    void
+    printTypes(std::ostream &out, int level, int)
     {
-        return static_cast<std::map<std::string, T> &>(*this);
+        out << '\n';
+        RestHelper::printIndent(out, level) << "{\n";
+        TypeDector<T>::type(out, level + 1);
+        RestHelper::printIndent(out, level) << "}";
     }
+
 };

core/intelligence_is_v2/CMakeLists.txt

@@ -1,7 +1,7 @@
-include_directories(include)
-add_library(
-    intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
-    intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc
-)
-
-add_subdirectory(intelligence_is_v2_ut)
+include_directories(include)
+add_library(
+    intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
+    intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
+)
+
+add_subdirectory(intelligence_is_v2_ut)

core/intelligence_is_v2/intelligence_comp_v2.cc

@@ -19,13 +19,200 @@
 #include "config.h"
 #include "table.h"
 #include "intelligence_is_v2/query_response_v2.h"
+#include "intelligence_invalidation.h"
 
 using namespace std;
 using namespace chrono;
 using namespace Intelligence_IS_V2;
+using namespace Intelligence;
 
 USE_DEBUG_FLAG(D_INTELLIGENCE);
 
+static const string primary_port_setting = "local intelligence server primary port";
+static const string secondary_port_setting = "local intelligence server secondary port";
+static const string invalidation_uri = "/api/v2/intelligence/invalidation";
+static const string registration_uri = "/api/v2/intelligence/invalidation/register";
+
+class I_InvalidationCallBack
+{
+public:
+    virtual void performCallBacks(const Invalidation &invalidation) const = 0;
+
+protected:
+    virtual ~I_InvalidationCallBack() {}
+};
+
+using MainAttrTypes = SerializableMultiMap<string, set<string>>;
+
+static const map<string, Intelligence::ObjectType> object_names = {
+    { "asset", Intelligence::ObjectType::ASSET },
+    { "zone", Intelligence::ObjectType::ZONE },
+    { "policyPackage", Intelligence::ObjectType::POLICY_PACKAGE },
+    { "configuration", Intelligence::ObjectType::CONFIGURATION },
+    { "session", Intelligence::ObjectType::SESSION }
+};
+
+class InvalidationRegistration
+{
+public:
+    class RestCall
+    {
+    public:
+        RestCall(const stringstream &input) : rest_body(input.str()) {}
+        Maybe<string> genJson() const { return rest_body; }
+        ostream & print(ostream &os) { return os << rest_body; }
+
+    private:
+        string rest_body;
+    };
+
+    void
+    addInvalidation(const Invalidation &invalidation)
+    {
+        if (!first) stream << ',';
+        stream << ' ' << invalidation.genObject();
+        first = false;
+    }
+
+    RestCall
+    genJson() const
+    {
+        stringstream res;
+
+        res << "{ \"apiVersion\": \"v2\", \"communicationType\": \"sync\", ";
+        auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
+        res << "\"name\": \"" << details->getAgentId() << "\", ";
+        auto rest = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
+        res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", ";
+        res << "\"dataMap\": [";
+        res << stream.str();
+        res << " ] }";
+
+        return res;
+    }
+
+private:
+    bool first = true;
+    stringstream stream;
+};
+
+class InvalidationCallBack : Singleton::Provide<I_InvalidationCallBack>::SelfInterface
+{
+public:
+    uint
+    emplace(const Invalidation &invalidation, function<void(const Invalidation &)> cb)
+    {
+        dbgDebug(D_INTELLIGENCE) << "registering " << invalidation.genObject();
+        do {
+            ++running_id;
+        } while (callbacks.find(running_id) != callbacks.end());
+        callbacks.emplace(running_id, make_pair(invalidation, cb));
+        return running_id;
+    }
+
+    void erase(uint id) { callbacks.erase(id); }
+    bool empty() const { return callbacks.empty(); }
+
+    InvalidationRegistration::RestCall
+    getRegistration() const
+    {
+        InvalidationRegistration registration;
+
+        for (auto &registed_invalidation : callbacks) {
+            registration.addInvalidation(registed_invalidation.second.first);
+        }
+
+        return registration.genJson();
+    }
+
+    void
+    performCallBacks(const Invalidation &invalidation) const override
+    {
+        for (auto &registed_invalidation : callbacks) {
+            performCallBacksImpl(invalidation, registed_invalidation.second);
+        }
+    }
+
+private:
+    void
+    performCallBacksImpl(
+        const Invalidation &actual_invalidation,
+        const pair<Invalidation, function<void(const Invalidation &)>> &invalidation_and_cb
+    ) const
+    {
+        auto &registereed_invalidation = invalidation_and_cb.first;
+        auto &cb = invalidation_and_cb.second;
+        if (registereed_invalidation.matches(actual_invalidation)) cb(actual_invalidation);
+    }
+
+    map<uint, pair<Invalidation, function<void(const Invalidation &)>>> callbacks;
+    uint running_id = 0;
+};
+
+class RecieveInvalidation : public ServerRest
+{
+public:
+    void
+    doCall() override
+    {
+        Invalidation invalidation(class_name);
+
+        if (category.isActive()) invalidation.setClassifier(ClassifierType::CATEGORY, category.get());
+        if (family.isActive()) invalidation.setClassifier(ClassifierType::FAMILY, family.get());
+        if (group.isActive()) invalidation.setClassifier(ClassifierType::GROUP, group.get());
+        if (order.isActive()) invalidation.setClassifier(ClassifierType::ORDER, order.get());
+        if (kind.isActive()) invalidation.setClassifier(ClassifierType::KIND, kind.get());
+
+        if (mainAttributes.isActive()) {
+            auto strings = getMainAttr<string>();
+            for (const auto &value : strings) {
+                invalidation.setStringAttr(value.first, value.second);
+            }
+
+            auto string_sets = getMainAttr<set<string>>();
+            for (const auto &value : string_sets) {
+                invalidation.setStringSetAttr(value.first, value.second);
+            }
+        }
+
+        if (objectType.isActive()) {
+            auto type = object_names.find(objectType.get());
+            if (type != object_names.end()) invalidation.setObjectType(type->second);
+        }
+
+        if (sourceId.isActive()) invalidation.setSourceId(sourceId.get());
+
+        auto i_cb = Singleton::Consume<I_InvalidationCallBack>::from<InvalidationCallBack>();
+        i_cb->performCallBacks(invalidation);
+    }
+
+private:
+    template <typename ValType>
+    map<string, ValType>
+    getMainAttr()
+    {
+        map<string, ValType> res;
+
+        for (auto &vec_entry : mainAttributes.get()) {
+            for (auto &attr : vec_entry.getMap<ValType>()) {
+                res[attr.first] = attr.second;
+            }
+        }
+
+        return res;
+    }
+
+    C2S_LABEL_PARAM(string, class_name, "class");
+    C2S_OPTIONAL_PARAM(string, category);
+    C2S_OPTIONAL_PARAM(string, family);
+    C2S_OPTIONAL_PARAM(string, group);
+    C2S_OPTIONAL_PARAM(string, order);
+    C2S_OPTIONAL_PARAM(string, kind);
+    C2S_OPTIONAL_PARAM(string, objectType);
+    C2S_OPTIONAL_PARAM(string, sourceId);
+    C2S_OPTIONAL_PARAM(vector<MainAttrTypes>, mainAttributes);
+};
+
 class IntelligenceComponentV2::Impl
         :
     Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
@@ -78,6 +265,37 @@ public:
         message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
         timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
         mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
+
+        mainloop->addRecurringRoutine(
+            I_MainLoop::RoutineType::System,
+            chrono::minutes(12),
+            [this] () { sendReccurringInvalidationRegistration(); },
+            "Sending intelligence invalidation"
+        );
+
+        auto rest_api = Singleton::Consume<I_RestApi>::by<IntelligenceComponentV2>();
+        rest_api->addRestCall<RecieveInvalidation>(RestAction::SET, "new-invalidation/source/invalidation");
+    }
+
+    bool
+    sendInvalidation(const Invalidation &invalidation) const override
+    {
+        if (offline_mode_only) return false;
+        return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
+    }
+
+    Maybe<uint>
+    registerInvalidation(const Invalidation &invalidation, const function<void(const Invalidation &)> &cb) override
+    {
+        if (!invalidation.isLegalInvalidation()) return genError("Attempting to register invalid invalidation");
+        if (!sendRegistration(invalidation)) return genError("Failed to register for invalidation");
+        return invalidations.emplace(invalidation, cb);
+    }
+
+    void
+    unregisterInvalidation(uint id) override
+    {
+        invalidations.erase(id);
     }
 
     I_Messaging *
@@ -102,11 +320,20 @@ public:
     getOfflineInfoString(const SerializableQueryFilter &query) const override
     {
         string ip_attr_key = "mainAttributes.ip";
-        string identifier_value = move(query.getConditionValueByKey(ip_attr_key));
-        if (identifier_value == "") {
+        auto valueVariant = query.getConditionValueByKey(ip_attr_key);
+
+        if (!valueVariant.ok()) {
             return genError("could not find IP main attribute in the given query.");
         }
-        return offline_intelligence.getValueByIdentifier(identifier_value);
+        const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
+        if (const string* identifier_value = boost::get<string>(&value)) {
+            if (*identifier_value == "") {
+                return genError("Could not find IP main attribute in the given query.");
+            }
+            return offline_intelligence.getValueByIdentifier(*identifier_value);
+        }
+
+        return genError("Value is not of type string.");
     }
 
     bool
@@ -116,8 +343,157 @@ public:
     }
 
 private:
+    bool
+    hasLocalIntelligence() const
+    {
+        return getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence");
+    }
+
+    bool
+    sendLocalInvalidation(const Invalidation &invalidation) const
+    {
+        dbgFlow(D_INTELLIGENCE) << "Starting local invalidation";
+        return sendLocalInvalidationImpl(invalidation) || sendGlobalInvalidation(invalidation);
+    }
+
+    bool
+    sendLocalInvalidationImpl(const Invalidation &invalidation) const
+    {
+        auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
+        if (!server.ok()) {
+            dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
+            return false;
+        }
+
+        return
+            sendLocalInvalidationImpl(invalidation, *server, primary_port_setting) ||
+            sendLocalInvalidationImpl(invalidation, *server, secondary_port_setting);
+    }
+
+    bool
+    sendLocalInvalidationImpl(const Invalidation &invalidation, const string &server, const string &port_setting) const
+    {
+        dbgFlow(D_INTELLIGENCE) << "Sending to local intelligence";
+
+        auto port = getSetting<uint>("intelligence", port_setting);
+        if (!port.ok()) {
+            dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
+            return false;
+        }
+
+        dbgTrace(D_INTELLIGENCE)
+            << "Invalidation value: "
+            << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
+
+        return message->sendNoReplyObject(
+            invalidation,
+            I_Messaging::Method::POST,
+            server,
+            *port,
+            Flags<MessageConnConfig>(),
+            invalidation_uri,
+            getHTTPHeaders(),
+            nullptr,
+            MessageTypeTag::INTELLIGENCE
+        );
+    }
+
+    bool
+    sendGlobalInvalidation(const Invalidation &invalidation) const
+    {
+        dbgFlow(D_INTELLIGENCE) << "Starting global invalidation";
+
+        dbgTrace(D_INTELLIGENCE)
+            << "Invalidation value: "
+            << (invalidation.genJson().ok() ? invalidation.genJson().unpack() : invalidation.genJson().getErr());
+
+        return message->sendNoReplyObject(
+            invalidation,
+            I_Messaging::Method::POST,
+            invalidation_uri,
+            getHTTPHeaders(),
+            nullptr,
+            true,
+            MessageTypeTag::INTELLIGENCE
+        );
+    }
+
+    string
+    getHTTPHeaders() const
+    {
+        auto details = Singleton::Consume<I_AgentDetails>::by<IntelligenceComponentV2>();
+        auto tenant = details->getTenantId();
+        if (tenant == "") tenant = "Global";
+        auto agent = details->getAgentId();
+
+        return "X-Tenant-Id: " + tenant + "\r\nX-Source-Id: " + agent;
+    }
+
+    bool
+    sendRegistration(const Invalidation &invalidation) const
+    {
+        if (offline_mode_only) return false;
+
+        InvalidationRegistration registration;
+        registration.addInvalidation(invalidation);
+
+        return sendLocalRegistrationImpl(registration.genJson());
+    }
+
+    bool
+    sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
+    {
+        auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
+        if (!server.ok()) {
+            dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
+            return false;
+        }
+        return
+            sendLocalRegistrationImpl(registration, *server, primary_port_setting) ||
+            sendLocalRegistrationImpl(registration, *server, secondary_port_setting);
+    }
+
+    bool
+    sendLocalRegistrationImpl(
+        const InvalidationRegistration::RestCall &registration,
+        const string &server,
+        const string &port_setting
+    ) const
+    {
+        dbgFlow(D_INTELLIGENCE) << "Sending to local registration";
+
+        auto port = getSetting<uint>("intelligence", port_setting);
+        if (!port.ok()) {
+            dbgWarning(D_INTELLIGENCE) << "Could not resolve port for " << port_setting;
+            return false;
+        }
+
+        dbgTrace(D_INTELLIGENCE) << "Invalidation value: " << registration.genJson();
+
+        return message->sendNoReplyObject(
+            registration,
+            I_Messaging::Method::POST,
+            server,
+            *port,
+            Flags<MessageConnConfig>(),
+            registration_uri,
+            "",
+            nullptr,
+            MessageTypeTag::INTELLIGENCE
+        );
+    }
+
+    void
+    sendReccurringInvalidationRegistration() const
+    {
+        if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
+
+        sendLocalRegistrationImpl(invalidations.getRegistration());
+    }
+
     OfflineIntelligeceHandler    offline_intelligence;
     bool                         offline_mode_only = false;
+    InvalidationCallBack         invalidations;
     I_Messaging                  *message = nullptr;
     I_TimeGet                    *timer = nullptr;
     I_MainLoop                   *mainloop = nullptr;
@@ -141,8 +517,8 @@ IntelligenceComponentV2::preload()
     registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
     registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
     registerExpectedSetting<string>("intelligence", "local intelligence server ip");
-    registerExpectedSetting<uint>("intelligence", "local intelligence server secondary port");
-    registerExpectedSetting<uint>("intelligence", "local intelligence server primary port");
+    registerExpectedSetting<uint>("intelligence", primary_port_setting);
+    registerExpectedSetting<uint>("intelligence", secondary_port_setting);
 
     registerExpectedConfigFile("agent-intelligence", Config::ConfigFileType::Policy);
 }

core/intelligence_is_v2/intelligence_is_v2_ut/CMakeLists.txt

@@ -1,7 +1,7 @@
-file(COPY offline_intelligence_files_v2 DESTINATION .)
-link_directories(${BOOST_ROOT}/lib)
-
-add_unit_test(
-    intelligence_is_v2_ut
-    "query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc" 
-    "intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")
+file(COPY offline_intelligence_files_v2 DESTINATION .)
+link_directories(${BOOST_ROOT}/lib)
+
+add_unit_test(
+    intelligence_is_v2_ut
+    "query_request_v2_ut.cc;query_response_v2_ut.cc;intelligence_comp_v2_ut.cc;invalidation_ut.cc" 
+    "intelligence_is_v2;singleton;shell_cmd;event_is;metric;message;agent_details;connkey;-lboost_regex")

core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc

@@ -9,6 +9,7 @@
 #include "mock/mock_messaging.h"
 #include "mock/mock_mainloop.h"
 #include "mock/mock_time_get.h"
+#include "mock/mock_rest_api.h"
 
 using namespace std;
 using namespace testing;
@@ -35,8 +36,18 @@ public:
 
         EXPECT_CALL(
             mock_ml,
-            addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _))
-                .WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
+            addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(600000000), _, _, _)
+        ).WillRepeatedly(DoAll(SaveArg<2>(&routine), Return(0)));
+
+        EXPECT_CALL(
+            mock_ml,
+            addRecurringRoutine(I_MainLoop::RoutineType::System, chrono::microseconds(720000000), _, _, _)
+        ).WillRepeatedly(Return(0));
+
+        EXPECT_CALL(
+            mock_rest,
+            mockRestCall(_, "new-invalidation/source/invalidation", _)
+        ).WillRepeatedly(Return(true));
 
         string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
         setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
@@ -52,6 +63,7 @@ public:
 
     stringstream debug_output;
     StrictMock<MockMainLoop> mock_ml;
+    StrictMock<MockRestApi> mock_rest;
     NiceMock<MockTimeGet> mock_time;
     ::Environment env;
     ConfigComponent conf;
@@ -853,3 +865,175 @@ TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
     EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
     EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
 }
+
+TEST_F(IntelligenceComponentTestV2, ignoreInProgressQueryTest_2)
+{
+    string paging_in_progress_response_str(
+    "{\n"
+        "  \"assetCollections\": [\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+        "      \"name\": \"fake-online-test-asset1\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"fake-online-test1\",\n"
+        "      \"mainAttributes\": {\n"
+        "          \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
+        "          \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"phase\": \"fake online test1\",\n"
+        "            \"user\": \"Omry\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    },\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+        "      \"name\": \"fake-online-test-asset2\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"fake-online-test2\",\n"
+        "      \"mainAttributes\": {\n"
+        "          \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
+        "          \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"phase\": \"fake online test2\",\n"
+        "            \"user\": \"Max\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    }\n"
+        "  ],\n"
+        "  \"status\": \"inProgress\",\n"
+        "  \"totalNumAssets\": 2,\n"
+        "  \"cursor\": \"efgh\"\n"
+    "}\n"
+    );
+
+    string paging_done_response_str(
+    "{\n"
+        "  \"assetCollections\": [\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+        "      \"name\": \"fake-online-test-asset1\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"fake-online-test1\",\n"
+        "      \"mainAttributes\": {\n"
+        "          \"deAssetId\": \"C0:3F:0E:A5:59:64_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
+        "          \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"phase\": \"fake online test1\",\n"
+        "            \"user\": \"Omry\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    },\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+        "      \"name\": \"fake-online-test-asset2\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"fake-online-test2\",\n"
+        "      \"mainAttributes\": {\n"
+        "          \"deAssetId\": \"20:F8:5E:2F:6D:4C_e1ea0005-6362-4a66-99bd-7f30932a2527\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"e1ea0005-6362-4a66-99bd-7f30932a2527\",\n"
+        "          \"sourceId\": \"fog-app-msrv-iot-assets\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd2\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"phase\": \"fake online test2\",\n"
+        "            \"user\": \"Max\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    }\n"
+        "  ],\n"
+        "  \"status\": \"done\",\n"
+        "  \"totalNumAssets\": 2,\n"
+        "  \"cursor\": \"efgh\"\n"
+    "}\n"
+    );
+
+    EXPECT_CALL(
+        messaging_mock,
+        sendMessage(true, _, I_Messaging::Method::POST, _, _, _, _, MessageTypeTag::INTELLIGENCE))
+        .WillOnce(Return(paging_in_progress_response_str))
+        .WillOnce(Return(paging_done_response_str));
+
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
+    QueryRequest request(Condition::EQUALS, "category", "cloud", true, AttributeKeyType::NONE);
+    request.activatePaging();
+    request.setAssetsLimit(10);
+    vector<AssetReply<Profile>> objects_reply;
+    vector<string> objects_ids;
+    do {
+        auto object_result = intell->queryIntelligence<Profile>(request, true);
+        if (!object_result.ok()) {
+            if (object_result.getErr() == "Query intelligence response with InProgress status") continue;
+            break;
+        }
+
+        objects_reply = object_result.unpack();
+        if (objects_reply.empty()) break;
+
+        for (const AssetReply<Profile> &current_object : objects_reply) {
+            if (current_object.getMainAttributes().empty()) {
+                continue;
+            }
+            const string &id = current_object.getMainAttributes().begin()->second[0];
+            objects_ids.push_back(id);
+        }
+    } while (!request.isPagingFinished());
+
+    EXPECT_EQ(objects_ids.size(), 2);
+}

core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc

@@ -10,29 +10,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
 TEST(QueryRequestTestV2, QueryTest)
 {
     QueryRequest request(Condition::EQUALS, "phase", "testing", true);
-    SerializableQueryFilter request_query1 = request.getQuery();
-    vector<SerializableQueryCondition> request_operands1 = request_query1.getConditionOperands();
-    SerializableQueryCondition request_condition = *request_operands1.begin();
-
-    EXPECT_EQ(request_query1.getOperator(), Operator::NONE);
-    EXPECT_EQ(request_condition.getKey(), "mainAttributes.phase");
-    EXPECT_EQ(request_condition.getValue(), "testing");
-
     request.addCondition(Condition::EQUALS, "user1", "Omry");
     request.addCondition(Condition::EQUALS, "user2", "Max");
-    SerializableQueryFilter request_query2 = request.getQuery();
-    vector<SerializableQueryCondition> request_operands2 = request_query2.getConditionOperands();
-
-    vector<SerializableQueryCondition>::iterator it = request_operands2.begin();
-    it++;
-
-    EXPECT_EQ(request_query2.getOperator(), Operator::AND);
-    EXPECT_EQ(it->getKey(), "mainAttributes.user1");
-    EXPECT_EQ(it->getValue(), "Omry");
-
-    it++;
-    EXPECT_EQ(it->getKey(), "mainAttributes.user2");
-    EXPECT_EQ(it->getValue(), "Max");
 
     string output_json =
     "{\n"
@@ -67,6 +46,38 @@ TEST(QueryRequestTestV2, QueryTest)
         request.saveToJson(out_ar);
     }
     EXPECT_EQ(out.str(), output_json);
+
+    QueryRequest request2(Condition::GREATER_THAN, "prev_time", 1676887025952, true);
+    request2.addCondition(Condition::LESS_THAN, "curr_time", 1676887025958);
+
+    string output_json2=
+    "{\n"
+    "    \"limit\": 20,\n"
+    "    \"fullResponse\": true,\n"
+    "    \"query\": {\n"
+    "        \"operator\": \"and\",\n"
+    "        \"operands\": [\n"
+    "            {\n"
+    "                \"operator\": \"greaterThan\",\n"
+    "                \"key\": \"mainAttributes.prev_time\",\n"
+    "                \"value\": 1676887025952\n"
+    "            },\n"
+    "            {\n"
+    "                \"operator\": \"lessThan\",\n"
+    "                \"key\": \"mainAttributes.curr_time\",\n"
+    "                \"value\": 1676887025958\n"
+    "            }\n"
+    "        ]\n"
+    "    }\n"
+    "}";
+
+
+    stringstream out2;
+    {
+        cereal::JSONOutputArchive out_ar2(out2);
+        request2.saveToJson(out_ar2);
+    }
+    EXPECT_EQ(out2.str(), output_json2);
 }
 
 TEST(QueryRequestTestV2, AttributesTest)
@@ -185,6 +196,104 @@ TEST(QueryRequestTestV2, OrQueryTest)
     EXPECT_EQ(out.str(), output_json);
 }
 
+TEST(QueryRequestTestV2, AndQueryTestThree)
+{
+    QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
+    QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
+    QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
+    QueryRequest and_request_1_2 = request1 && (request2 && request3);
+    QueryRequest and_request_2_1 = (request1 && request2) && request3;
+
+    stringstream out_1_2;
+    {
+        cereal::JSONOutputArchive out_1_2_ar(out_1_2);
+        and_request_1_2.saveToJson(out_1_2_ar);
+    }
+
+    stringstream out_2_1;
+    {
+        cereal::JSONOutputArchive out_2_1_ar(out_2_1);
+        and_request_2_1.saveToJson(out_2_1_ar);
+    }
+
+    string output_json =
+        "{\n"
+        "    \"limit\": 20,\n"
+        "    \"fullResponse\": true,\n"
+        "    \"query\": {\n"
+        "        \"operator\": \"and\",\n"
+        "        \"operands\": [\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing1\"\n"
+        "            },\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing2\"\n"
+        "            },\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing3\"\n"
+        "            }\n"
+        "        ]\n"
+        "    }\n"
+        "}";
+    EXPECT_EQ(out_1_2.str(), output_json);
+    EXPECT_EQ(out_2_1.str(), output_json);
+}
+
+TEST(QueryRequestTestV2, OrQueryTestThree)
+{
+    QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);
+    QueryRequest request2(Condition::EQUALS, "phase", "testing2", true);
+    QueryRequest request3(Condition::EQUALS, "phase", "testing3", true);
+    QueryRequest or_request_1_2 = request1 || (request2 || request3);
+    QueryRequest or_request_2_1 = (request1 || request2) || request3;
+
+    stringstream out_1_2;
+    {
+        cereal::JSONOutputArchive out_1_2_ar(out_1_2);
+        or_request_1_2.saveToJson(out_1_2_ar);
+    }
+
+    stringstream out_2_1;
+    {
+        cereal::JSONOutputArchive out_2_1_ar(out_2_1);
+        or_request_2_1.saveToJson(out_2_1_ar);
+    }
+
+    string output_json =
+        "{\n"
+        "    \"limit\": 20,\n"
+        "    \"fullResponse\": true,\n"
+        "    \"query\": {\n"
+        "        \"operator\": \"or\",\n"
+        "        \"operands\": [\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing1\"\n"
+        "            },\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing2\"\n"
+        "            },\n"
+        "            {\n"
+        "                \"operator\": \"equals\",\n"
+        "                \"key\": \"mainAttributes.phase\",\n"
+        "                \"value\": \"testing3\"\n"
+        "            }\n"
+        "        ]\n"
+        "    }\n"
+        "}";
+    EXPECT_EQ(out_1_2.str(), output_json);
+    EXPECT_EQ(out_2_1.str(), output_json);
+}
+
 TEST(QueryRequestTestV2, AndWithConditionQueryTest)
 {
     QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

core/intelligence_is_v2/intelligence_types_v2.cc

@@ -36,6 +36,8 @@ Intelligence_IS_V2::convertConditionTypeToString(const Condition &condition_type
         {Condition::CONTAINS, "contains"},
         {Condition::IN, "in"},
         {Condition::NOT_IN, "notIn"},
+        {Condition::GREATER_THAN, "greaterThan"},
+        {Condition::LESS_THAN, "lessThan"},
     };
 
     auto condition_str = condition_type_to_string_map.find(condition_type);

core/intelligence_is_v2/invalidation.cc

@@ -0,0 +1,240 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_invalidation.h"
+
+#include <sstream>
+
+#include "i_intelligence_is_v2.h"
+
+using namespace Intelligence;
+using namespace std;
+
+Invalidation::Invalidation(const string &class_value)
+        :
+    source_id(genError<void>()),
+    object_type(genError<void>()),
+    listening_id(genError<void>())
+{
+    setClassifier(ClassifierType::CLASS, class_value);
+}
+
+Invalidation &
+Invalidation::setClassifier(ClassifierType type, const string &val)
+{
+    classifiers[type] = val;
+    return *this;
+}
+
+Invalidation &
+Invalidation::setStringAttr(const string &attr, const string &val)
+{
+    string_main_attr[attr] = val;
+    return *this;
+}
+
+Invalidation &
+Invalidation::setStringSetAttr(const string &attr, const set<string> &val)
+{
+    set_string_main_attr[attr] = val;
+    return *this;
+}
+
+Invalidation &
+Invalidation::setSourceId(const string &id)
+{
+    source_id = id;
+    return *this;
+}
+
+Invalidation &
+Invalidation::setObjectType(ObjectType type)
+{
+    object_type = type;
+    return *this;
+}
+
+Maybe<string, void>
+Invalidation::getStringAttr(const string &attr) const
+{
+    auto val_ref = string_main_attr.find(attr);
+    if (val_ref == string_main_attr.end()) return genError<void>();
+    return val_ref->second;
+}
+
+Maybe<set<string>, void>
+Invalidation::getStringSetAttr(const string &attr) const
+{
+    auto val_ref = set_string_main_attr.find(attr);
+    if (val_ref == set_string_main_attr.end()) return genError<void>();
+    return val_ref->second;
+}
+
+bool
+Invalidation::report(I_Intelligence_IS_V2 *interface) const
+{
+    if (!isLegalInvalidation()) return false;
+    return interface->sendInvalidation(*this);
+}
+
+Maybe<uint>
+Invalidation::startListening(I_Intelligence_IS_V2 *interface, const function<void(const Invalidation &)> &cb)
+{
+    auto res = interface->registerInvalidation(*this, cb);
+    if (res.ok()) listening_id = *res;
+    return res;
+}
+
+void
+Invalidation::stopListening(I_Intelligence_IS_V2 *interface)
+{
+    if (listening_id.ok()) interface->unregisterInvalidation(*listening_id);
+}
+
+static const map<Intelligence::ObjectType, string> convertObjectType = {
+    { Intelligence::ObjectType::ASSET, "asset" },
+    { Intelligence::ObjectType::ZONE, "zone" },
+    { Intelligence::ObjectType::POLICY_PACKAGE, "policyPackage" },
+    { Intelligence::ObjectType::CONFIGURATION, "configuration" },
+    { Intelligence::ObjectType::SESSION, "session" }
+};
+
+Maybe<string>
+Invalidation::genJson() const
+{
+    if (!isLegalInvalidation()) return genError("Incomplete intelligence invalidation");
+
+    stringstream invalidation;
+
+    invalidation << "{ \"invalidations\": [ " << genObject() <<" ] }";
+
+    return invalidation.str();
+}
+
+string
+Invalidation::genObject() const
+{
+    stringstream invalidation;
+
+    invalidation << "{ \"class\": \"" << classifiers[ClassifierType::CLASS] << '"';
+    if (classifiers[ClassifierType::CATEGORY] != "") {
+        invalidation <<", \"category\": \"" << classifiers[ClassifierType::CATEGORY] << '"';
+    }
+    if (classifiers[ClassifierType::FAMILY] != "") {
+        invalidation <<", \"family\": \"" << classifiers[ClassifierType::FAMILY] << '"';
+    }
+    if (classifiers[ClassifierType::GROUP] != "") {
+        invalidation <<", \"group\": \"" << classifiers[ClassifierType::GROUP] << '"';
+    }
+    if (classifiers[ClassifierType::ORDER] != "") {
+        invalidation <<", \"order\": \"" << classifiers[ClassifierType::ORDER] << '"';
+    }
+    if (classifiers[ClassifierType::KIND] != "") {
+        invalidation <<", \"kind\": \"" << classifiers[ClassifierType::KIND] << '"';
+    }
+
+    if (object_type.ok()) invalidation <<", \"objectType\": \"" << convertObjectType.at(*object_type) << '"';
+    if (source_id.ok()) invalidation <<", \"sourceId\": \"" << *source_id << '"';
+
+    if (!string_main_attr.empty() || !set_string_main_attr.empty()) {
+        invalidation << ", \"mainAttributes\": [ ";
+        bool first = true;
+        for (auto &attr : string_main_attr) {
+            if (!first) invalidation << ", ";
+            invalidation << "{ \"" << attr.first << "\": \"" << attr.second << "\" }";
+            first = false;
+        }
+
+        for (auto &attr : set_string_main_attr) {
+            if (!first) invalidation << ", ";
+            auto val = makeSeparatedStr(attr.second, ", ");
+            invalidation << "{ \"" << attr.first << "\": [ ";
+            bool internal_first = true;
+            for (auto &val : attr.second) {
+                if (!internal_first) invalidation << ", ";
+                invalidation << "\"" << val << "\"";
+                internal_first = false;
+            }
+            invalidation << " ] }";
+            first = false;
+        }
+
+        invalidation << " ]";
+    }
+
+    invalidation << " }";
+
+    return invalidation.str();
+}
+
+bool
+Invalidation::isLegalInvalidation() const
+{
+    if (!set_string_main_attr.empty() || !string_main_attr.empty()) {
+        if (classifiers[ClassifierType::FAMILY] == "") return false;
+    }
+
+    bool is_prev_empty = false;
+    for (auto &classifer : classifiers) {
+        if (is_prev_empty && classifer != "") return false;
+        is_prev_empty = classifer == "";
+    }
+
+    return true;
+}
+
+template <>
+class EnumCount<ClassifierType> : public EnumCountSpecialization<ClassifierType, 6> {};
+
+bool
+Invalidation::matches(const Invalidation &other) const
+{
+    for (auto key : NGEN::Range<ClassifierType>()) {
+        if (classifiers[key] != "" && classifiers[key] != other.classifiers[key]) return false;
+    }
+
+    if (object_type.ok()) {
+        if (!other.object_type.ok() || *object_type != *other.object_type) return false;
+    }
+
+    if (source_id.ok()) {
+        if (!other.source_id.ok() || *source_id != *other.source_id) return false;
+    }
+
+    for (auto &key_value : string_main_attr) {
+        if (!other.hasAttr(key_value.first, key_value.second)) return false;
+    }
+
+
+    for (auto &key_values : set_string_main_attr) {
+        for (auto &value : key_values.second) {
+            if (!other.hasAttr(key_values.first, value)) return false;
+        }
+    }
+
+    return true;
+}
+
+bool
+Invalidation::hasAttr(const string &key, const string &value) const
+{
+    auto string_elem = string_main_attr.find(key);
+    if (string_elem != string_main_attr.end()) return string_elem->second == value;
+
+    auto set_string_elem = set_string_main_attr.find(key);
+    if (set_string_elem != set_string_main_attr.end()) {
+        return set_string_elem->second.find(value) != set_string_elem->second.end();
+    }
+
+    return false;
+}

core/intelligence_is_v2/json_stream.cc

@@ -0,0 +1,50 @@
+// Copyright (C) 2023 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_is_v2/json_stream.h"
+
+JsonStream::JsonStream(std::ostream *os, bool is_pretty) : std::ostream(this), os(os), is_pretty(is_pretty) {}
+
+int
+JsonStream::overflow(int c)
+{
+    if (c != std::streambuf::traits_type::eof()) {
+        add(std::streambuf::traits_type::to_char_type(c));
+    }
+    return c;
+}
+
+void
+JsonStream::emplace(char c)
+{
+    *os << c;
+}
+
+void
+JsonStream::add(char c)
+{
+    if (is_pretty) {
+        emplace(c);
+        return;
+    }
+
+    if (is_prev_single_backslash) {
+        emplace(c);
+        is_prev_single_backslash = false;
+        return;
+    }
+
+    if (c == '"') in_string = !in_string;
+    if (c == '\\') is_prev_single_backslash = true;
+    if (in_string || !std::isspace(c)) emplace(c);
+}

core/intelligence_is_v2/query_filter_v2.cc

@@ -23,19 +23,46 @@ using namespace Intelligence_IS_V2;
 
 USE_DEBUG_FLAG(D_INTELLIGENCE);
 
+struct Visitor : public boost::static_visitor<void>
+{
+    Visitor(cereal::JSONOutputArchive& _ar) : ar(_ar) {}
+
+    template <typename T>
+    void operator()(const T& value)
+    {
+        ar(cereal::make_nvp("value", value));
+    }
+
+private:
+    cereal::JSONOutputArchive &ar;
+};
+
 void
 SerializableQueryCondition::save(cereal::JSONOutputArchive &ar) const
 {
     ar(
         cereal::make_nvp("operator", string(convertConditionTypeToString(condition_type))),
-        cereal::make_nvp("key", key),
-        cereal::make_nvp("value", value)
+        cereal::make_nvp("key", key)
     );
+
+    Visitor visitor(ar);
+    boost::apply_visitor(visitor, value);
 }
 
-SerializableQueryFilter::SerializableQueryFilter(Condition condition_type, const string &key, const string &value)
-{
-    condition_operands.push_back(SerializableQueryCondition(condition_type, key, value));
+SerializableQueryFilter::SerializableQueryFilter(
+    Condition condition_type,
+    const std::string &key,
+    const std::string &value
+) {
+    condition_operands.emplace_back(condition_type, key, value);
+}
+
+SerializableQueryFilter::SerializableQueryFilter(
+    Condition condition_type,
+    const std::string &key,
+    const int64_t &value
+) {
+    condition_operands.emplace_back(condition_type, key, value);
 }
 
 void
@@ -52,31 +79,28 @@ void
 SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const string &value)
 {
     if (queries_operands.size() > 0) {
-        SerializableQueryFilter new_query_filter(condition_type, key, value);
-        queries_operands.push_back(new_query_filter);
+        queries_operands.emplace_back(condition_type, key, value);
         return;
     }
-    if (condition_operands.size() == 1 && operator_type == Operator::NONE) {
-        operator_type = Operator::AND;
+    if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
+    condition_operands.emplace_back(condition_type, key, value);
+}
+
+void
+SerializableQueryFilter::addCondition(Condition condition_type, const string &key, const int64_t &value)
+{
+    if (queries_operands.size() > 0) {
+        queries_operands.emplace_back(condition_type, key, value);
+        return;
     }
-    SerializableQueryCondition cond(condition_type, key, value);
-    condition_operands.push_back(cond);
+    if (condition_operands.size() == 1 && operator_type == Operator::NONE) operator_type = Operator::AND;
+    condition_operands.emplace_back(condition_type, key, value);
 }
 
 void
 SerializableQueryFilter::saveCondition(cereal::JSONOutputArchive &ar) const
 {
-    SerializableQueryCondition cond = *condition_operands.begin();
-    Condition condition_type = cond.getConditionType();
-    string condition_str = convertConditionTypeToString(condition_type);
-    string filter_key = cond.getKey();
-    string filter_value = cond.getValue();
-
-    ar(
-        cereal::make_nvp("operator", condition_str),
-        cereal::make_nvp("key", filter_key),
-        cereal::make_nvp("value", filter_value)
-    );
+    condition_operands.begin()->save(ar);
 }
 
 void
@@ -101,7 +125,7 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
     }
 }
 
-const string &
+Maybe<SerializableQueryCondition::ValueVariant>
 SerializableQueryFilter::getConditionValueByKey(const string &key) const
 {
     for (const SerializableQueryCondition &condition : condition_operands) {
@@ -110,21 +134,49 @@ SerializableQueryFilter::getConditionValueByKey(const string &key) const
         }
     }
 
-    static string empty_str = "";
-    return empty_str;
+    return genError("Key not found.");
+}
+
+bool
+SerializableQueryFilter::isOperatorComp(const Operator &oper) const
+{
+    return operator_type == Operator::NONE || operator_type == oper;
 }
 
 SerializableQueryFilter
-SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &operator_type)
+SerializableQueryFilter::calcOperator(const SerializableQueryFilter &other_query, const Operator &oper)
 {
     SerializableQueryFilter query_filter_res;
-    vector<SerializableQueryFilter> new_queries_operands;
 
-    new_queries_operands.push_back(*this);
-    new_queries_operands.push_back(other_query);
+    query_filter_res.operator_type = oper;
+
+    if (isOperatorComp(oper) && other_query.isOperatorComp(oper)) {
+        size_t queries_size = queries_operands.size() + other_query.queries_operands.size();
+        size_t conditions_size = condition_operands.size() + other_query.condition_operands.size();
+        query_filter_res.queries_operands.reserve(queries_size);
+        query_filter_res.condition_operands.reserve(conditions_size);
+
+        for (const auto &subquery : queries_operands) {
+            query_filter_res.queries_operands.push_back(subquery);
+        }
+
+        for (const auto &condition : condition_operands) {
+            query_filter_res.condition_operands.push_back(condition);
+        }
+
+        for (const auto &subquery : other_query.queries_operands) {
+            query_filter_res.queries_operands.push_back(subquery);
+        }
+
+        for (const auto &condition : other_query.condition_operands) {
+            query_filter_res.condition_operands.push_back(condition);
+        }
+    } else {
+        query_filter_res.queries_operands.reserve(2);
+        query_filter_res.queries_operands.push_back(*this);
+        query_filter_res.queries_operands.push_back(other_query);
+    }
 
-    query_filter_res.queries_operands = new_queries_operands;
-    query_filter_res.operator_type = operator_type;
     return query_filter_res;
 }
 

core/intelligence_is_v2/query_request_v2.cc

@@ -58,6 +58,18 @@ QueryRequest::QueryRequest(
     full_response = full_reponse;
 }
 
+QueryRequest::QueryRequest(
+    Condition condition_type,
+    const string &key,
+    const int64_t &value,
+    bool full_reponse,
+    AttributeKeyType attribute_type
+) {
+    query = SerializableQueryFilter(condition_type, createAttributeString(key, attribute_type), value);
+    assets_limit = default_assets_limit;
+    full_response = full_reponse;
+}
+
 Maybe<string>
 QueryRequest::convertObjectTypeToString() const
 {
@@ -139,6 +151,16 @@ QueryRequest::addCondition (
     query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
 }
 
+void
+QueryRequest::addCondition (
+    Condition condition_type,
+    const string &key,
+    const int64_t &value,
+    AttributeKeyType attribute_type
+) {
+    query.addCondition(condition_type, createAttributeString(key, attribute_type), value);
+}
+
 void
 QueryRequest::setRequestedAttr(const string &attr, AttributeKeyType attr_type)
 {

core/logging/cef_stream.cc

@@ -58,6 +58,9 @@ CefStream::sendLog(const Report &log)
     }
     dbgTrace(D_REPORT) << "Connected to socket.";
     string cef_report = log.getCef();
+    if (protocol == I_Socket::SocketType::TCP) {
+        cef_report = to_string(cef_report.length()) + " " + cef_report;
+    }
     vector<char> data(cef_report.begin(), cef_report.end());
     for (size_t tries = 0; tries < 3; tries++) {
         if (i_socket->writeData(socket.unpack(), data)) {

core/logging/logging_ut/logging_ut.cc

@@ -1,4 +1,5 @@
 #include "log_generator.h"
+#include "log_utils.h"
 
 #include <sstream>
 #include <fstream>
@@ -466,6 +467,52 @@ TEST_F(LogTest, LogGen)
     );
     EXPECT_THAT(getMessages(), HasSubstr(str3));
     EXPECT_THAT(readLogFile(), HasSubstr(str3));
+
+
+    enum class TestErrors { CPU, MEMORY, DISK };
+    string str4(
+        "{\n"
+        "    \"eventTime\": \"0:0:0\",\n"
+        "    \"eventName\": \"Install policy\",\n"
+        "    \"eventSeverity\": \"Info\",\n"
+        "    \"eventPriority\": \"Low\",\n"
+        "    \"eventType\": \"Event Driven\",\n"
+        "    \"eventLevel\": \"Log\",\n"
+        "    \"eventLogLevel\": \"info\",\n"
+        "    \"eventAudience\": \"Internal\",\n"
+        "    \"eventAudienceTeam\": \"\",\n"
+        "    \"eventFrequency\": 0,\n"
+        "    \"eventTags\": [\n"
+        "        \"Policy Installation\"\n"
+        "    ],\n"
+        "    \"eventSource\": {\n"
+        "        \"agentId\": \"Unknown\",\n"
+        "        \"eventTraceId\": \"\",\n"
+        "        \"eventSpanId\": \"\",\n"
+        "        \"issuingEngineVersion\": \"" + Version::getFullVersion() + "\",\n"
+        "        \"serviceName\": \"007\"\n"
+        "    },\n"
+        "    \"eventData\": {\n"
+        "        \"logIndex\": 4,\n"
+        "        \"eventCode\": \"015-0002\"\n"
+        "    }\n"
+        "}"
+    );
+    EXPECT_EQ(
+        toJson(
+            LogGen(
+                "Install policy",
+                Audience::INTERNAL,
+                Severity::INFO,
+                Priority::LOW,
+                tag1,
+                Enreachments::BEAUTIFY_OUTPUT
+            ) << ErrorCode<ReportIS::Tags::IOT>::logError(TestErrors::DISK)
+        ),
+        str4
+    );
+    EXPECT_THAT(getMessages(), HasSubstr(str4));
+    EXPECT_THAT(readLogFile(), HasSubstr(str4));
 }
 
 TEST_F(LogTest, LogSpecificStream)

core/logging/syslog_stream.cc

@@ -51,6 +51,9 @@ void
 SyslogStream::sendLog(const Report &log)
 {
     string syslog_report = log.getSyslog();
+    if (protocol == I_Socket::SocketType::TCP) {
+        syslog_report = to_string(syslog_report.length()) + " " + syslog_report;
+    }
     vector<char> data(syslog_report.begin(), syslog_report.end());
     mainloop->addOneTimeRoutine(
         I_MainLoop::RoutineType::Offline,

core/message/message.cc

@@ -160,6 +160,7 @@ private:
     bool setSocket();
     bool connect(const string &host, const string &overwrite_port);
     bool shouldIgnoreSslValidation()const;
+    bool isBioSocketReady() const;
     Maybe<string> calculatePublicKey(const BioUniquePtr<X509> &cert) const;
     Maybe<string> getPinnedCertificate();
     bool verifyCertPinning(const BioUniquePtr<X509> &cert);
@@ -1431,6 +1432,11 @@ MessageConnection::doHandshake(const BioUniquePtr<BIO> &bio)
     );
     auto end_time = timer->getMonotonicTime() + timeout;
     while (timer->getMonotonicTime() < end_time) {
+        if (!isBioSocketReady()) {
+            dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
+            if (mainloop != nullptr) mainloop->yield(true);
+            continue;
+        }
         if (BIO_do_handshake(bio.get()) > 0 || shouldIgnoreSslValidation()) {
             return Maybe<void>();
         }
@@ -1533,6 +1539,17 @@ MessageConnection::setSocket()
     return true;
 }
 
+bool
+MessageConnection::isBioSocketReady() const
+{
+#if OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
+    return BIO_socket_wait(BIO_get_fd(bio.get(), NULL), 0, time(NULL));
+#else // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
+    return true;
+#endif // OPENSSL_VERSION_NUMBER >= 0x30000000 && !defined(OPENSSL_NO_SOCK)
+}
+
+
 bool
 MessageConnection::connect(const string &host, const string &overwrite_port)
 {
@@ -1547,6 +1564,11 @@ MessageConnection::connect(const string &host, const string &overwrite_port)
 
     while (timer->getMonotonicTime() < end_time) {
         counter++;
+        if (!isBioSocketReady()) {
+            dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
+            if (mainloop != nullptr) mainloop->yield(true);
+            continue;
+        }
         if (BIO_do_connect(bio.get()) > 0) {
             dbgDebug(D_COMMUNICATION)
                 << "Successfully established new BIO connection. "
@@ -1595,6 +1617,11 @@ MessageConnection::receiveResponse(I_MessageDecoder<T> &decoder)
     uint counter = 0;
     char buf[1000];
     while (timer->getMonotonicTime() < end_time) {
+        if (!isBioSocketReady()) {
+            dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
+            if (mainloop != nullptr) mainloop->yield(true);
+            continue;
+        }
         int len_or_error_ret_val = BIO_read(bio.get(), buf, sizeof(buf) - 1);
         if (len_or_error_ret_val <= 0) {
             if (!BIO_should_retry(bio.get())) {
@@ -1838,6 +1865,11 @@ MessageConnection::sendData(const string &data) const
     int remaining_data_len = data.length();
     while (timer->getMonotonicTime() < end_time) {
         int offset = data.length() - remaining_data_len;
+        if (!isBioSocketReady()) {
+            dbgDebug(D_COMMUNICATION) << "Socket is not ready for use.";
+            if (mainloop != nullptr) mainloop->yield(true);
+            continue;
+        }
         int data_sent_len = BIO_write(bio.get(), data.c_str() + offset, remaining_data_len);
         if (data_sent_len > 0) {
             if (remaining_data_len - data_sent_len < 0) {

core/report/tag_and_enum_management.cc

@@ -105,7 +105,9 @@ TagAndEnumManagement::convertStringToTag(const string &tag)
         {"Embedded Deployment", ReportIS::Tags::DEPLOYMENT_EMBEDDED},
         {"Kubernetes Deployment", ReportIS::Tags::DEPLOYMENT_K8S},
         {"Layer 7 Access Control", ReportIS::Tags::LAYER_7_ACCESS_CONTROL},
-        {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS}
+        {"Horizon Telemetry Metrics", ReportIS::Tags::HORIZON_TELEMETRY_METRICS},
+        {"Crowdsec", ReportIS::Tags::CROWDSEC},
+        {"Playground", ReportIS::Tags::PLAYGROUND}
     };
 
     auto report_is_tag = strings_to_tags.find(tag);
@@ -305,7 +307,9 @@ EnumArray<Tags, string> TagAndEnumManagement::tags_translation_arr {
     "Embedded Deployment",
     "Kubernetes Deployment",
     "Layer 7 Access Control",
-    "Horizon Telemetry Metrics"
+    "Horizon Telemetry Metrics",
+    "Crowdsec",
+    "Playground"
 };
 
 EnumArray<AudienceTeam, string> TagAndEnumManagement::audience_team_translation {

core/rest/rest_conn.cc

@@ -70,7 +70,7 @@ RestConn::parseConn() const
 
     string uri;
     os >> uri;
-    string identifier = uri.substr(uri.find_last_of('/') + 1);
+    string identifier = uri.substr(uri.find_first_of('/') + 1);
 
     uint len = 0;
     while (true) {

core/rest/rest_server.cc

@@ -47,6 +47,7 @@ public:
 
     bool bindRestServerSocket(struct sockaddr_in &addr, vector<uint16_t> port_range);
     bool addRestCall(RestAction oper, const string &uri, unique_ptr<RestInit> &&init) override;
+    uint16_t getListeningPort() const override { return listening_port; }
     Maybe<std::string> getSchema(const std::string &uri) const override;
     Maybe<std::string> invokeRest(const std::string &uri, istream &in) const override;
 
@@ -60,7 +61,7 @@ private:
     I_MainLoop::RoutineID id;
     I_MainLoop *mainloop;
     map<string, unique_ptr<RestInit>> rest_calls;
-    uint16_t port_used = 0;
+    uint16_t listening_port = 0;
     vector<uint16_t> port_range;
 };
 
@@ -138,7 +139,7 @@ RestServer::Impl::init()
             is_primary.ok() && *is_primary
         );
 
-        uint16_t listening_port = ntohs(addr.sin_port);
+        listening_port = ntohs(addr.sin_port);
         dbgInfo(D_API) << "REST server started: " << listening_port;
         Singleton::Consume<I_Environment>::by<RestServer>()->registerValue<int>("Listening Port", listening_port);
     };

core/rest/rest_ut/rest_config_ut.cc

@@ -99,6 +99,8 @@ TEST_F(RestConfigTest, alternative_port_used)
     );
     mainloop->run();
 
+    EXPECT_EQ(Singleton::Consume<I_RestApi>::from(rest_server)->getListeningPort(), *alternative_port);
+
     sa.sin_port = htons(alternative_port.unpack());
     EXPECT_EQ(bind(file_descriptor, reinterpret_cast<struct sockaddr *>(&sa), sizeof(struct sockaddr_in)), -1);
 

core/rest/rest_ut/rest_schema_ut.cc

@@ -19,6 +19,7 @@
 #include <arpa/inet.h>
 #include <sstream>
 #include "customized_cereal_map.h"
+#include "customized_cereal_multimap.h"
 
 using namespace std;
 
@@ -175,6 +176,14 @@ class MustMapInt :  public ServerRest
     C2S_PARAM(mapStringInt, must_map_int);
 };
 
+class MustMultiMap :  public ServerRest
+{
+    void doCall() override {}
+
+    using mapStringInt = SerializableMultiMap<string, int>;
+    C2S_PARAM(mapStringInt, must_multimap);
+};
+
 TEST(RestSchema, must_map)
 {
     stringstream string_map_schema;
@@ -214,6 +223,32 @@ TEST(RestSchema, must_map)
         "    ]\n"
         "}"
     );
+
+    stringstream multi_map_schema;
+    MustMultiMap().performOutputingSchema(multi_map_schema);
+    EXPECT_EQ(
+        multi_map_schema.str(),
+        "{\n"
+        "    \"properties\": {\n"
+        "        \"must_multimap\": {\n"
+        "            \"type\": \"object\",\n"
+        "            \"additionalProperties\": {\n"
+        "                \"anyOf\": [\n"
+        "                    {\n"
+        "                        \"type\": \"string\"\n"
+        "                    },\n"
+        "                    {\n"
+        "                        \"type\": \"integer\"\n"
+        "                    }\n"
+        "                ]\n"
+        "            }\n"
+        "        }\n"
+        "    },\n"
+        "    \"required\": [\n"
+        "        \"must_multimap\"\n"
+        "    ]\n"
+        "}"
+    );
 }
 
 class MustObject : public ServerRest

core/tenant_manager/tenant_manager.cc

@@ -24,6 +24,7 @@
 #include "hash_combine.h"
 
 using namespace std;
+using ProfilesPerTenantMap = map<string, set<string>>;
 
 USE_DEBUG_FLAG(D_TENANT_MANAGER);
 
@@ -75,10 +76,10 @@ public:
     void init();
     void fini();
 
-    void uponNewTenants(const newTenantCB &cb) override;
     bool areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const override;
 
-    map<string, set<string>> fetchActiveTenantsAndProfiles() const override;
+    ProfilesPerTenantMap fetchActiveTenantsAndProfiles() const override;
+    ProfilesPerTenantMap fetchAndUpdateActiveTenantsAndProfiles(bool update) override;
     set<string> fetchAllActiveTenants() const override;
     set<string> fetchActiveTenants() const override;
     set<string> getInstances(const string &tenant_id, const string &profile_id) const override;
@@ -88,8 +89,6 @@ public:
 
     void deactivateTenant(const string &tenant_id, const string &profile_id) override;
 
-    chrono::microseconds getTimeoutVal() const override;
-
     set<string> getProfileIdsForRegionAccount(
         const string &tenant_id,
         const string &region,
@@ -97,33 +96,24 @@ public:
     ) const override;
 
     void
-    addInstance(const string &tenant_id, const string &profile_id, const string &instace_id)
+    addInstance(const string &tenant_id, const string &profile_id, const string &instace_id) override
     {
         auto tenant_profile_pair = TenantProfilePair(tenant_id, profile_id);
         auto tenant_cache = mapper.find(tenant_profile_pair);
         if (tenant_cache == mapper.end()) {
             tenant_cache = mapper.insert(make_pair(tenant_profile_pair, TemporaryCache<string, void>())).first;
-            tenant_cache->second.startExpiration(
-                getTimeoutVal(),
-                Singleton::Consume<I_MainLoop>::by<TenantManager>(),
-                Singleton::Consume<I_TimeGet>::by<TenantManager>()
-            );
         }
 
         tenant_cache->second.createEntry(instace_id);
     }
 
 private:
-    void runUponNewTenants(const vector<string> &new_tenants);
-    void sendTenantAndProfile(const string &tenant_id, const string &profile_id);
     set<string> getAllTenants() const;
     set<string> fetchAllProfileIds(const string &tenant_id) const;
     set<string> getProfileIds(const string &tenant_id) const;
-    bool sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port);
 
     TemporaryCache<TenantProfilePair, void> active_tenants;
     map<TenantProfilePair, TemporaryCache<string, void>> mapper;
-    vector<I_TenantManager::newTenantCB> upon_cb;
 
     I_Messaging *i_messaging = nullptr;
     TenantManagerType type;
@@ -218,111 +208,20 @@ TenantManager::Impl::init()
     conn_flags.setFlag(MessageConnConfig::ONE_TIME_CONN);
     i_messaging = Singleton::Consume<I_Messaging>::by<TenantManager>();
 
-    auto cache_timeout = getTimeoutVal();
-
-    active_tenants.startExpiration(
-        cache_timeout,
-        Singleton::Consume<I_MainLoop>::by<TenantManager>(),
-        Singleton::Consume<I_TimeGet>::by<TenantManager>()
-    );
-
     if (type == TenantManagerType::SERVER) {
         auto rest = Singleton::Consume<I_RestApi>::by<TenantManager>();
         rest->addRestCall<LoadNewTenants>(RestAction::SET, "tenant-id");
         rest->addRestCall<FetchActiveTenants>(RestAction::SHOW, "active-tenants");
         rest->addRestCall<FetchProfileIds>(RestAction::SHOW, "profile-ids");
     }
-
-    if (type == TenantManagerType::CLIENT) {
-        auto interval = chrono::seconds(
-            getProfileAgentSettingWithDefault<uint32_t>(600, "agentConfig.tenantReportIntervalSeconds")
-        );
-        interval = chrono::seconds(
-            getConfigurationWithDefault(interval.count(), "Tenant Manager", "Report interval")
-        );
-
-        Singleton::Consume<I_MainLoop>::by<TenantManager>()->addRecurringRoutine(
-            I_MainLoop::RoutineType::System,
-            interval,
-            [this] ()
-            {
-                auto tenants_ids = fetchActiveTenants();
-                for (auto tenant_id : tenants_ids) {
-                    auto profile_ids = fetchAllProfileIds(tenant_id);
-                    for (auto profile_id : profile_ids) {
-                        sendTenantAndProfile(tenant_id, profile_id);
-                    }
-                }
-            },
-            "Tenant manager client reporter"
-        );
-    }
 }
 
 void
 TenantManager::Impl::fini()
 {
-    active_tenants.endExpiration();
     i_messaging = nullptr;
 }
 
-bool
-TenantManager::Impl::sendWithCustomPort(const string &tenant_id, const string &profile_id, const uint16_t port)
-{
-    if (i_messaging == nullptr) {
-        i_messaging = Singleton::Consume<I_Messaging>::by<TenantManager>();
-    }
-
-    SendNewTenants new_tenant_and_profile(tenant_id, profile_id);
-
-    return i_messaging->sendNoReplyObject(
-        new_tenant_and_profile,
-        I_Messaging::Method::POST,
-        "127.0.0.1",
-        port,
-        conn_flags,
-        "/set-tenant-id"
-    );
-}
-
-void
-TenantManager::Impl::runUponNewTenants(const vector<string> &new_tenants)
-{
-    for (auto &cb: upon_cb) {
-        Singleton::Consume<I_MainLoop>::by<TenantManager>()->addOneTimeRoutine(
-            I_MainLoop::RoutineType::System,
-            [this, cb, new_tenants] () { cb(new_tenants); },
-            "New tenant event handler"
-        );
-    }
-}
-
-void
-TenantManager::Impl::sendTenantAndProfile(const string &tenant_id, const string &profile_id)
-{
-    auto res = sendWithCustomPort(
-        tenant_id,
-        profile_id,
-        getConfigurationWithDefault<uint16_t>(
-            7777,
-            "Tenant Manager",
-            "Orchestrator's primary port"
-        )
-    );
-
-    if (!res) {
-        sendWithCustomPort(
-            tenant_id,
-            profile_id,
-            getConfigurationWithDefault<uint16_t>(
-                7778,
-                "Tenant Manager",
-                "Orchestrator's secondary port"
-            )
-        );
-    }
-}
-
 set<string>
 TenantManager::Impl::getAllTenants() const
 {
@@ -459,12 +358,6 @@ TenantManager::Impl::getProfileIdsForRegionAccount(
     return set<string>();
 }
 
-void
-TenantManager::Impl::uponNewTenants(const newTenantCB &cb)
-{
-    upon_cb.push_back(cb);
-}
-
 bool
 TenantManager::Impl::areTenantAndProfileActive(const string &tenant_id, const string &profile_id) const
 {
@@ -485,24 +378,40 @@ TenantManager::Impl::addActiveTenantAndProfile(const string &tenant_id, const st
         << ", Profile ID: "
         << profile_id;
     active_tenants.createEntry(tenant_profile);
-    if (type == TenantManagerType::CLIENT) {
-        sendTenantAndProfile(tenant_id, profile_id);
-    } else {
-        runUponNewTenants({tenant_id});
-    }
 }
 
 void
 TenantManager::Impl::deactivateTenant(const string &tenant_id, const string &profile_id)
 {
+    dbgTrace(D_TENANT_MANAGER)
+        << "Deactivate tenant and profile. Tenant ID: "
+        << tenant_id
+        << ", Profile ID: "
+        << profile_id;
     active_tenants.deleteEntry(TenantProfilePair(tenant_id, profile_id));
 }
 
-map<string, set<string>>
+ProfilesPerTenantMap
+TenantManager::Impl::fetchAndUpdateActiveTenantsAndProfiles(bool update)
+{
+    if (!update) return fetchActiveTenantsAndProfiles();
+
+    active_tenants.clear();
+    ProfilesPerTenantMap update_active_tenants = fetchActiveTenantsAndProfiles();
+    for (const auto &tenant_profile_set : update_active_tenants) {
+        auto tenant_id = tenant_profile_set.first;
+        for (const auto &profile_id : tenant_profile_set.second) {
+            active_tenants.createEntry(TenantProfilePair(tenant_id, profile_id));
+        }
+    }
+    return update_active_tenants;
+}
+
+ProfilesPerTenantMap
 TenantManager::Impl::fetchActiveTenantsAndProfiles() const
 {
     dbgFlow(D_TENANT_MANAGER) << "Fetching active teants and profiles map";
-    map<string, set<string>> active_tenants_and_profiles;
+    ProfilesPerTenantMap active_tenants_and_profiles;
     set<string> tenants = fetchAllActiveTenants();
     for (const string &tenant : tenants) {
         active_tenants_and_profiles[tenant] = fetchProfileIds(tenant);
@@ -567,19 +476,6 @@ TenantManager::Impl::fetchProfileIds(const string &tenant_id) const
     return (type == TenantManagerType::CLIENT) ? getProfileIds(tenant_id) : fetchAllProfileIds(tenant_id);
 }
 
-chrono::microseconds
-TenantManager::Impl::getTimeoutVal() const
-{
-    auto cache_timeout = chrono::seconds(
-        getProfileAgentSettingWithDefault<uint32_t>(900, "Orchestration.TenantTimeoutSeconds")
-    );
-    cache_timeout = chrono::seconds(
-        getConfigurationWithDefault(cache_timeout.count(), "Tenant Manager", "Tenant timeout")
-    );
-
-    return chrono::duration_cast<chrono::microseconds>(cache_timeout);
-}
-
 TenantManager::TenantManager()
         :
     Component("TenantManager"),
@@ -604,7 +500,6 @@ TenantManager::fini()
 void
 TenantManager::preload()
 {
-    registerExpectedConfiguration<uint32_t>("Tenant Manager", "Tenant timeout");
     registerExpectedConfiguration<string>("Tenant Manager", "Tenant manager type");
     registerExpectedSetting<AccountRegionSet>("accountRegionSet");
     registerExpectedSetting<string>("region");