Feb_06_2024-Dev

README.md

@@ -106,12 +106,13 @@ Before compiling the services, you'll need to ensure the latest development vers
 * cURL
 * Redis
 * Hiredis
+* MaxmindDB
 
 An example of installing the packages on Alpine:
 
 ```bash
  $ apk update
- $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis
+ $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis libmaxminddb-dev
 ```
 
 ## Compiling and packaging the agent code

components/include/health_checker.h

@@ -19,6 +19,7 @@
 #include "i_socket_is.h"
 #include "i_health_check_manager.h"
 #include "i_shell_cmd.h"
+#include "i_orchestration_status.h"
 #include "component.h"
 
 class HealthChecker
@@ -27,7 +28,8 @@ class HealthChecker
     Singleton::Consume<I_MainLoop>,
     Singleton::Consume<I_Socket>,
     Singleton::Consume<I_Health_Check_Manager>,
-    Singleton::Consume<I_ShellCmd>
+    Singleton::Consume<I_ShellCmd>,
+    Singleton::Consume<I_OrchestrationStatus>
 {
 public:
     HealthChecker();

components/include/reverse_proxy_defaults.h

@@ -0,0 +1,46 @@
+#ifndef __REVERSE_PROXY_MANAGER_DEFAULTS_H__
+#define __REVERSE_PROXY_MANAGER_DEFAULTS_H__
+
+#include <string>
+
+static const std::string product_name = getenv("DOCKER_RPM_ENABLED") ? "CloudGuard AppSec" : "AppSec Gateway";
+static const std::string default_cp_cert_file = "/etc/cp/cpCert.pem";
+static const std::string default_cp_key_file = "/etc/cp/cpKey.key";
+static const std::string default_rpm_conf_path = "/etc/cp/conf/rpmanager/";
+static const std::string default_certificate_path = "/etc/cp/rpmanager/certs";
+static const std::string default_additional_files_path = "/etc/cp/conf/rpmanager/include";
+static const std::string default_server_config = "additional_server_config.conf";
+static const std::string default_location_config = "additional_location_config.conf";
+static const std::string default_trusted_ca_suffix = "_user_ca_bundle.crt";
+static const std::string default_nginx_log_files_path = "/var/log/nginx/";
+static const std::string default_log_files_host_path = "/var/log/nano_agent/rpmanager/nginx_log/";
+static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers";
+static const std::string default_template_path = "/etc/cp/conf/rpmanager/nginx-template-clear";
+static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/";
+static const std::string default_server_certificate_path = "/etc/cp/rpmanager/certs/sslCertificate_";
+static const std::string default_server_certificate_key_path = "/etc/cp/rpmanager/certs/sslPrivateKey_";
+static const std::string default_container_name = "cp_nginx_gaia";
+static const std::string default_docker_image = "cp_nginx_gaia";
+static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/nginx.conf";
+static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
+static const std::string default_nginx_config_include_file =
+    "/etc/cp/conf/rpmanager/servers/nginx_conf_include";
+static const std::string default_global_conf_include_template =
+    "/etc/cp/conf/rpmanager/nginx-conf-include-template";
+static const std::string default_global_conf_include_template_no_responses =
+    "/etc/cp/conf/rpmanager/nginx-conf-include-template-no-responses";
+static const std::string default_cloud_vendor_file = "/etc/cp/conf/rpmanager/cloud-vendor.json";
+static const std::string default_cloud_cert_location = "/tmp/";
+static const std::string default_dns_resolver_file = "/etc/resolv.conf";
+static const std::string default_nginx_multi_lines_key = "nginxIncludeLines";
+static const std::string default_ip = "127.0.0.1";
+static const std::string default_aws_resolver_ip = "169.254.169.253";
+static const std::string default_azure_resolver_ip = "168.63.129.16";
+static const std::string default_syslog_socket_address = "127.0.0.1:1514";
+static const std::string rpm_full_load_path = "/tmp/rpm_full_load";
+static const std::string rpm_partial_load_path = "/tmp/rpm_partial_load";
+static const std::string first_rpm_policy_load_path = "/tmp/first_rpm_policy_load";
+
+static const int default_port = 5555;
+
+#endif //__REVERSE_PROXY_MANAGER_DEFAULTS_H__

components/security_apps/CMakeLists.txt

@@ -1,4 +1,4 @@
-#add_subdirectory(http_geo_filter)
+add_subdirectory(http_geo_filter)
 add_subdirectory(ips)
 add_subdirectory(layer_7_access_control)
 add_subdirectory(local_policy_mgmt_gen)

components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h

@@ -69,19 +69,11 @@ checkIDP(shared_ptr<istream> file_stream)
 #if defined(gaia) || defined(smb)
 
 Maybe<string>
-checkIsCpviewRunning(const string &command_output)
+checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
 {
-    if (command_output == "true" || command_output == "false") return command_output;
+    if (command_output == "" ) return string("false");
 
-    return genError("cpview is not running");
+    return command_output;
-}
-
-Maybe<string>
-checkIsCPotelcolGRET64(const string &command_output)
-{
-    if (command_output == "true" || command_output == "false") return command_output;
-
-    return genError("CPotelcol is not installed or its take is below T64");
 }
 
 Maybe<string>
@@ -103,6 +95,10 @@ checkCanUpdateSDWanData(const string &command_output)
 Maybe<string>
 getMgmtObjType(const string &command_output)
 {
+    if (getenv("WAAP_DIR")) {
+        return string("CloudGuard WAF Gateway");
+    }
+
     if (!command_output.empty()) {
         if (command_output[0] == '1') return string("management");
         if (command_output[0] == '0') return string("gateway");

components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h

@@ -39,14 +39,10 @@ SHELL_PRE_CMD("read sdwan data",
 #ifdef SHELL_CMD_HANDLER
 #if defined(gaia) || defined(smb)
 SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
-SHELL_CMD_HANDLER("isCpviewRunning",
+SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
-    "pidof cpview_api_service > /dev/null 2>&1 && [ -f $CPDIR/conf/cpview_api_service.version ] "
+    "[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] "
-    "&& echo 'true' || echo 'false'",
+    "&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
-    checkIsCpviewRunning)
+    checkIsInstallHorizonTelemetrySucceeded)
-SHELL_CMD_HANDLER("isCPotelcolGRET64",
-    "grep -A 10 '(BUNDLE_CPOTELCOL_AUTOUPDATE' ${CPDIR}/registry/HKLM_registry.data | "
-    "awk '/SU_Build_Take/{val = substr($2, 2, length($2)-2); if (val >=64) print \"true\"; else print \"false\" }'",
-    checkIsCPotelcolGRET64)
 SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
 SHELL_CMD_HANDLER(
     "canUpdateSDWanData",

components/security_apps/orchestration/health_check/health_check.cc

@@ -18,6 +18,7 @@
 #include <sys/socket.h>
 #include <unordered_map>
 
+#include "reverse_proxy_defaults.h"
 #include "config.h"
 #include "log_generator.h"
 #include "health_check_manager.h"
@@ -28,6 +29,8 @@ using namespace ReportIS;
 
 USE_DEBUG_FLAG(D_HEALTH_CHECK);
 
+static const std::string all_services_running = "/tmp/wd.all_running";
+
 class HealthChecker::Impl
 {
 public:
@@ -36,6 +39,7 @@ public:
     {
         i_mainloop = Singleton::Consume<I_MainLoop>::by<HealthChecker>();
         i_socket = Singleton::Consume<I_Socket>::by<HealthChecker>();
+        i_orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<HealthChecker>();
         initConfig();
         initServerSocket();
 
@@ -255,35 +259,33 @@ private:
     {
         if (!getenv("DOCKER_RPM_ENABLED")) return HealthCheckStatus::IGNORED;
 
-        static const string standalone_cmd = "/usr/sbin/cpnano -s --docker-rpm; echo $?";
+        if (i_orchestration_status->getPolicyVersion().empty()) {
-        static int timeout_tolerance = 1;
+            dbgTrace(D_HEALTH_CHECK) << "Policy version is empty, returning unhealthy status";
-        static HealthCheckStatus health_status = HealthCheckStatus::HEALTHY;
-
-        dbgTrace(D_HEALTH_CHECK) << "Checking the standalone docker health status with command: " << standalone_cmd;
-
-        auto maybe_result = Singleton::Consume<I_ShellCmd>::by<HealthChecker>()->getExecOutput(standalone_cmd, 5000);
-        if (!maybe_result.ok()) {
-            if (maybe_result.getErr().find("Reached timeout") != string::npos) {
-                dbgWarning(D_HEALTH_CHECK)
-                    << "Reached timeout while querying standalone health status, attempt number: "
-                    << timeout_tolerance;
-
-                return health_status == HealthCheckStatus::UNHEALTHY || timeout_tolerance++ > 3 ?
-                    HealthCheckStatus::UNHEALTHY :
-                    health_status;
-            }
-
-            dbgWarning(D_HEALTH_CHECK) << "Unable to get the standalone docker status. Returning unhealthy status.";
             return HealthCheckStatus::UNHEALTHY;
         }
-        dbgTrace(D_HEALTH_CHECK) << "Got response: " << maybe_result.unpack();
 
-        auto response = NGEN::Strings::removeTrailingWhitespaces(maybe_result.unpack());
+        if (!NGEN::Filesystem::exists(all_services_running)) {
+            dbgTrace(D_HEALTH_CHECK) << all_services_running << " does not exist, returning unhealthy status";
+            return HealthCheckStatus::UNHEALTHY;
+        }
 
-        if (response.back() == '1') return health_status = HealthCheckStatus::UNHEALTHY;
+        if (NGEN::Filesystem::exists(rpm_full_load_path)) {
+            dbgTrace(D_HEALTH_CHECK) << rpm_full_load_path << " exists, returning healthy status";
+            return HealthCheckStatus::HEALTHY;
+        }
 
-        timeout_tolerance = 1;
+        if (NGEN::Filesystem::exists(rpm_partial_load_path)) {
-        return health_status = (response.back() == '0') ? HealthCheckStatus::HEALTHY : HealthCheckStatus::DEGRADED;
+            dbgTrace(D_HEALTH_CHECK) << rpm_partial_load_path << " exists, returning degraded status";
+            return HealthCheckStatus::DEGRADED;
+        }
+
+        if (!NGEN::Filesystem::exists(first_rpm_policy_load_path)) {
+            dbgTrace(D_HEALTH_CHECK) << "Could not load latest RPM policy, returning degraded status";
+            return HealthCheckStatus::DEGRADED;
+        }
+
+        dbgTrace(D_HEALTH_CHECK) << "RPM is not loaded, returning unhealthy status";
+        return HealthCheckStatus::UNHEALTHY;
     }
 
     bool
@@ -390,7 +392,7 @@ private:
 
                     if (standalone_status == HealthCheckStatus::UNHEALTHY) {
                         dbgDebug(D_HEALTH_CHECK)
-                            << "Standalone status in unhealthy, returning the following response: "
+                            << "Standalone status is unhealthy, returning the following response: "
                             << failure_response;
                         i_socket->writeData(curr_client_socket, failure_response_buffer);
                         closeCurrentSocket(curr_client_socket, curr_routine_id);
@@ -439,6 +441,7 @@ private:
     I_MainLoop *i_mainloop                          = nullptr;
     I_Socket *i_socket                              = nullptr;
     I_Health_Check_Manager *i_health_check_manager  = nullptr;
+    I_OrchestrationStatus *i_orchestration_status  = nullptr;
 };
 
 HealthChecker::HealthChecker() : Component("HealthChecker"), pimpl(make_unique<Impl>()) {}

components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc

@@ -7,6 +7,7 @@
 #include "mock/mock_socket_is.h"
 #include "mock/mock_mainloop.h"
 #include "mock/mock_shell_cmd.h"
+#include "mock/mock_orchestration_status.h"
 #include "health_check_manager.h"
 
 #include "config.h"
@@ -65,6 +66,7 @@ public:
     AgentDetails                        agent_details;
     StrictMock<MockSocketIS>            mock_socket;
     NiceMock<MockShellCmd>              mock_shell_cmd;
+    NiceMock<MockOrchestrationStatus>   mock_orchestration_status;
     I_Socket::socketFd                  server_socket = -1;
     Context                             ctx;
     ConfigComponent                     config;
@@ -72,7 +74,6 @@ public:
     I_MainLoop::Routine                 connection_handler_routine;
     I_MainLoop::Routine                 client_connection_handler_routine;
     I_MainLoop::Routine                 handle_probe_routine;
-    //StrictMock<MockHealthCheckManager>  mock_health_check_manager;
     HealthCheckManager                  health_check_manager;
     I_Health_Check_Manager              *i_health_check_manager;
 };

components/security_apps/orchestration/include/fog_authenticator.h

@@ -102,6 +102,7 @@ public:
 
     Maybe<void> authenticateAgent() override;
     void setAddressExtenesion(const std::string &extension) override;
+    static std::string getUserEdition();
 
 protected:
     class UserCredentials
@@ -142,7 +143,6 @@ protected:
     std::string base64Encode(const std::string &in) const;
     std::string buildBasicAuthHeader(const std::string &username, const std::string &pass) const;
     std::string buildOAuth2Header(const std::string &token) const;
-    std::string getUserEdition() const;
 
     // This apps which the orchestrations requires them from Fog.
     std::vector<std::string> required_security_apps;

components/security_apps/orchestration/include/hybrid_communication.h

@@ -56,7 +56,9 @@ public:
 
 private:
     Maybe<std::string> getNewVersion();
+    void doLocalFogOperations(const std::string &policy) const;
 
+    std::string tuning_host;
     I_DeclarativePolicy *i_declarative_policy = nullptr;
 };
 

components/security_apps/orchestration/orchestration_comp.cc

@@ -41,6 +41,7 @@
 #include "env_details.h"
 #include "hybrid_communication.h"
 #include "agent_core_utilities.h"
+#include "fog_communication.h"
 
 using namespace std;
 using namespace chrono;
@@ -190,7 +191,8 @@ public:
         );
 
         auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<OrchestrationComp>();
-        orchestration_tools->getClusterId();
+
+        if (getAttribute("no-setting", "IGNORE_CLUSTER_ID") != "TRUE") orchestration_tools->getClusterId();
 
         hybrid_mode_metric.init(
             "Watchdog Metrics",
@@ -1484,6 +1486,8 @@ private:
             agent_data_report << AgentReportFieldWithLabel("managedMode", "management");
         }
 
+        agent_data_report << AgentReportFieldWithLabel("userEdition", FogCommunication::getUserEdition());
+
 #if defined(gaia) || defined(smb)
         if (i_details_resolver->compareCheckpointVersion(8100, greater_equal<int>())) {
             agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER81", "true");

components/security_apps/orchestration/update_communication/fog_helper_open_source.cc

@@ -3,7 +3,7 @@
 #include <string>
 
 std::string
-FogAuthenticator::getUserEdition() const
+FogAuthenticator::getUserEdition()
 {
     return "community";
 }

components/security_apps/orchestration/update_communication/hybrid_communication.cc

@@ -27,15 +27,19 @@ using namespace std;
 
 USE_DEBUG_FLAG(D_ORCHESTRATOR);
 
-#define TUNING_HOST_ENV_NAME "TUNING_HOST"
+static const string agent_resource_api = "/api/v2/agents/resources";
-static const string defaultTuningHost = "appsec-tuning-svc";
 
 void
 HybridCommunication::init()
 {
+    dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component";
+
     FogAuthenticator::init();
     i_declarative_policy = Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>();
-    dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component";
+
+    auto env_tuning_host = getenv("TUNING_HOST");
+    tuning_host = env_tuning_host != nullptr ? env_tuning_host : "appsec-tuning-svc";
+
     if (getConfigurationFlag("otp") != "") {
         otp = getConfigurationFlag("otp");
     } else {
@@ -49,10 +53,9 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request)
     string manifest_checksum = "";
     dbgTrace(D_ORCHESTRATOR) << "Getting updates in Hybrid Communication";
     if (access_token.ok()) {
-        static const string check_update_str = "/api/v2/agents/resources";
         auto request_status = Singleton::Consume<I_Messaging>::by<HybridCommunication>()->sendSyncMessage(
             HTTPMethod::POST,
-            check_update_str,
+            agent_resource_api,
             request
         );
 
@@ -78,45 +81,7 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request)
 
     string policy_response = i_declarative_policy->getUpdate(request);
 
-    auto env = Singleton::Consume<I_EnvDetails>::by<HybridCommunication>()->getEnvType();
+    doLocalFogOperations(policy_response);
-    if (env == EnvType::K8S && !policy_response.empty()) {
-        dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host";
-        I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<HybridCommunication>();
-
-        auto get_tuning_host = []()
-            {
-                static string tuning_host;
-                if (tuning_host != "") return tuning_host;
-
-                char* tuning_host_env = getenv(TUNING_HOST_ENV_NAME);
-                if (tuning_host_env != NULL) {
-                    tuning_host = string(tuning_host_env);
-                    return tuning_host;
-                }
-                dbgWarning(D_ORCHESTRATOR) << "tuning host is not set. using default";
-                tuning_host = defaultTuningHost;
-
-                return tuning_host;
-            };
-
-        MessageMetadata update_policy_crd_md(get_tuning_host(), 80);
-        update_policy_crd_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
-        UpdatePolicyCrdObject policy_change_object(policy_response);
-
-        auto i_messaging = Singleton::Consume<I_Messaging>::by<HybridCommunication>();
-        bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse(
-            HTTPMethod::POST,
-            "/api/update-policy-crd",
-            policy_change_object,
-            MessageCategory::GENERIC,
-            update_policy_crd_md
-        );
-        if (!tuning_req_status) {
-            dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification";
-        } else {
-            dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification";
-        }
-    }
 
     request = CheckUpdateRequest(manifest_checksum, policy_response, "", "", "", "");
 
@@ -141,10 +106,9 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file,
 
         auto unpacked_access_token = access_token.unpack().getToken();
 
-        static const string file_attribute_str = "/api/v2/agents/resources/";
         auto attribute_file = Singleton::Consume<I_Messaging>::by<HybridCommunication>()->downloadFile(
             resourse_file.getRequestMethod(),
-            file_attribute_str + resourse_file.getFileName(),
+            agent_resource_api + '/' + resourse_file.getFileName(),
             file_path
         );
         if (!attribute_file.ok()) {
@@ -164,3 +128,33 @@ HybridCommunication::sendPolicyVersion(const string &policy_version, const strin
     policy_version.empty();
     return Maybe<void>();
 }
+
+void
+HybridCommunication::doLocalFogOperations(const string &policy) const
+{
+    if (policy.empty()) return;
+    if (Singleton::Consume<I_EnvDetails>::by<HybridCommunication>()->getEnvType() != EnvType::K8S) return;
+
+    dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host";
+
+    MessageMetadata update_policy_crd_md(tuning_host, 80);
+    const auto &tenant_id = Singleton::Consume<I_AgentDetails>::by<HybridCommunication>()->getTenantId();
+    update_policy_crd_md.insertHeader("X-Tenant-Id", tenant_id);
+
+    UpdatePolicyCrdObject policy_change_object(policy);
+    auto i_messaging = Singleton::Consume<I_Messaging>::by<HybridCommunication>();
+    bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse(
+        HTTPMethod::POST,
+        "/api/update-policy-crd",
+        policy_change_object,
+        MessageCategory::GENERIC,
+        update_policy_crd_md
+    );
+
+    if (!tuning_req_status) {
+        dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification";
+    } else {
+        dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification";
+    }
+
+}

components/security_apps/waap/waap_clib/CMakeLists.txt

@@ -85,6 +85,7 @@ add_library(waap_clib
     ParserGql.cc
     ParserPercentEncode.cc
     ParserPairs.cc
+    Waf2Util2.cc
 )
 
 add_definitions("-Wno-unused-function")

components/security_apps/waap/waap_clib/DeepParser.cc

@@ -864,10 +864,6 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
     return rc;
 }
 
-static bool err = false;
-static const SingleRegex json_detector_re("\\A[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", err, "json_detector");
-static const SingleRegex json_quoteless_detector_re("^[{\\[][[,0-9nul\\]]+", err, "json_quoteless_detector");
-
 //intended to keep and process all types of leftovers detected as separate cases for parsing
 int
 DeepParser::createUrlParserForJson(
@@ -1103,11 +1099,7 @@ DeepParser::createInternalParser(
         } else {
             dbgTrace(D_WAAP_DEEP_PARSER) << "attempt to find JSON by '{' or '['";
             bool percent_encoded_doublequote_detected = cur_val.find("%22") != std::string::npos;
-            if (json_detector_re.hasMatch(cur_val)
+            if (Waap::Util::isValidJson(cur_val)) {
-                && (valueStats.hasDoubleQuote
-                    || json_quoteless_detector_re.hasMatch(cur_val)
-                    || percent_encoded_doublequote_detected)) {
-                // JSON value detected
                 if (percent_encoded_doublequote_detected && !valueStats.hasDoubleQuote) {
                     // We have JSOn but it %-encoded, first start percent decoding for it. Very narrow case
                     dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file from percent decoding";

components/security_apps/waap/waap_clib/ParserBase.cc

@@ -19,7 +19,8 @@ USE_DEBUG_FLAG(D_WAAP_PARSER);
 
 // Max size for key and value that can be stored in memory (per thread)
 #define MAX_KEY_SIZE 64*1024
-#define MAX_VALUE_SIZE 64*1024
+#define MAX_VALUE_SIZE 16*1024
+#define MAX_PROCESSING_BUFFER_SIZE 64*1024
 
 BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) :
     m_receiver(receiver),
@@ -69,14 +70,14 @@ int BufferedReceiver::onValue(const char *v, size_t v_len)
         << m_parser_depth;
     while (v_len > 0) {
         // Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size
-        size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size());
+        size_t bytesToFill = std::min(v_len, MAX_PROCESSING_BUFFER_SIZE - m_value.size());
         m_value += std::string(v, bytesToFill);
         // Update v and v_len (input buffer) to reflect that we already consumed part (or all) of it
         v += bytesToFill;
         v_len -= bytesToFill;
 
         // Only push full buffers to the m_receiver
-        if (m_value.size() == MAX_VALUE_SIZE) {
+        if (m_value.size() >= MAX_VALUE_SIZE) {
             // The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag
             dbgTrace(D_WAAP_PARSER)
                 << " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag"

components/security_apps/waap/waap_clib/ParserJson.h

@@ -22,8 +22,9 @@
 #include "yajl/yajl_parse.h"
 #include "singleton.h"
 #include "i_oa_schema_updater.h"
-
+// must be at least 4 first bytes to allow unicode autodetection (BOM).
-#define FIRST_JSON_BUFFER_SIZE 4 // must buffer at least 4 first bytes to allow unicode autodetection (BOM).
+// BUT... reduced to 1 in order to allow better work of schema validation and API discovery
+#define FIRST_JSON_BUFFER_SIZE 1
 
 typedef size_t yajl_size_t;
 

components/security_apps/waap/waap_clib/Waf2Engine.cc

@@ -312,6 +312,7 @@ Waf2Transaction::Waf2Transaction() :
     m_responseInjectReasons(),
     m_index(-1),
     m_triggerLog(),
+    is_schema_validation(false),
     m_waf2TransactionFlags()
 {}
 
@@ -343,6 +344,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
     m_responseInjectReasons(),
     m_index(-1),
     m_triggerLog(),
+    is_schema_validation(false),
     m_waf2TransactionFlags()
 {}
 
@@ -515,7 +517,6 @@ bool Waf2Transaction::checkIsScanningRequired()
         m_siteConfig = &m_ngenAPIConfig;
         auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
         result |= m_siteConfig->get_WebAttackMitigation();
-
         if(rateLimitingPolicy) {
             result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus();
         }

components/security_apps/waap/waap_clib/Waf2Engine.h

@@ -345,6 +345,7 @@ private:
 
     // Cached pointer to const triggerLog (hence mutable)
     mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
+    bool is_schema_validation = false;
     Waf2TransactionFlags m_waf2TransactionFlags;
 };
 

components/security_apps/waap/waap_clib/Waf2Util.cc

@@ -2055,31 +2055,18 @@ string extractForwardedIp(const string &x_forwarded_hdr_val)
     return forward_ip;
 }
 
-
 bool isUuid(const string& str) {
     if (str.length() != 36) {
         return false;
     }
-
     static bool err;
     static const SingleRegex uuid_detector_re(
         "[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}",
         err,
         "uuid_detector"
     );
-        // Check if the string matches the UUID format
+    // Check if the string matches the UUID format
     return uuid_detector_re.hasMatch(str);
-/*
-    boost::cmatch what;
-    try {
-        static const boost::regex uuidRegex("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}");
-        // Check if the string matches the UUID format
-        return boost::regex_match(str.c_str(), what, uuidRegex);
-    } catch (std::runtime_error &e) {
-        dbgError(D_WAAP) << e.what();
-    }
-    return false;
-*/
 }
 
 bool isIpAddress(const string &ip_address)

components/security_apps/waap/waap_clib/Waf2Util.h

@@ -876,6 +876,8 @@ namespace Util {
         std::string::const_iterator e,
         std::string &repl);
 
+    bool isValidJson(const std::string &input);
+
     bool detectJSONasParameter(const std::string &s,
             std::string &key,
             std::string &value);

components/security_apps/waap/waap_clib/Waf2Util2.cc

@@ -0,0 +1,632 @@
+#include "Waf2Util.h"
+#include <string>
+
+namespace Waap {
+namespace Util {
+using namespace std;
+
+static const char *trueString = "true";
+static const size_t trueStringLen = strlen(trueString);
+static const char *falseString = "false";
+static const size_t falseStringLen = strlen(falseString);
+static const char *nullString = "null";
+static const size_t nullStringLen = strlen(nullString);
+static const char *quoteString = "%22";
+static const size_t quoteStringLen = strlen(quoteString);
+
+int
+isAlignedPrefix(
+    const char *sample,
+    const size_t sample_len,
+    const char *buffer,
+    const size_t buffer_len)
+{
+    size_t lookup_len = 0;
+    if (buffer_len < sample_len) {
+        lookup_len = buffer_len;
+    } else {
+        lookup_len = sample_len;
+    }
+    if (strncmp(sample, buffer, lookup_len) == 0) {
+        return lookup_len;
+    }
+    return -1;
+}
+
+int
+isBoolean(
+    const char *buffer,
+    const size_t buffer_len)
+{
+    int status;
+
+    status = isAlignedPrefix(trueString, trueStringLen, buffer, buffer_len);
+    if (status >= 0) {
+        return status;
+    }
+    status = isAlignedPrefix(falseString, falseStringLen, buffer, buffer_len);
+    if (status >= 0) {
+        return status;
+    }
+    status = isAlignedPrefix(nullString, nullStringLen, buffer, buffer_len);
+    if (status >= 0) {
+        return status;
+    }
+    return -1;
+}
+
+bool
+isValidExponent(
+    const char * buffer,
+    const size_t buffer_len,
+    size_t *i)
+{
+    if (buffer_len == *i + 1) {
+        return true; // e or E is the last char in buffer
+    }
+    if (*i + 1 < buffer_len && (isdigit(buffer[*i + 1]) || (buffer[*i + 1] == '+' || buffer[*i + 1] == '-'))) {
+        (*i) += 1;
+        if (isdigit(buffer[*i + 1])) {
+            return true;
+        }
+    } else {
+        return false;
+    }
+    return false;
+}
+
+bool
+isObjectStart(const char c, int *object_count)
+{
+    if (c == '{') {
+        (*object_count)++;
+        return true;
+    }
+    return false;
+}
+
+bool
+isObjectEnd(const char c, int *object_count)
+{
+    if (c == '}') {
+        (*object_count)--;
+        return true;
+    }
+    return false;
+}
+
+bool
+isArrayStart(const char c, int *array_count)
+{
+    if (c == '[') {
+        (*array_count)++;
+        return true;
+    }
+    return false;
+}
+
+bool
+isArrayEnd(const char c, int *array_count)
+{
+    if (c == ']') {
+        (*array_count)--;
+        return true;
+    }
+    return false;
+}
+
+bool
+isValidJson(const std::string &input)
+{
+    static const size_t MAX_JSON_INSPECT_SIZE = 16;
+
+    enum state
+    {
+        S_START,          // 0
+        S_OBJECT_START,   // 1
+        S_OBJECT_END,     // 2
+        S_ARRAY_START,    // 3
+        S_ARRAY_END,      // 4
+        S_NUMBER,         // 5
+        S_NUMBER_END,     // 6
+        S_STRING_START,   // 7
+        S_STRING_BODY,    // 8
+        S_STRING_END,     // 9
+        S_VARIABLE_START, // 10
+        S_VARIABLE_BODY,  // 11
+        S_VARIABLE_END,   // 12
+        S_COMMA,          // 13
+        S_COLON,          // 14
+        S_BOOLEAN,        // 15
+        S_ERROR,          // 16
+        S_END             // 17
+    };
+
+    state m_state;
+    bool encoded = false;
+    size_t i = 0;
+    char c;
+    const char *buf = input.c_str();
+    size_t len = input.length();
+    int array_count = 0;
+    int object_count = 0;
+    int status;
+    if (len < 2) {
+        return false;
+    }
+
+    m_state = S_START;
+    while (i < len && i < MAX_JSON_INSPECT_SIZE) {
+        c = buf[i];
+        if (c == 0x0) { // UTF16 to UTF8 support
+            i++;
+            continue;
+        }
+        switch (m_state) {
+            case S_START:
+                if (isObjectStart(c, &object_count)) {
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) {
+                    m_state = S_ARRAY_START;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_START
+
+            case S_OBJECT_START:
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                if (c == '\"') {
+                    m_state = S_VARIABLE_START;
+                    break;
+                }
+                if (isspace(c)) {
+                    break;
+                }
+                status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_VARIABLE_START;
+                    encoded = true;
+                    i += status - 1;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // object_start
+
+            case S_ARRAY_START:
+                if (isObjectStart(c, &object_count)) {
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) {
+                    // keep state unchanged
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isdigit(c)) {
+                    m_state = S_NUMBER;
+                    break;
+                }
+                if (c == '-') {
+                    if (i + 1 == len) { // End of buffer case
+                        m_state = S_NUMBER;
+                        break;
+                    }
+                    if (i + 1 < len && isdigit(buf[i + 1])) {
+                        m_state = S_NUMBER;
+                        i++;
+                        break;
+                    }
+                    m_state = S_ERROR;
+                    break;
+                }
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == '\"') {
+                    m_state = S_STRING_START;
+                    break;
+                }
+                status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_STRING_START;
+                    encoded = true;
+                    i += status - 1;
+                    break;
+                } else {
+                    m_state = S_ERROR;
+                }
+                status = isBoolean(buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_BOOLEAN;
+                    i += status - 1;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // array_start
+
+            case S_OBJECT_END:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org
+                    m_state = S_ARRAY_START;
+                    break;
+                }
+                if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_OBJECT_END
+
+            case S_ARRAY_END:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org
+                    m_state = S_ARRAY_START;
+                    break;
+                }
+                if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_ARRAY_END
+
+            case S_NUMBER:
+                if (isdigit(c)) {
+                    break;
+                }
+                if (c == '.') {
+                    if (i + 1 == len) { // End of buffer case
+                        m_state = S_NUMBER;
+                        break;
+                    }
+                    if (i + 1 < len && isdigit(buf[i + 1])) {
+                        m_state = S_NUMBER;
+                        i++;
+                        break;
+                    }
+                    m_state = S_ERROR;
+                    break;
+                }
+                if (c == 'e' || c == 'E') {
+                    if (isValidExponent(buf, len, &i)) {
+                        m_state = S_NUMBER;
+                        break;
+                    }
+                    m_state = S_ERROR;
+                    break;
+                }
+                if (isspace(c)) {
+                    m_state = S_NUMBER_END;
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_NUMBER
+
+            case S_NUMBER_END:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_NUMBER_END
+
+            case S_STRING_START:
+                if (c == '\"') {
+                    m_state = S_STRING_END;
+                    break;
+                }
+                if (encoded) { // url_encoded quote
+                    status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                    if (status >= 0) {
+                        m_state = S_STRING_END;
+                        encoded = true;
+                        i += status - 1;
+                        break;
+                    } else {
+                        m_state = S_ERROR;
+                    }
+                }
+                m_state = S_STRING_BODY;
+                break; // S_STRING_START
+
+            case S_STRING_BODY:
+                if (c == '\"') {
+                    if (buf[i - 1] == '\\' && buf[i - 2] != '\\') {
+                        m_state = S_STRING_BODY;
+                        break;
+                    } else {
+                        m_state = S_STRING_END;
+                        break;
+                    }
+                }
+                if (encoded) { // url_encoded quote
+                    status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                    if (status >= 0) {
+                        m_state = S_STRING_END;
+                        encoded = true;
+                        i += status - 1;
+                        break;
+                    } else {
+                        m_state = S_ERROR;
+                    }
+                }
+                m_state = S_STRING_BODY;
+                break; // S_STRING_BODY;
+
+            case S_STRING_END:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                if (c == ':') {
+                    m_state = S_COLON;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // s_sting_end
+
+            case S_VARIABLE_START:
+                if (c == '\"') {
+                    m_state = S_VARIABLE_END;
+                    break;
+                }
+                if (encoded) { // url_encoded quote
+                    status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                    if (status >= 0) {
+                        m_state = S_VARIABLE_END;
+                        encoded = true;
+                        i += status - 1;
+                        break;
+                    } else {
+                        m_state = S_ERROR;
+                    }
+                }
+                m_state = S_VARIABLE_BODY;
+                break; // S_VARIABLE_START
+
+            case S_VARIABLE_BODY:
+                if (c == '\"') {
+                    if (buf[i - 1] == '\\' && buf[i - 2] != '\\') {
+                        m_state = S_VARIABLE_BODY;
+                        break;
+                    } else {
+                        m_state = S_VARIABLE_END;
+                        break;
+                    }
+                }
+                if (encoded) { // url_encoded quote
+                    status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                    if (status >= 0) {
+                        m_state = S_VARIABLE_END;
+                        encoded = true;
+                        i += status - 1;
+                        break;
+                    } else {
+                        m_state = S_ERROR;
+                    }
+                }
+                m_state = S_VARIABLE_BODY;
+                break; // S_VARIABLE_BODY
+
+            case S_VARIABLE_END:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ':') {
+                    m_state = S_COLON;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_VARIABLE_END
+
+            case S_COMMA:
+                if (isObjectStart(c, &object_count)) {
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) {
+                    m_state = S_ARRAY_START;
+                    break;
+                }
+                if (isdigit(c)) {
+                    m_state = S_NUMBER;
+                    break;
+                }
+                if (c == '-') {
+                    if (i + 1 == len) { // End of buffer case
+                        m_state = S_NUMBER;
+                        break;
+                    }
+                    if (i + 1 < len && isdigit(buf[i + 1])) {
+                        m_state = S_NUMBER;
+                        i++;
+                        break;
+                    }
+                    m_state = S_ERROR;
+                    break;
+                }
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == '\"') {
+                    m_state = S_STRING_START;
+                    break;
+                }
+                status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_STRING_START;
+                    encoded = true;
+                    i += status - 1;
+                    break;
+                } else {
+                    m_state = S_ERROR;
+                }
+                status = isBoolean(buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_BOOLEAN;
+                    i += status - 1;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_COMMA
+
+            case S_COLON:
+                if (isObjectStart(c, &object_count)) {
+                    m_state = S_OBJECT_START;
+                    break;
+                }
+                if (isArrayStart(c, &array_count)) {
+                    m_state = S_ARRAY_START;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isdigit(c)) {
+                    m_state = S_NUMBER;
+                    break;
+                }
+                if (c == '-') {
+                    if (i + 1 == len) { // End of buffer case
+                        m_state = S_NUMBER;
+                        break;
+                    }
+                    if (i + 1 < len && isdigit(buf[i + 1])) {
+                        m_state = S_NUMBER;
+                        i++;
+                        break;
+                    }
+                    m_state = S_ERROR;
+                    break;
+                }
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == '\"') {
+                    m_state = S_STRING_START;
+                    break;
+                }
+                status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_STRING_START;
+                    encoded = true;
+                    i += status - 1;
+                    break;
+                } else {
+                    m_state = S_ERROR;
+                }
+                status = isBoolean(buf + i, len - i);
+                if (status >= 0) {
+                    m_state = S_BOOLEAN;
+                    i += status - 1;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_COLON
+
+            case S_BOOLEAN:
+                if (isspace(c)) {
+                    break;
+                }
+                if (c == ',') {
+                    m_state = S_COMMA;
+                    break;
+                }
+                if (isArrayEnd(c, &array_count)) {
+                    m_state = S_ARRAY_END;
+                    break;
+                }
+                if (isObjectEnd(c, &object_count)) {
+                    m_state = S_OBJECT_END;
+                    break;
+                }
+                m_state = S_ERROR;
+                break; // S_BOOLEAN
+
+            case S_ERROR: break;
+            case S_END: break;
+        }
+        if (m_state == S_ERROR) {
+            return false;
+        }
+        i++;
+    }
+
+    if (m_state != S_ERROR && array_count >= 0 && object_count >= 0)
+        return true;
+    return false;
+}
+
+} // namespace Util
+} // namespace Waap

components/utils/CMakeLists.txt

@@ -1,4 +1,4 @@
-#add_subdirectory(geo_location)
+add_subdirectory(geo_location)
 add_subdirectory(http_transaction_data)
 add_subdirectory(ip_utilities)
 add_subdirectory(keywords)

core/agent_core_utilities/agent_core_utilities.cc

@@ -20,6 +20,7 @@
 #include <unistd.h>
 #include <sys/syscall.h>
 #include <dirent.h>
+#include <fstream>
 #include <sstream>
 #include <algorithm>
 
@@ -244,6 +245,22 @@ deleteDirectory(const string &path, bool delete_content)
     return res;
 }
 
+bool
+touchFile(const string &path)
+{
+    dbgFlow(D_INFRA_UTILS) << "Trying to touch file, path: " << path;
+
+    ofstream t_file(path);
+    if (!t_file.is_open()) {
+        dbgDebug(D_INFRA_UTILS) << "Failed to touch file. Path: " << path;
+        return false;
+    }
+
+    t_file.close();
+    dbgTrace(D_INFRA_UTILS) << "Successfully touched file, path: " << path;
+    return true;
+}
+
 string
 convertToHumanReadable(uint64_t size_in_bytes)
 {

core/include/services_sdk/interfaces/i_intelligence_is_v2.h

@@ -17,17 +17,16 @@
 #include <chrono>
 #include <string>
 
-#include "maybe_res.h"
+#include "intelligence_is_v2/asset_reply.h"
-#include "i_messaging.h"
+#include "intelligence_is_v2/intelligence_response.h"
-#include "i_time_get.h"
-#include "i_mainloop.h"
 #include "intelligence_is_v2/intelligence_types_v2.h"
-#include "intelligence_is_v2/intelligence_query_v2.h"
+#include "intelligence_is_v2/query_request_v2.h"
-#include "config.h"
+#include "maybe_res.h"
 
 namespace Intelligence {
 
 class Invalidation;
+class Response;
 
 } // namespace Intelligence
 
@@ -40,315 +39,27 @@ public:
         const std::function<void(const Intelligence::Invalidation &)> &callback
     ) = 0;
     virtual void unregisterInvalidation(uint id) = 0;
+    virtual Maybe<Intelligence::Response>
+    getResponse(
+        const std::vector<QueryRequest> &query_requests,
+        bool is_pretty,
+        bool is_bulk
+    ) const = 0;
+
+    virtual Maybe<Intelligence::Response> getResponse(const QueryRequest &query_request, bool is_pretty) const = 0;
 
     template<typename Data>
     Maybe<std::vector<AssetReply<Data>>>
-    queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true)
+    queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true);
-    {
-        uint assets_limit = query_request.getAssetsLimit();
-        static const uint upper_assets_limit = 50;
-        if (assets_limit == 0 || assets_limit > upper_assets_limit) {
-            return genError("Assets limit must be in the range of [1, " + std::to_string(upper_assets_limit) + "]");
-        }
-
-        static const uint upper_confidence_limit = 1000;
-        bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
-        if (!min_conf_res) {
-            return genError(
-                "Minimum confidence value must be in the range of [1, " + std::to_string(upper_confidence_limit) + "]"
-            );
-        }
-
-        if (query_request.isPagingActivated() && query_request.isPagingFinished()) {
-            return genError("Paging is activated and already finished. No need for more queries.");
-        }
-
-        IntelligenceQuery<Data> intelligence_query(query_request, is_pretty);
-        static const std::string query_uri = "/api/v2/intelligence/assets/query";
-
-        bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit);
-        if (!res) {
-            dbgTrace(D_INTELLIGENCE) << "Could not message fog, trying to get offline intelligence.";
-            Maybe<std::string> offline_res = getOfflineInfoString(query_request.getQuery());
-            if (!offline_res.ok()) {
-                dbgDebug(D_INTELLIGENCE) << "Offline intelligence error: " << offline_res.getErr();
-                return genError("Could not query intelligence");
-            }
-            if (!intelligence_query.loadJson(offline_res.unpack())) {
-                dbgWarning(D_INTELLIGENCE) << "Offline intelligence error: invalid JSON for requested asset";
-                return genError("Could not query intelligence");
-            }
-        }
-
-        if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
-            return genError("Query intelligence response with InProgress status");
-        }
-        return intelligence_query.getData();
-    }
 
     template<typename Data>
     Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
-    queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true)
+    queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true);
-    {
-        static const uint upper_assets_limit = 50;
-        static const uint upper_confidence_limit = 1000;
-        for (QueryRequest &query_request : query_requests) {
-            uint assets_limit = query_request.getAssetsLimit();
-            if (assets_limit == 0 || assets_limit > upper_assets_limit) {
-                dbgTrace(D_INTELLIGENCE)
-                    << "Assets limit for request is "
-                    << upper_assets_limit
-                    << ", requests assets: "
-                    << assets_limit;
-                return genError("Assets limit valid range is of [1, " + std::to_string(upper_assets_limit) + "]");
-            }
 
-            bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
+protected:
-            if (!min_conf_res) {
+    virtual ~I_Intelligence_IS_V2() {}
-                dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
-                return genError(
-                    "Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
-                );
-            }
-        }
-        IntelligenceQuery<Data> intelligence_query(query_requests, is_pretty);
-        static const std::string query_uri = "/api/v2/intelligence/assets/queries";
-
-        dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items";
-        bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, upper_assets_limit);
-        if (!res) {
-            dbgTrace(D_INTELLIGENCE) << "Could not message fog, bulk request failed.";
-            return genError("Could not query intelligence");
-        }
-
-        return intelligence_query.getBulkData();
-    }
-
-private:
-    template<typename Data>
-    bool
-    sendMessage(
-        IntelligenceQuery<Data> &intelligence_query,
-        const std::string &query_uri,
-        I_Messaging *i_message,
-        Flags<MessageConnectionConfig> conn_flags,
-        const std::string &ip,
-        uint server_port
-    ) {
-        if (ip == "" && server_port == 0) {
-            auto req_status = i_message->sendSyncMessage(
-                HTTPMethod::POST,
-                query_uri,
-                intelligence_query,
-                MessageCategory::INTELLIGENCE
-            );
-            return req_status.ok();
-        }
-
-        dbgTrace(D_INTELLIGENCE)
-            << "Sending intelligence request with IP: "
-            << ip
-            << " port: "
-            << server_port
-            << " query_uri: "
-            << query_uri;
-        MessageMetadata req_md(ip, server_port, conn_flags);
-        auto req_status = i_message->sendSyncMessage(
-            HTTPMethod::POST,
-            query_uri,
-            intelligence_query,
-            MessageCategory::INTELLIGENCE,
-            req_md
-        );
-        return req_status.ok();
-    }
-
-    template<typename Data>
-    bool
-    sendQueryMessage(
-        IntelligenceQuery<Data> &intelligence_query,
-        const std::string &query_uri,
-        I_Messaging *i_message,
-        Flags<MessageConnectionConfig> conn_flags,
-        const std::string &ip = "",
-        uint server_port = 0
-    ) {
-        auto i_timer = getTimer();
-        auto i_mainloop = getMainloop();
-
-        uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
-            20,
-            "intelligence",
-            "request overall timeout"
-        );
-
-        uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
-            5,
-            "intelligence",
-            "request lap timeout"
-        );
-
-        std::chrono::seconds request_overall_timeout = std::chrono::seconds(request_overall_timeout_conf);
-        std::chrono::seconds request_lap_timeout = std::chrono::seconds(request_lap_timeout_conf);
-
-        std::chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
-        std::chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
-        std::chrono::seconds seconds_since_start = std::chrono::seconds(0);
-        std::chrono::seconds seconds_since_last_lap = std::chrono::seconds(0);
-
-        bool res = true;
-        while (res &&
-            intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS &&
-            seconds_since_start < request_overall_timeout &&
-            seconds_since_last_lap < request_lap_timeout
-        ) {
-            res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
-
-            if (res && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
-                i_mainloop->yield(true);
-            }
-
-            seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
-                i_timer->getMonotonicTime() - send_request_start_time
-            );
-
-            seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
-                i_timer->getMonotonicTime() - last_lap_time
-            );
-            last_lap_time = i_timer->getMonotonicTime();
-        }
-
-        return res;
-    }
-
-    template<typename Data>
-    bool
-    sendPagingQueryMessage(
-        IntelligenceQuery<Data> &intelligence_query,
-        const std::string &query_uri,
-        int assets_limit,
-        I_Messaging *i_message,
-        Flags<MessageConnectionConfig> conn_flags,
-        const std::string &ip = "",
-        uint server_port = 0
-    ) {
-        bool res= true;
-
-        res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
-
-        if (intelligence_query.getResponseStatus() == ResponseStatus::DONE &&
-            intelligence_query.getResponseAssetCollectionsSize() < assets_limit
-        ) {
-            intelligence_query.setRequestCursor(Intelligence_IS_V2::CursorState::DONE, "");
-        } else {
-            intelligence_query.setRequestCursor(
-                Intelligence_IS_V2::CursorState::IN_PROGRESS,
-                intelligence_query.getResponseCursorVal()
-            );
-        }
-
-        return res;
-    }
-
-// LCOV_EXCL_START Reason: one templated instance is tested in intelligence ut. the rest are tested in system tests
-    template<typename Data>
-    bool
-    sendQueryObjectToLocalServer(
-        IntelligenceQuery<Data> &intelligence_query,
-        const std::string &query_uri,
-        const std::string &ip,
-        bool is_primary_port,
-        int assets_limit,
-        I_Messaging *i_message,
-        Flags<MessageConnectionConfig> conn_flags
-    ) {
-        static const std::string primary_port_setting = "local intelligence server primary port";
-        static const std::string secondary_port_setting = "local intelligence server secondary port";
-        auto server_port = getSetting<uint>(
-            "intelligence",
-            is_primary_port ? primary_port_setting : secondary_port_setting
-        );
-
-        if (!server_port.ok()) return false;
-
-        conn_flags.reset();
-        
-        if (intelligence_query.getPagingStatus().ok()) {
-            return sendPagingQueryMessage(
-                intelligence_query,
-                query_uri,
-                assets_limit,
-                i_message,
-                conn_flags,
-                ip,
-                *server_port
-            );
-        }
-
-        return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags, ip, *server_port);
-    }
-// LCOV_EXCL_STOP
-
-    template<typename Data>
-    bool
-    sendQueryObject(IntelligenceQuery<Data> &intelligence_query, const std::string &query_uri, int assets_limit)
-    {
-        auto i_message = getMessaging();
-        Flags<MessageConnectionConfig> conn_flags;
-
-        bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
-            std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
-            false;
-
-        crowdsec_enabled = getProfileAgentSettingWithDefault<bool>(
-            crowdsec_enabled,
-            "layer7AccessControl.crowdsec.enabled"
-        );
-
-        bool use_local_intelligence = getProfileAgentSettingWithDefault<bool>(
-            false,
-            "agent.config.useLocalIntelligence"
-        );
-
-        auto server_ip = getSetting<std::string>("intelligence", "local intelligence server ip");
-        if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) {
-            if (sendQueryObjectToLocalServer(
-                    intelligence_query,
-                    query_uri,
-                    *server_ip,
-                    true,
-                    assets_limit,
-                    i_message,
-                    conn_flags
-                )
-            ) {
-                return true;
-            }
-            if (sendQueryObjectToLocalServer(
-                    intelligence_query,
-                    query_uri,
-                    *server_ip,
-                    false,
-                    assets_limit,
-                    i_message,
-                    conn_flags
-                )
-            ) {
-                return true;
-            };
-        }
-
-        if (intelligence_query.getPagingStatus().ok()) {
-            return sendPagingQueryMessage(intelligence_query, query_uri, assets_limit, i_message, conn_flags);
-        }
-
-        return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags);
-    }
-
-    virtual I_Messaging * getMessaging() const = 0;
-    virtual I_TimeGet * getTimer() const = 0;
-    virtual I_MainLoop * getMainloop() const = 0;
-    virtual Maybe<std::string> getOfflineInfoString(const SerializableQueryFilter &query) const = 0;
-    virtual bool getIsOfflineOnly() const = 0;
 };
+
+#include "intelligence_is_v2/intelligence_interface_impl.h"
+
 #endif // __I_INTELLIGENCE_IS_V2_H__

core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h

@@ -11,18 +11,16 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef __QUERY_RESPONSE_V2_IMPL_H_
+#ifndef __ASSET_REPLAY_IMPL_H__
-#define __QUERY_RESPONSE_V2_IMPL_H_
+#define __ASSET_REPLAY_IMPL_H__
 
-#ifndef __QUERY_RESPONSE_V2_H__
+#ifndef __ASSET_REPLY_H__
-#error intelligence_query_response_v2_impl.h should not be included directly!
+#error asset_replay_impl.h should not be included directly!
-#endif // __QUERY_RESPONSE_V2_H__
+#endif // __ASSET_REPLY_H__
 
-#include "debug.h"
+#include "customized_cereal_multimap.h"
 #include "intelligence_types_v2.h"
 
-USE_DEBUG_FLAG(D_INTELLIGENCE);
-
 template <typename UserSerializableReplyAttr>
 void
 AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
@@ -84,87 +82,72 @@ AssetReply<UserSerializableReplyAttr>::getData() const
 }
 
 template <typename UserSerializableReplyAttr>
-void
+template <typename Values>
-IntelligenceQueryResponse<UserSerializableReplyAttr>::loadFromJson(cereal::JSONInputArchive &ar)
+bool
+AssetReply<UserSerializableReplyAttr>::matchValues(const Values &values) const
 {
-    std::string raw_data;
+    for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
-    ar(
+        if (source.template matchValues<Values>(values)) return true;
-        cereal::make_nvp("status", raw_data),
+    }
-        cereal::make_nvp("totalNumAssets", total_num_assets),
+    return false;
-        cereal::make_nvp("assetCollections", asset_collections)
+}
-    );
-    status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
 
-    try {
+template <typename UserSerializableReplyAttr>
-        ar(cereal::make_nvp("cursor", cursor));
+UserSerializableReplyAttr
-    } catch(...) {}
+AssetReply<UserSerializableReplyAttr>::mergeReplyData() const
+{
+    UserSerializableReplyAttr reply_data;
+    for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
+        UserSerializableReplyAttr data_by_source = source.mergeReplyData();
+        reply_data.insert(data_by_source);
+    }
+    return reply_data;
+}
+
+template<typename UserSerializableReplyAttr>
+void
+IntelligenceQueryResponseT<UserSerializableReplyAttr>::loadFromJson(const std::string &json_response)
+{
+    std::stringstream in;
+    in.str(json_response);
+    cereal::JSONInputArchive in_ar(in);
+    serialize(in_ar);
 }
 
 template<typename UserSerializableReplyAttr>
 template<class Archive>
 void
-IntelligenceQueryResponse<UserSerializableReplyAttr>::serialize(Archive &ar)
+IntelligenceQueryResponseT<UserSerializableReplyAttr>::serialize(Archive &ar)
 {
-    std::string raw_data;
     ar(
-        cereal::make_nvp("status", raw_data),
-        cereal::make_nvp("totalNumAssets", total_num_assets),
         cereal::make_nvp("assetCollections", asset_collections)
     );
-    status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
 
     try {
-        ar(cereal::make_nvp("cursor", cursor));
+        IntelligenceQueryResponse::serialize(ar);
     } catch(...) {}
 }
 
-template <typename UserSerializableReplyAttr>
-Intelligence_IS_V2::ResponseStatus
-IntelligenceQueryResponse<UserSerializableReplyAttr>::getResponseStatus() const
-{
-    return status;
-}
 
 template <typename UserSerializableReplyAttr>
 uint
-IntelligenceQueryResponse<UserSerializableReplyAttr>::getAmountOfAssets() const
+IntelligenceQueryResponseT<UserSerializableReplyAttr>::getAssetCollectionsSize() const
-{
-    return total_num_assets;
-}
-
-template <typename UserSerializableReplyAttr>
-const std::string &
-IntelligenceQueryResponse<UserSerializableReplyAttr>::getCursor() const
-{
-    return cursor;
-}
-
-template <typename UserSerializableReplyAttr>
-int
-IntelligenceQueryResponse<UserSerializableReplyAttr>::getAssetCollectionsSize() const
 {
     return asset_collections.size();
 }
 
 template <typename UserSerializableReplyAttr>
 const std::vector<AssetReply<UserSerializableReplyAttr>> &
-IntelligenceQueryResponse<UserSerializableReplyAttr>::getData() const
+IntelligenceQueryResponseT<UserSerializableReplyAttr>::getData() const
 {
     return asset_collections;
 }
 
 template <typename UserSerializableReplyAttr>
 bool
-IntelligenceQueryResponse<UserSerializableReplyAttr>::isValidInBulk() const
+IntelligenceQueryResponseT<UserSerializableReplyAttr>::isLast(uint asset_limit)
 {
-    return !partial_fail_in_bulk;
+    return getResponseStatus() ==  ResponseStatus::DONE && getAssetCollectionsSize() < asset_limit;
 }
 
-template <typename UserSerializableReplyAttr>
+#endif // __ASSET_REPLAY_IMPL_H__
-void
-IntelligenceQueryResponse<UserSerializableReplyAttr>::setFailInBulk()
-{
-    partial_fail_in_bulk = true;
-}
-
-#endif // __QUERY_RESPONSE_V2_IMPL_H_

core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h

@@ -11,25 +11,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef __QUERY_RESPONSE_V2_H__
+#ifndef __ASSET_REPLY_H__
-#define __QUERY_RESPONSE_V2_H__
+#define __ASSET_REPLY_H__
 
-#include <sstream>
-#include <vector>
 #include <map>
+#include <string>
+#include <vector>
 
-#include "cereal/archives/json.hpp"
+#include "asset_source.h"
-#include "cereal/types/vector.hpp"
+#include "query_request_v2.h"
-#include "cereal/types/map.hpp"
-
-#include "debug.h"
-#include "maybe_res.h"
-#include "customized_cereal_map.h"
-#include "customized_cereal_multimap.h"
 #include "intelligence_types_v2.h"
-#include "asset_source_v2.h"
+#include "maybe_res.h"
-
-USE_DEBUG_FLAG(D_INTELLIGENCE);
 
 template <typename UserSerializableReplyAttr>
 class AssetReply
@@ -55,26 +47,10 @@ public:
     const std::string & getAssetOrder() const { return asset_order; }
     const std::string & getAssetKind() const { return asset_kind; }
 
-    UserSerializableReplyAttr
+    UserSerializableReplyAttr mergeReplyData() const;
-    mergeReplyData() const
-    {
-        UserSerializableReplyAttr reply_data;
-        for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
-            UserSerializableReplyAttr data_by_source = source.mergeReplyData();
-            reply_data.insert(data_by_source);
-        }
-        return reply_data;
-    }
 
     template <typename Values>
-    bool
+    bool matchValues(const Values &values) const;
-    matchValues(const Values &values) const
-    {
-        for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
-            if (source.template matchValues<Values>(values)) return true;
-        }
-        return false;
-    }
 
 private:
     uint asset_schema_version = 0;
@@ -94,33 +70,48 @@ private:
     std::vector<SerializableAssetSource<UserSerializableReplyAttr>> sources;
 };
 
-template <typename UserSerializableReplyAttr>
 class IntelligenceQueryResponse
 {
 public:
     IntelligenceQueryResponse() {}
 
-    void loadFromJson(cereal::JSONInputArchive &ar);
+    void loadFromJson(const std::string &json_response);
 
     template<class Archive>
     void serialize(Archive &ar);
 
-    Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
+    Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; }
-    uint getAmountOfAssets() const;
+    const std::string & getCursor() const { return cursor; }
-    const std::string & getCursor() const;
+    uint getAmountOfAssets() const { return total_num_assets; }
-    int getAssetCollectionsSize() const;
+    bool isValidInBulk() const { return !partial_fail_in_bulk; }
-    const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
+    void setFailInBulk() { partial_fail_in_bulk = true; }
-    bool isValidInBulk() const;
-    void setFailInBulk();
 
 private:
     Intelligence_IS_V2::ResponseStatus status = Intelligence_IS_V2::ResponseStatus::IN_PROGRESS;
     uint total_num_assets = 0;
     std::string cursor = "";
-    std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
     bool partial_fail_in_bulk = false;
 };
 
-#include "query_response_v2_impl.h"
+template <typename UserSerializableReplyAttr>
+class IntelligenceQueryResponseT : public IntelligenceQueryResponse
+{
+public:
+    void loadFromJson(const std::string &json_response);
 
-#endif // __QUERY_RESPONSE_V2_H__
+    template<class Archive>
+    void serialize(Archive &ar);
+
+    uint getAssetCollectionsSize() const;
+
+    bool isLast(uint asset_limit);
+
+    const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
+
+private:
+    std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
+};
+
+#include "asset_replay_impl.h"
+
+#endif // __ASSET_REPLY_H__

core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h

@@ -11,14 +11,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#ifndef __ASSET_SOURCE_V2_H__
+#ifndef __ASSET_SOURCE_H__
-#define __ASSET_SOURCE_V2_H__
+#define __ASSET_SOURCE_H__
+
+#include <chrono>
+#include <string>
+#include <vector>
 
-#include "debug.h"
-#include "intelligence_types_v2.h"
 #include "cereal/archives/json.hpp"
-#include "cereal/types/vector.hpp"
+#include "intelligence_types_v2.h"
-#include "customized_cereal_map.h"
 
 template <typename UserSerializableReplyAttr>
 class SerializableAssetSource
@@ -26,7 +27,26 @@ class SerializableAssetSource
 public:
     SerializableAssetSource() {}
 
-    void load(cereal::JSONInputArchive &ar);
+    void load(cereal::JSONInputArchive &ar)
+    {
+        uint raw_seconds;
+        ar(
+            cereal::make_nvp("tenantId", tenant_id),
+            cereal::make_nvp("sourceId", source_id),
+            cereal::make_nvp("assetId", asset_id),
+            cereal::make_nvp("ttl", raw_seconds),
+            cereal::make_nvp("expirationTime", expiration_time),
+            cereal::make_nvp("confidence", confidence)
+        );
+        ttl = std::chrono::seconds(raw_seconds);
+
+        UserSerializableReplyAttr raw_attribute;
+        try {
+            ar(cereal::make_nvp("attributes", raw_attribute));
+            attributes.clear();
+            attributes.push_back(raw_attribute);
+        } catch(const std::exception &e) {}
+    }
 
     const std::string & getTenantId() const { return tenant_id; }
     const std::string & getSourceId() const { return source_id; }
@@ -66,6 +86,4 @@ private:
     std::vector<UserSerializableReplyAttr> attributes;
 };
 
-#include "asset_source_v2_impl.h"
+#endif // __ASSET_SOURCE_H__
-
-#endif //__ASSET_SOURCE_V2_H__

core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h

@@ -1,47 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __ASSET_SOURCE_V2_IMPL_H__
-#define __ASSET_SOURCE_V2_IMPL_H__
-
-#ifndef __ASSET_SOURCE_V2_H__
-#error intelligence_query_impl_8_0.h should not be included directly!
-#endif //__ASSET_V2_SOURCE_H__
-
-USE_DEBUG_FLAG(D_INTELLIGENCE);
-
-template <typename UserSerializableReplyAttr>
-void
-SerializableAssetSource<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
-{
-    uint raw_seconds;
-    ar(
-        cereal::make_nvp("tenantId", tenant_id),
-        cereal::make_nvp("sourceId", source_id),
-        cereal::make_nvp("assetId", asset_id),
-        cereal::make_nvp("ttl", raw_seconds),
-        cereal::make_nvp("expirationTime", expiration_time),
-        cereal::make_nvp("confidence", confidence)
-    );
-    ttl = std::chrono::seconds(raw_seconds);
-
-    UserSerializableReplyAttr raw_attribute;
-    try {
-        ar(cereal::make_nvp("attributes", raw_attribute));
-        attributes.clear();
-        attributes.push_back(raw_attribute);
-    } catch(const std::exception &e) {}
-
-}
-
-#endif //__ASSET_SOURCE_V2_IMPL_H__

core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h

@@ -15,12 +15,12 @@
 #define __BULK_QUERY_RESPONSE_V2_H__
 
 #include <sstream>
-#include <vector>
 #include <string>
+#include <vector>
 
+#include "asset_reply.h"
 #include "cereal/archives/json.hpp"
 #include "cereal/types/vector.hpp"
-
 #include "debug.h"
 #include "intelligence_types_v2.h"
 
@@ -29,15 +29,7 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
 class BulkResponseError
 {
 public:
-    void
+    void serialize(cereal::JSONInputArchive &ar);
-    serialize(cereal::JSONInputArchive &ar)
-    {
-        ar(
-            cereal::make_nvp("index", index),
-            cereal::make_nvp("statusCode", status_code),
-            cereal::make_nvp("message", message)
-        );
-    }
 
     unsigned int getIndex() const { return index; }
     int getStatusCode() const { return status_code; }
@@ -49,46 +41,69 @@ private:
     std::string message;
 };
 
-template <typename UserSerializableReplyAttr>
 class ValidBulkQueryResponse
 {
+public:
+    void serialize(cereal::JSONInputArchive &ar);
+
+    unsigned int getIndex() const { return index; }
+    const IntelligenceQueryResponse & getResponse() const { return response; }
+
+private:
+    unsigned int index;
+    IntelligenceQueryResponse response;
+};
+
+template <typename UserSerializableReplyAttr>
+class ValidBulkQueryResponseT : public ValidBulkQueryResponse
+{
 public:
     void
     serialize(cereal::JSONInputArchive &ar)
     {
+        try {
+            ValidBulkQueryResponse::serialize(ar);
+        } catch (...) {}
         ar(
-            cereal::make_nvp("index", index),
             cereal::make_nvp("response", response)
         );
     }
 
-    unsigned int getIndex() const { return index; }
+    const IntelligenceQueryResponseT<UserSerializableReplyAttr> & getResponse() const { return response; }
-    const IntelligenceQueryResponse<UserSerializableReplyAttr> & getResponse() const { return response; }
 
 private:
-    unsigned int index;
+    IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
-    IntelligenceQueryResponse<UserSerializableReplyAttr> response;
+};
+
+class IntelligenceQueryBulkResponse
+{
+public:
+    void serialize(cereal::JSONInputArchive &ar);
+
+    const std::vector<ValidBulkQueryResponse> & getValid() { return valid_responses; }
+    const std::vector<BulkResponseError> & getErrors() { return errors; }
+private:
+    std::vector<ValidBulkQueryResponse> valid_responses;
+    std::vector<BulkResponseError> errors;
 };
 
 template <typename UserSerializableReplyAttr>
-class IntelligenceQueryBulkResponse
+class IntelligenceQueryBulkResponseT : public IntelligenceQueryBulkResponse
 {
 public:
     void
     serialize(cereal::JSONInputArchive &ar)
     {
-        ar(cereal::make_nvp("queriesResponse", valid_responses));
         try {
-            ar(cereal::make_nvp("errors", errors));
+            IntelligenceQueryBulkResponse::serialize(ar);
         } catch(...) {}
+        ar(cereal::make_nvp("queriesResponse", valid_responses));
     }
 
-    const std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
+    const std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
-    const std::vector<BulkResponseError> & getErrors() { return errors; }
 
 private:
-    std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> valid_responses;
+    std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> valid_responses;
-    std::vector<BulkResponseError> errors;
 };
 
 #endif // __BULK_QUERY_RESPONSE_V2_H__

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h

@@ -0,0 +1,50 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __INTELLIGENCE_INTERFACE_IMPL_H__
+#define __INTELLIGENCE_INTERFACE_IMPL_H__
+
+#ifndef __I_INTELLIGENCE_IS_V2_H__
+#error intelligence_interface_impl.h should not be included directly!
+#endif // __I_INTELLIGENCE_IS_V2_H__
+
+template <typename Data>
+Maybe<std::vector<AssetReply<Data>>>
+I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore_in_progress, bool is_pretty)
+{
+    auto response = getResponse(query_request, is_pretty);
+
+    if (!response.ok()) return response.passErr();
+    auto serializable_response = response->getSerializableResponse<Data>();
+    if (!query_request.isPagingActivated()) return serializable_response.getData();
+    if (serializable_response.isLast(query_request.getAssetsLimit())) {
+        query_request.setCursor(Intelligence_IS_V2::CursorState::DONE, "");
+    } else {
+        query_request.setCursor(Intelligence_IS_V2::CursorState::IN_PROGRESS, response->getCursor());
+        if (ignore_in_progress) return genError("Query intelligence response with InProgress status");
+    }
+
+    return serializable_response.getData();
+}
+
+template<typename Data>
+Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
+I_Intelligence_IS_V2::queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty)
+{
+    auto res = getResponse(query_requests, is_pretty, true);
+    if (!res.ok()) return res.passErr();
+
+    return res->getBulkData<Data>();
+}
+
+#endif // __INTELLIGENCE_INTERFACE_IMPL_H__

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h

@@ -1,77 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __INTELLIGENCE_QUERY_V2_H__
-#define __INTELLIGENCE_QUERY_V2_H__
-
-#include<vector>
-
-#include "cereal/archives/json.hpp"
-#include "intelligence_types_v2.h"
-#include "query_request_v2.h"
-#include "query_response_v2.h"
-#include "bulk_query_response_v2.h"
-#include "rest.h"
-
-template <typename UserSerializableReplyAttr>
-class IntelligenceQuery
-{
-public:
-    IntelligenceQuery(QueryRequest &filter, bool is_pretty)
-            :
-        request(filter),
-        response(),
-        responses(),
-        is_bulk(false),
-        is_pretty(is_pretty)
-    {}
-
-    IntelligenceQuery(std::vector<QueryRequest> &filters, bool is_pretty)
-            :
-        requests(filters),
-        response(),
-        responses(),
-        is_bulk(true),
-        is_pretty(is_pretty)
-    {}
-
-    Maybe<std::string> genJson() const;
-    bool loadJson(const std::string &json);
-
-    void load(cereal::JSONInputArchive &ar);
-    void save(cereal::JSONOutputArchive &ar) const;
-
-    std::vector<AssetReply<UserSerializableReplyAttr>> getData();
-    std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> getBulkData();
-    ResponseStatus getResponseStatus();
-    int getResponseAssetCollectionsSize() const { return response.getAssetCollectionsSize(); }
-    const std::string & getResponseCursorVal() const { return response.getCursor(); }
-
-    void activatePaging();
-    Maybe<Intelligence_IS_V2::CursorState> getPagingStatus();
-    void setRequestCursor(CursorState state, const std::string &value);
-
-private:
-    static QueryRequest dummy_query_request;
-    static std::vector<QueryRequest> dummy_query_requests;
-    std::vector<QueryRequest> &requests = dummy_query_requests;
-    QueryRequest &request = dummy_query_request;
-    IntelligenceQueryResponse<UserSerializableReplyAttr> response;
-    std::vector<IntelligenceQueryResponse<UserSerializableReplyAttr>> responses;
-    bool is_bulk;
-    bool is_pretty;
-};
-
-#include "intelligence_query_v2_impl.h"
-
-#endif // __INTELLIGENCE_QUERY_V2_H__

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h

@@ -1,191 +0,0 @@
-// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
-
-// Licensed under the Apache License, Version 2.0 (the "License");
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-#ifndef __INTELLIGENCE_QUERY_V2_IMPL_H_
-#define __INTELLIGENCE_QUERY_V2_IMPL_H_
-
-#ifndef __INTELLIGENCE_QUERY_V2_H__
-#error intelligence_query_impl_v2.h should not be included directly!
-#endif // __INTELLIGENCE_QUERY_V2_H__
-
-#include <sstream>
-#include "json_stream.h"
-
-USE_DEBUG_FLAG(D_INTELLIGENCE);
-
-template <typename UserSerializableReplyAttr>
-QueryRequest IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_request = QueryRequest();
-
-template <typename UserSerializableReplyAttr>
-std::vector<QueryRequest> IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_requests =
-    std::vector<QueryRequest>();
-
-template <typename UserSerializableReplyAttr>
-Maybe<std::string>
-IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
-{
-    {
-        std::stringstream str_stream;
-        JsonStream json_stream(&str_stream, is_pretty);
-        {
-            cereal::JSONOutputArchive out_ar(json_stream);
-            if (is_bulk) {
-                std::vector<BulkQueryRequest> bulk_requests;
-                int index = 0;
-                for (QueryRequest &request : requests) {
-                    bulk_requests.push_back(BulkQueryRequest(request, index++));
-                }
-                out_ar(cereal::make_nvp("queries", bulk_requests));
-            } else {
-                request.saveToJson(out_ar);
-            }
-        }
-
-        return str_stream.str();
-    }
-}
-
-template <typename UserSerializableReplyAttr>
-bool
-IntelligenceQuery<UserSerializableReplyAttr>::loadJson(const std::string &json)
-{
-    try {
-        std::stringstream in;
-        in.str(json);
-        try {
-            cereal::JSONInputArchive in_ar(in);
-            load(in_ar);
-        } catch (const Intelligence_IS_V2::IntelligenceException &e) {
-            dbgWarning(D_INTELLIGENCE) << "Failed to load query response. Error: " << e.what();
-            return false;
-        }
-        return true;
-    } catch (const std::exception &e) {
-        return false;
-    }
-}
-
-template <typename UserSerializableReplyAttr>
-void
-IntelligenceQuery<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
-{
-    if (is_bulk) {
-        IntelligenceQueryBulkResponse<UserSerializableReplyAttr> bulk_response;
-        bulk_response.serialize(ar);
-        unsigned int error_idx = 0;
-        unsigned int valid_idx = 0;
-        const auto &valid_response = bulk_response.getValid();
-        const auto &errors = bulk_response.getErrors();
-        responses.clear();
-        responses.reserve(requests.size());
-        dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items";
-        for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) {
-            if (valid_response[valid_idx].getIndex() == query_idx) {
-                responses.push_back(valid_response[valid_idx].getResponse());
-                dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
-                valid_idx++;
-            } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
-                responses.emplace_back();
-                responses[query_idx].setFailInBulk();
-                dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
-                error_idx++;
-            } else {
-                dbgWarning(D_INTELLIGENCE)
-                    << "Query index was not found neither in valid nor error responses, assuming error";
-                responses[query_idx].setFailInBulk();
-            }
-        }
-    } else {
-        response.loadFromJson(ar);
-    }
-}
-
-template <typename UserSerializableReplyAttr>
-void
-IntelligenceQuery<UserSerializableReplyAttr>::save(cereal::JSONOutputArchive &ar) const
-{
-    if (!is_bulk) {
-        request.saveToJson(ar);
-    } else {
-        ar(cereal::make_nvp("queries", requests));
-    }
-}
-
-template <typename UserSerializableReplyAttr>
-std::vector<AssetReply<UserSerializableReplyAttr>>
-IntelligenceQuery<UserSerializableReplyAttr>::getData()
-{
-    return response.getData();
-}
-
-template <typename UserSerializableReplyAttr>
-std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
-IntelligenceQuery<UserSerializableReplyAttr>::getBulkData()
-{
-    std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
-    bulk_data.reserve(responses.size());
-    int index = 0;
-    for (const auto &res: responses) {
-        if (!res.isValidInBulk()) {
-            dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
-            bulk_data.push_back(genError("Received error for request in bulk"));
-            index++;
-        } else {
-            dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
-            bulk_data.push_back(res.getData());
-            index++;
-        }
-    }
-    return bulk_data;
-}
-
-template <typename UserSerializableReplyAttr>
-void
-IntelligenceQuery<UserSerializableReplyAttr>::activatePaging()
-{
-    request.setCursor(Intelligence_IS_V2::CursorState::START, "start");
-}
-
-template <typename UserSerializableReplyAttr>
-Maybe<Intelligence_IS_V2::CursorState>
-IntelligenceQuery<UserSerializableReplyAttr>::getPagingStatus()
-{
-    if (is_bulk) return genError("Paging not activated in bulk mode");
-    if (!request.isPagingActivated()) return genError("Paging not activated");
-    return request.getCursorState();
-}
-
-template <typename UserSerializableReplyAttr>
-ResponseStatus
-IntelligenceQuery<UserSerializableReplyAttr>::getResponseStatus()
-{
-    if (!is_bulk) return response.getResponseStatus();
-
-    if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
-    for (const auto &response_itr : responses) {
-        if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
-            return ResponseStatus::IN_PROGRESS;
-        }
-    }
-
-    return ResponseStatus::DONE;
-}
-
-template <typename UserSerializableReplyAttr>
-void
-IntelligenceQuery<UserSerializableReplyAttr>::setRequestCursor(CursorState state, const std::string &value)
-{
-    request.setCursor(state, value);
-}
-
-#endif //__INTELLIGENCE_QUERY_V2_IMPL_H_

core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h

@@ -0,0 +1,111 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __INTELLIGENCE_RESPONSE_H__
+#define __INTELLIGENCE_RESPONSE_H__
+
+#include <chrono>
+#include <string>
+#include <vector>
+
+#include <maybe_res.h>
+#include "asset_reply.h"
+#include "bulk_query_response_v2.h"
+
+USE_DEBUG_FLAG(D_INTELLIGENCE);
+
+namespace Intelligence
+{
+
+class Response
+{
+public:
+    Response() = default;
+    Response(const std::string &json_body, size_t size, bool is_bulk)
+            :
+        json_response(json_body), size(size), is_bulk(is_bulk)
+    {}
+
+    Maybe<void> load();
+    Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
+    const std::string getCursor() const { return single_response.getCursor(); }
+    void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; }
+    template <typename UserSerializableReplyAttr>
+    IntelligenceQueryResponseT<UserSerializableReplyAttr> getSerializableResponse() const
+    {
+        IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
+        response.loadFromJson(json_response);
+        return response;
+    }
+
+    template <typename UserSerializableReplyAttr>
+    std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
+    getBulkData() const
+    {
+        std::stringstream in;
+        in.str(json_response);
+        cereal::JSONInputArchive in_ar(in);
+
+        IntelligenceQueryBulkResponseT<UserSerializableReplyAttr> bulk_response;
+        bulk_response.serialize(in_ar);
+        unsigned int error_idx = 0;
+        unsigned int valid_idx = 0;
+        const auto &valid_response = bulk_response.getValid();
+        const auto &errors = bulk_response.getErrors();
+        std::vector<IntelligenceQueryResponseT<UserSerializableReplyAttr>> serializable_responses;
+        serializable_responses.reserve(size);
+        dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
+        for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
+            if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
+                serializable_responses.push_back(valid_response[valid_idx].getResponse());
+                dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
+                valid_idx++;
+            } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
+                serializable_responses.emplace_back();
+                serializable_responses[query_idx].setFailInBulk();
+                dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
+                error_idx++;
+            } else {
+                dbgWarning(D_INTELLIGENCE)
+                    << "Query index was not found neither in valid nor error responses, assuming error";
+                serializable_responses[query_idx].setFailInBulk();
+            }
+        }
+        std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
+        bulk_data.reserve(serializable_responses.size());
+        int index = 0;
+        for (const auto &res: serializable_responses) {
+            if (!res.isValidInBulk()) {
+                dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
+                bulk_data.push_back(genError("Received error for request in bulk"));
+                index++;
+            } else {
+                dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
+                bulk_data.push_back(res.getData());
+                index++;
+            }
+        }
+        return bulk_data;
+    }
+
+private:
+    std::string json_response;
+    std::vector<IntelligenceQueryResponse> responses;
+    IntelligenceQueryResponse single_response;
+    size_t size = 0;
+    bool is_bulk = false;
+};
+
+}
+
+#endif // __INTELLIGENCE_RESPONSE_H__

core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h

@@ -59,7 +59,6 @@ public:
 
     Condition getConditionType() const { return condition_type; }
     const std::string & getKey() const { return key; }
-    const ValueVariant & getValue() const { return value; }
 
 private:
     Condition condition_type = Condition::EQUALS;
@@ -85,8 +84,6 @@ public:
     const std::vector<SerializableQueryCondition> & getConditionOperands() const { return condition_operands; }
     const std::vector<SerializableQueryFilter> & getQueriesOperands() const { return queries_operands; }
 
-    Maybe<SerializableQueryCondition::ValueVariant> getConditionValueByKey(const std::string &key) const;
-
     bool empty() const { return condition_operands.empty() && queries_operands.empty(); }
 
     SerializableQueryFilter operator &&(const SerializableQueryFilter &other_query);

core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h

@@ -97,11 +97,11 @@ public:
     void setObjectType(const ObjectType &obj_type);
 
     void setAssetsLimit(uint _assets_limit);
-    bool checkMinConfidence(uint upper_confidence_limit);
+    bool checkMinConfidence(uint upper_confidence_limit) const;
 
     void activatePaging();
     bool isPagingActivated();
-    Maybe<CursorState> getCursorState();
+    Maybe<CursorState> getCursorState() const;
     bool isPagingFinished();
     void setCursor(CursorState state, const std::string &value);
     bool empty() const { return query.empty(); }
@@ -124,21 +124,4 @@ private:
     Maybe<std::string> convertObjectTypeToString() const;
 };
 
-class BulkQueryRequest
-{
-public:
-    BulkQueryRequest() {}
-
-    BulkQueryRequest(QueryRequest &request, int index);
-    
-    void saveToJson(cereal::JSONOutputArchive &ar) const;
-    void save(cereal::JSONOutputArchive &ar) const;
-
-    QueryRequest getQueryRequest() const;
-
-private:
-    QueryRequest request;
-    int index;
-};
-
 #endif // __QUERY_REQUEST_V2_H__

core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h

@@ -51,7 +51,7 @@ public:
     uint getSize() const { return requested_attributes.size(); }
     bool isRequestedAttributesMapEmpty() const { return requested_attributes.empty(); }
 
-    bool checkMinConfidence(uint upper_confidence_limit);
+    bool checkMinConfidence(uint upper_confidence_limit) const;
 
 private:
     std::unordered_map<std::string, uint> requested_attributes;

core/include/services_sdk/interfaces/mock/mock_intelligence.h

@@ -0,0 +1,38 @@
+#ifndef __MOCK_INTELLIGENCE_H__
+#define __MOCK_INTELLIGENCE_H__
+
+#include "i_intelligence_is_v2.h"
+#include "cptest.h"
+
+std::ostream &
+operator<<(std::ostream &os, const Intelligence::Response &)
+{
+    return os;
+}
+
+std::ostream &
+operator<<(std::ostream &os, const Intelligence::Invalidation &)
+{
+    return os;
+}
+
+class MockIntelligence : public Singleton::Provide<I_Intelligence_IS_V2>::From<MockProvider<I_Intelligence_IS_V2>>
+{
+public:
+    using InvalidationCb = std::function<void(const Intelligence::Invalidation &)>;
+    using Invalidation = Intelligence::Invalidation;
+    using Response = Intelligence::Response;
+
+    MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
+    MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
+    MOCK_METHOD1(unregisterInvalidation, void(uint id));
+    MOCK_CONST_METHOD3(
+        getResponse,
+        Maybe<Response>(const std::vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk)
+    );
+    MOCK_CONST_METHOD2(getResponse, Maybe<Response>(const QueryRequest &query_request, bool is_pretty));
+    MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void));
+    MOCK_CONST_METHOD1(getOfflineInfoString, Maybe<std::string>(const SerializableQueryFilter &query));
+};
+
+#endif // __MOCK_INTELLIGENCE_H__

core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h

@@ -14,6 +14,9 @@
 #ifndef __DATA_STRING_V2_H__
 #define __DATA_STRING_V2_H__
 
+#include <string>
+#include "cereal/archives/json.hpp"
+
 class DataString
 {
 public:

core/include/services_sdk/utilities/agent_core_utilities.h

@@ -32,6 +32,7 @@ Maybe<std::vector<std::string>> getDirectoryFiles(const std::string &path);
 bool makeDir(const std::string &path, mode_t permission = S_IRWXU);
 bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU);
 bool deleteDirectory(const std::string &path, bool delete_content = false);
+bool touchFile(const std::string &path);
 
 bool
 copyFile(

core/intelligence_is_v2/CMakeLists.txt

@@ -1,7 +1,7 @@
 include_directories(include)
 add_library(
-    intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
+    intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc intelligence_server.cc intelligence_response.cc asset_replay.cc bulk_query_response_v2.cc
-    intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
+    intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc intelligence_request.cc
 )
 
-add_subdirectory(intelligence_is_v2_ut)
+#add_subdirectory(intelligence_is_v2_ut)

core/intelligence_is_v2/asset_replay.cc

@@ -0,0 +1,38 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_is_v2/asset_reply.h"
+
+using namespace std;
+
+void
+IntelligenceQueryResponse::loadFromJson(const std::string &json_response)
+{
+    std::stringstream in;
+    in.str(json_response);
+    cereal::JSONInputArchive in_ar(in);
+    serialize(in_ar);
+}
+
+template <class Archive>
+void
+IntelligenceQueryResponse::serialize(Archive &ar)
+{
+    std::string raw_data;
+    ar(cereal::make_nvp("status", raw_data), cereal::make_nvp("totalNumAssets", total_num_assets));
+    status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
+
+    try {
+        ar(cereal::make_nvp("cursor", cursor));
+    } catch (...) {}
+}

core/intelligence_is_v2/bulk_query_response_v2.cc

@@ -0,0 +1,39 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_is_v2/bulk_query_response_v2.h"
+
+using namespace std;
+
+void
+BulkResponseError::serialize(cereal::JSONInputArchive &ar)
+{
+    ar(
+        cereal::make_nvp("index", index),
+        cereal::make_nvp("statusCode", status_code),
+        cereal::make_nvp("message", message)
+    );
+}
+
+void
+ValidBulkQueryResponse::serialize(cereal::JSONInputArchive &ar)
+{
+    ar(cereal::make_nvp("index", index), cereal::make_nvp("response", response));
+}
+
+void
+IntelligenceQueryBulkResponse::serialize(cereal::JSONInputArchive &ar)
+{
+    ar(cereal::make_nvp("errors", errors));
+    ar(cereal::make_nvp("queriesResponse", valid_responses));
+}

core/intelligence_is_v2/include/intelligence_request.h

@@ -0,0 +1,53 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __INTELLIGENCE_REQUEST_H__
+#define __INTELLIGENCE_REQUEST_H__
+#include "intelligence_is_v2/query_request_v2.h"
+
+#include <vector>
+#include "maybe_res.h"
+
+namespace Intelligence {
+
+class IntelligenceRequest : ClientRest
+{
+public:
+    IntelligenceRequest(const std::vector<QueryRequest> &queries, bool is_pretty, bool is_bulk)
+            :
+        queries(queries), is_pretty(is_pretty), is_bulk(is_bulk)
+    {}
+
+    Maybe<void> checkAssetsLimit() const;
+    Maybe<void> checkMinConfidence() const;
+    bool isPagingAllowed() const;
+    bool isPagingActivated() const;
+    Maybe<bool> isPagingFinished() const;
+    Maybe<Intelligence_IS_V2::CursorState> getPagingStatus() const;
+    bool loadJson(const std::string &json);
+    Maybe<std::string> genJson() const;
+    Maybe<std::string> getResponseFromFog() const;
+
+    size_t getSize() const { return queries.size(); }
+    bool isBulk() const { return is_bulk; }
+
+private:
+    const std::vector<QueryRequest> &queries;
+    bool is_pretty = true;
+    bool is_bulk = false;
+    Maybe<std::string> response_from_fog = genError("Uninitialized");
+};
+
+}
+
+#endif // __INTELLIGENCE_REQUEST_H__

core/intelligence_is_v2/include/intelligence_server.h

@@ -0,0 +1,50 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __INTELLIGENCE_SERVER_H__
+#define __INTELLIGENCE_SERVER_H__
+
+#include <vector>
+#include "maybe_res.h"
+
+#include "i_messaging.h"
+#include "intelligence_is_v2/intelligence_response.h"
+#include "intelligence_request.h"
+
+namespace Intelligence {
+
+class Sender
+{
+public:
+    Sender(IntelligenceRequest request);
+    Maybe<Response> sendIntelligenceRequest();
+
+private:
+    Maybe<Response> sendQueryObjectToLocalServer(bool is_primary_port);
+    Maybe<Response> sendQueryMessage();
+    Maybe<Response> sendMessage();
+    Maybe<Response> createResponse();
+
+    IntelligenceRequest request;
+    Flags<MessageConnectionConfig> conn_flags;
+    bool is_local_intelligence;
+    Maybe<std::string> server_ip = genError("No server ip set");
+    Maybe<unsigned int> server_port = genError("No port is set");
+    I_Messaging * i_message = nullptr;
+    I_TimeGet * i_timer = nullptr;
+    I_MainLoop * i_mainloop = nullptr;
+};
+
+}
+
+#endif // __INTELLIGENCE_SERVER_H__

core/intelligence_is_v2/intelligence_comp_v2.cc

@@ -17,9 +17,10 @@
 
 #include "cache.h"
 #include "config.h"
-#include "table.h"
-#include "intelligence_is_v2/query_response_v2.h"
 #include "intelligence_invalidation.h"
+#include "intelligence_is_v2/intelligence_response.h"
+#include "intelligence_request.h"
+#include "intelligence_server.h"
 
 using namespace std;
 using namespace chrono;
@@ -221,50 +222,10 @@ class IntelligenceComponentV2::Impl
     Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
 {
 public:
-    class OfflineIntelligeceHandler
-    {
-    public:
-        void
-        init()
-        {
-            filesystem_prefix = getFilesystemPathConfig();
-            dbgTrace(D_INTELLIGENCE) << "OfflineIntelligeceHandler init. file systen prefix: " << filesystem_prefix;
-            offline_intelligence_path = getConfigurationWithDefault<string>(
-                filesystem_prefix + "/conf/offline/intelligence",
-                "intelligence",
-                "offline intelligence path"
-            );
-        }
-
-        Maybe<string>
-        getValueByIdentifier(const string &identifier) const
-        {
-            string asset_file_path = offline_intelligence_path + "/" + identifier;
-            ifstream asset_info(asset_file_path);
-            if (!asset_info.is_open()) {
-                return genError("Could not open file: " + asset_file_path);
-            }
-
-            stringstream info_txt;
-            info_txt << asset_info.rdbuf();
-            asset_info.close();
-            return info_txt.str();
-        }
-
-    private:
-        string filesystem_prefix = "";
-        string offline_intelligence_path = "";
-    };
 
     void
     init()
     {
-        offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
-        registerConfigLoadCb([&]() {
-            offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
-        });
-        offline_intelligence.init();
-
         message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
         timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
         mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
@@ -283,7 +244,6 @@ public:
     bool
     sendInvalidation(const Invalidation &invalidation) const override
     {
-        if (offline_mode_only) return false;
         return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
     }
 
@@ -301,48 +261,28 @@ public:
         invalidations.erase(id);
     }
 
-    I_Messaging *
+    Maybe<Response>
-    getMessaging() const override
+    getResponse(const vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk) const override
     {
-        return message != NULL ? message : Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
+        IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk);
-    }
+        if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr();
-
+        if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr();
-    I_TimeGet *
+        if (intelligence_req.isPagingActivated()) {
-    getTimer() const override
+            auto is_paging_finished = intelligence_req.isPagingFinished();
-    {
+            if (is_paging_finished.ok() && *is_paging_finished) {
-        return timer != NULL ? timer : Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
+                return genError("Paging is activated and already finished. No need for more queries.");
-    }
-
-    I_MainLoop *
-    getMainloop() const override
-    {
-        return mainloop != NULL ? mainloop : Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
-    }
-
-    Maybe<string>
-    getOfflineInfoString(const SerializableQueryFilter &query) const override
-    {
-        string ip_attr_key = "mainAttributes.ip";
-        auto valueVariant = query.getConditionValueByKey(ip_attr_key);
-
-        if (!valueVariant.ok()) {
-            return genError("could not find IP main attribute in the given query.");
-        }
-        const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
-        if (const string* identifier_value = boost::get<string>(&value)) {
-            if (*identifier_value == "") {
-                return genError("Could not find IP main attribute in the given query.");
             }
-            return offline_intelligence.getValueByIdentifier(*identifier_value);
         }
-
+        Sender intelligence_server(intelligence_req);
-        return genError("Value is not of type string.");
+        auto response = intelligence_server.sendIntelligenceRequest();
+        return response;
     }
 
-    bool
+    Maybe<Intelligence::Response>
-    getIsOfflineOnly() const override
+    getResponse(const QueryRequest &query_request, bool is_pretty) const override
     {
-        return offline_mode_only;
+        vector<QueryRequest> queries = {query_request};
+        return getResponse(queries, is_pretty, false);
     }
 
 private:
@@ -362,7 +302,7 @@ private:
     bool
     sendLocalInvalidationImpl(const Invalidation &invalidation) const
     {
-        auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
+        auto server = getSetting<string>("intelligence", "local intelligence server ip");
         if (!server.ok()) {
             dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
             return false;
@@ -435,8 +375,6 @@ private:
     bool
     sendRegistration(const Invalidation &invalidation) const
     {
-        if (offline_mode_only) return false;
-
         InvalidationRegistration registration;
         registration.addInvalidation(invalidation);
 
@@ -446,7 +384,7 @@ private:
     bool
     sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
     {
-        auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
+        auto server = getSetting<string>("intelligence", "local intelligence server ip");
         if (!server.ok()) {
             dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
             return false;
@@ -485,13 +423,11 @@ private:
     void
     sendReccurringInvalidationRegistration() const
     {
-        if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
+        if (!hasLocalIntelligence() || invalidations.empty()) return;
 
         sendLocalRegistrationImpl(invalidations.getRegistration());
     }
 
-    OfflineIntelligeceHandler    offline_intelligence;
-    bool                         offline_mode_only = false;
     InvalidationCallBack         invalidations;
     I_Messaging               *message = nullptr;
     I_TimeGet                    *timer = nullptr;
@@ -511,8 +447,6 @@ void IntelligenceComponentV2::init() { pimpl->init(); }
 void
 IntelligenceComponentV2::preload()
 {
-    registerExpectedConfiguration<string>("intelligence", "offline intelligence path");
-    registerExpectedConfiguration<bool>("intelligence", "offline intelligence only");
     registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
     registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
     registerExpectedSetting<string>("intelligence", "local intelligence server ip");

core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc

@@ -1,15 +1,15 @@
 #include "intelligence_comp_v2.h"
 
-#include "intelligence_is_v2/intelligence_query_v2.h"
-#include "read_attribute_v2.h"
-#include "cptest.h"
-#include "singleton.h"
 #include "config.h"
 #include "config_component.h"
-#include "mock/mock_messaging.h"
+#include "cptest.h"
+#include "mock/mock_intelligence.h"
 #include "mock/mock_mainloop.h"
-#include "mock/mock_time_get.h"
+#include "mock/mock_messaging.h"
 #include "mock/mock_rest_api.h"
+#include "mock/mock_time_get.h"
+#include "read_attribute_v2.h"
+#include "singleton.h"
 
 using namespace std;
 using namespace testing;
@@ -49,9 +49,7 @@ public:
             mockRestCall(_, "new-invalidation/source/invalidation", _)
         ).WillRepeatedly(Return(true));
 
-        string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
+        conf.preload();
-        setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
-
         intelligence.preload();
         intelligence.init();
     }
@@ -72,6 +70,31 @@ public:
     I_MainLoop::Routine routine;
 };
 
+class IntelligenceComponentMockTest : public Test, Singleton::Consume<I_Intelligence_IS_V2>
+{
+public:
+    IntelligenceComponentMockTest()
+    {
+        debug_output.clear();
+        Debug::setNewDefaultStdout(&debug_output);
+        Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
+        Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
+        setConfiguration<bool>(false, string("metric"), string("fogMetricSendEnable"));
+
+        conf.preload();
+    }
+
+    ~IntelligenceComponentMockTest()
+    {
+        Debug::setNewDefaultStdout(&cout);
+    }
+
+    ::Environment env;
+    ConfigComponent conf;
+    stringstream debug_output;
+    StrictMock<MockIntelligence> intelligence_mock;
+};
+
 class Profile
 {
 public:
@@ -97,6 +120,252 @@ private:
     DataString phase;
 };
 
+TEST_F(IntelligenceComponentMockTest, getResponseErrorTest)
+{
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
+    QueryRequest request(Condition::EQUALS, "category", "cloud", true);
+
+    Maybe<Intelligence::Response> res_error = genError("Test error");
+    EXPECT_CALL(intelligence_mock, getResponse(_, _)
+    ).WillOnce(Return(res_error));
+
+    auto maybe_ans = intell->queryIntelligence<Profile>(request);
+    EXPECT_FALSE(maybe_ans.ok());
+}
+
+TEST_F(IntelligenceComponentMockTest, getResponseTest)
+{
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
+    QueryRequest request(Condition::EQUALS, "category", "cloud", true);
+
+    string response_str(
+        "{\n"
+        "  \"assetCollections\": [\n"
+        "    {\n"
+        "      \"schemaVersion\": 1,\n"
+        "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+        "      \"assetTypeSchemaVersion\": 1,\n"
+        "      \"permissionType\": \"tenant\",\n"
+        "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+        "      \"name\": \"fake-online-test-asset\",\n"
+        "      \"class\": \"workload\",\n"
+        "      \"category\": \"cloud\",\n"
+        "      \"family\": \"fake-online-test\",\n"
+        "      \"mainAttributes\": {\n"
+        "          \"ipv4Addresses\": \"1.1.1.1\",\n"
+        "          \"phase\": \"testing\"\n"
+        "      },\n"
+        "      \"sources\": [\n"
+        "        {\n"
+        "          \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
+        "          \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+        "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "          \"ttl\": 120,\n"
+        "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "          \"confidence\": 500,\n"
+        "          \"attributes\": {\n"
+        "            \"phase\": \"testing\",\n"
+        "            \"user\": \"Omry\",\n"
+        "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "          }\n"
+        "        }\n"
+        "      ]\n"
+        "    }\n"
+        "  ],\n"
+        "  \"status\": \"done\",\n"
+        "  \"totalNumAssets\": 2,\n"
+        "  \"cursor\": \"start\"\n"
+        "}\n"
+    );
+
+    Intelligence::Response response(response_str, 1, false);
+
+    EXPECT_CALL(intelligence_mock, getResponse(_, _)
+    ).WillOnce(Return(response));
+
+    auto maybe_ans = intell->queryIntelligence<Profile>(request);
+    EXPECT_TRUE(maybe_ans.ok());
+    auto vec = maybe_ans.unpack();
+    EXPECT_EQ(vec.size(), 1u);
+    auto iter = vec.begin();
+    EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
+    EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
+}
+
+TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest)
+{
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentMockTest>();
+    vector<QueryRequest> requests;
+    requests.emplace_back(Condition::EQUALS, "category", "whatever", true);
+    requests.emplace_back(Condition::EQUALS, "category", "cloud", true);
+    requests.emplace_back(Condition::EQUALS, "category", "nothing", true);
+    requests.emplace_back(Condition::EQUALS, "category", "iot", true);
+
+    string response_str(
+        "{\n"
+        "  \"errors\": [\n"
+        "    {\n"
+        "      \"index\": 0,\n"
+        "      \"statusCode\": 400,\n"
+        "      \"message\": \"Bad request. Error: Invalid cursor\"\n"
+        "    },"
+        "    {\n"
+        "      \"index\": 2,\n"
+        "      \"statusCode\": 405,\n"
+        "      \"message\": \"Bad request. Error: Something else\"\n"
+        "    }"
+        "  ],\n" // errors
+        "  \"queriesResponse\": [\n"
+        "    {\n"
+        "      \"index\": 1,\n"
+        "      \"response\": {\n"
+        "        \"assetCollections\": [\n"
+        "          {\n"
+        "            \"schemaVersion\": 1,\n"
+        "            \"assetType\": \"workload-cloud-ip\",\n"
+        "            \"assetTypeSchemaVersion\": 1,\n"
+        "            \"permissionType\": \"tenant\",\n"
+        "            \"permissionGroupId\": \"some-group-id\",\n"
+        "            \"name\": \"[1.1.1.1]\",\n"
+        "            \"class\": \"workload\",\n"
+        "            \"category\": \"cloud\",\n"
+        "            \"family\": \"ip\",\n"
+        "            \"group\": \"\",\n"
+        "            \"order\": \"\",\n"
+        "            \"kind\": \"\",\n"
+        "            \"mainAttributes\": {\n"
+        "               \"ipv4Addresses\": [\n"
+        "                   \"1.1.1.1\",\n"
+        "                   \"2.2.2.2\"\n"
+        "               ],\n"
+        "               \"phase\": \"testing\"\n"
+        "            },\n"  // mainAttributes
+        "            \"sources\": [\n"
+        "               {\n"
+        "                 \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
+        "                 \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+        "                 \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "                 \"ttl\": 120,\n"
+        "                 \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "                 \"confidence\": 500,\n"
+        "                 \"attributes\": {\n"
+        "                   \"color\": \"red\",\n"
+        "                   \"user\": \"Omry\",\n"
+        "                   \"phase\": \"testing\",\n"
+        "                   \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "                 }\n"
+        "               },\n" // source 1
+        "               {\n"
+        "                 \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
+        "                 \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
+        "                 \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
+        "                 \"ttl\": 120,\n"
+        "                 \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "                 \"confidence\": 600,\n"
+        "                 \"attributes\": {\n"
+        "                   \"color\": \"white\",\n"
+        "                   \"user\": \"Max\",\n"
+        "                   \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "                  }\n"
+        "               }\n" // source 2
+        "            ]\n" // sources
+        "          }\n" // asset 1
+        "        ],\n" // asset collection
+        "        \"status\": \"done\",\n"
+        "        \"totalNumAssets\": 2,\n"
+        "        \"cursor\": \"start\"\n"
+        "      }\n" // response
+        "    },\n" // queryresponse 1
+        "    {\n"
+        "      \"index\": 3,\n"
+        "      \"response\": {\n"
+        "        \"assetCollections\": [\n"
+        "          {\n"
+        "            \"schemaVersion\": 1,\n"
+        "            \"assetType\": \"workload-cloud-ip\",\n"
+        "            \"assetTypeSchemaVersion\": 1,\n"
+        "            \"permissionType\": \"tenant\",\n"
+        "            \"permissionGroupId\": \"some-group-id\",\n"
+        "            \"name\": \"[2.2.2.2]\",\n"
+        "            \"class\": \"workload\",\n"
+        "            \"category\": \"iot\",\n"
+        "            \"family\": \"ip\",\n"
+        "            \"group\": \"\",\n"
+        "            \"order\": \"\",\n"
+        "            \"kind\": \"\",\n"
+        "            \"mainAttributes\": {\n"
+        "               \"ipv4Addresses\": [\n"
+        "                   \"1.1.1.1\",\n"
+        "                   \"2.2.2.2\"\n"
+        "               ],\n"
+        "               \"phase\": \"testing\"\n"
+        "            },\n"  // mainAttributes
+        "            \"sources\": [\n"
+        "               {\n"
+        "                 \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
+        "                 \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+        "                 \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+        "                 \"ttl\": 120,\n"
+        "                 \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "                 \"confidence\": 500,\n"
+        "                 \"attributes\": {\n"
+        "                   \"color\": \"red\",\n"
+        "                   \"user\": \"Omry2\",\n"
+        "                   \"phase\": \"testing2\",\n"
+        "                   \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "                 }\n"
+        "               },\n" // source 1
+        "               {\n"
+        "                 \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
+        "                 \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
+        "                 \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
+        "                 \"ttl\": 120,\n"
+        "                 \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+        "                 \"confidence\": 600,\n"
+        "                 \"attributes\": {\n"
+        "                   \"color\": \"white\",\n"
+        "                   \"user\": \"Max\",\n"
+        "                   \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+        "                  }\n"
+        "               }\n" // source 2
+        "            ]\n" // sources
+        "          }\n" // asset 1
+        "        ],\n" // asset collection
+        "        \"status\": \"done\",\n"
+        "        \"totalNumAssets\": 2,\n"
+        "        \"cursor\": \"start\"\n"
+        "      }\n" // response
+        "    }\n" // queryresponse 1
+        "  ]\n" // queryresponses
+        "}\n"
+    );
+    Intelligence::Response response(response_str, 4, true);
+
+    EXPECT_CALL(intelligence_mock, getResponse(_, _, _)
+    ).WillOnce(Return(response));
+
+    auto maybe_ans = intell->queryIntelligence<Profile>(requests);
+    EXPECT_TRUE(maybe_ans.ok());
+    auto vec = maybe_ans.unpack();
+    EXPECT_EQ(vec.size(), 4u);
+    EXPECT_FALSE(vec[0].ok());
+    EXPECT_TRUE(vec[1].ok());
+    EXPECT_FALSE(vec[2].ok());
+    EXPECT_TRUE(vec[3].ok());
+
+    auto assets1_vec = vec[1].unpack();
+    EXPECT_EQ(assets1_vec.size(), 1u);
+    auto iter = assets1_vec.begin();
+    EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
+    EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
+
+    auto assets3_vec = vec[3].unpack();
+    EXPECT_EQ(assets1_vec.size(), 1u);
+    iter = assets3_vec.begin();
+    EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
+    EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
+}
+
 TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
 {
     I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@@ -154,6 +423,75 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
     EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
 }
 
+TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
+{
+    stringstream configuration;
+    configuration << "{";
+    configuration << "  \"agentSettings\":[";
+    configuration << "    {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
+    configuration << "  ],";
+    configuration << "  \"intelligence\":{";
+    configuration << "    \"local intelligence server ip\":\"127.0.0.1\",";
+    configuration << "    \"local intelligence server primary port\":9090";
+    configuration << "  }";
+    configuration << "}";
+    Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
+
+    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
+    QueryRequest request(Condition::EQUALS, "category", "cloud", true);
+
+
+    string response_str(
+    "{\n"
+    "  \"assetCollections\": [\n"
+    "    {\n"
+    "      \"schemaVersion\": 1,\n"
+    "      \"assetType\": \"workload-cloud-fake-online-test\",\n"
+    "      \"assetTypeSchemaVersion\": 1,\n"
+    "      \"permissionType\": \"tenant\",\n"
+    "      \"permissionGroupId\": \"fake-online-test-group\",\n"
+    "      \"name\": \"fake-online-test-asset\",\n"
+    "      \"class\": \"workload\",\n"
+    "      \"category\": \"cloud\",\n"
+    "      \"family\": \"fake-online-test\",\n"
+    "      \"mainAttributes\": {\n"
+    "          \"ipv4Addresses\": \"1.1.1.1\",\n"
+    "          \"phase\": \"testing\"\n"
+    "      },\n"
+    "      \"sources\": [\n"
+    "        {\n"
+    "          \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
+    "          \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
+    "          \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
+    "          \"ttl\": 120,\n"
+    "          \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
+    "          \"confidence\": 500,\n"
+    "          \"attributes\": {\n"
+    "            \"phase\": \"testing\",\n"
+    "            \"user\": \"Omry\",\n"
+    "            \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
+    "          }\n"
+    "        }\n"
+    "      ]\n"
+    "    }\n"
+    "  ],\n"
+    "  \"status\": \"done\",\n"
+    "  \"totalNumAssets\": 2,\n"
+    "  \"cursor\": \"start\"\n"
+    "}\n"
+    );
+
+    MessageMetadata md;
+
+    EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
+    ).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
+
+    auto maybe_ans = intell->queryIntelligence<Profile>(request);
+    EXPECT_TRUE(maybe_ans.ok());
+
+    EXPECT_EQ(md.getHostName(), "127.0.0.1");
+}
+
 TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)
 {
     I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@@ -626,6 +964,7 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
     ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str)));
 
     auto maybe_ans3 = intell->queryIntelligence<Profile>(request);
+    if (!maybe_ans3.ok()) cout << maybe_ans3.getErr() + "\n";
     EXPECT_TRUE(maybe_ans3.ok());
     auto vec3 = maybe_ans3.unpack();
     EXPECT_EQ(vec3.size(), 1);
@@ -636,54 +975,6 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
     EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
 }
 
-TEST_F(IntelligenceComponentTestV2, offlineIntelligenceTest)
-{
-    setConfiguration<bool>(true, string("intelligence"), string("offline intelligence only"));
-    intelligence.init();
-
-    I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
-    QueryRequest request(Condition::EQUALS, "ip", "1.2.3.4", true);
-    auto maybe_ans = intell->queryIntelligence<Profile>(request);
-
-    ASSERT_TRUE(maybe_ans.ok());
-    vector<AssetReply<Profile>> vec = maybe_ans.unpack();
-    vector<AssetReply<Profile>>::iterator assets_iter = vec.begin();
-    EXPECT_EQ(assets_iter->getAssetSchemaVersion(), 1);
-    EXPECT_EQ(assets_iter->getAssetType(), "workload-cloud-ip");
-    EXPECT_EQ(assets_iter->getAssetTypeSchemaVersion(), 1);
-    EXPECT_EQ(assets_iter->getAssetPermissionGroupId(), "offline-group-id");
-    EXPECT_EQ(assets_iter->getAssetName(), "offline-asset");
-    EXPECT_EQ(assets_iter->getAssetClass(), "workload");
-    EXPECT_EQ(assets_iter->getAssetCategory(), "cloud");
-    EXPECT_EQ(assets_iter->getAssetFamily(), "offline family");
-    EXPECT_EQ(assets_iter->getAssetGroup(), "offline testing");
-    EXPECT_EQ(assets_iter->getAssetOrder(), "");
-    EXPECT_EQ(assets_iter->getAssetKind(), "");
-
-    map<string, vector<string>> attributes_map = assets_iter->getMainAttributes();
-    EXPECT_EQ(attributes_map["ip"].front(), "1.2.3.4");
-
-    vector<SerializableAssetSource<Profile>>::const_iterator sources_iter = assets_iter->getSources().begin();
-    EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229aa00");
-    EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00");
-    EXPECT_EQ(sources_iter->getAssetId(), "50255c3172b4fb7fda93025f0bfaa7abefd1");
-    EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
-    EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
-    EXPECT_EQ(sources_iter->getConfidence(), 700);
-    EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
-    EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
-
-    sources_iter++;
-    EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229bb11");
-    EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7bbb11");
-    EXPECT_EQ(sources_iter->getAssetId(), "cb068860528cb6bfb000cc35e79f11aeefed2");
-    EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
-    EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
-    EXPECT_EQ(sources_iter->getConfidence(), 600);
-    EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Max");
-    EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
-}
-
 TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
 {
     I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();

core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc

@@ -11,7 +11,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include "intelligence_is_v2/intelligence_query_v2.h"
+#include "intelligence_request.h"
+#include "intelligence_is_v2/query_request_v2.h"
 
 #include "cptest.h"
 
@@ -22,7 +23,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
 
 TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
     QueryRequest request(Condition::EQUALS, "phase", "testing", true);
-    IntelligenceQuery<int> query(request, true);
+    vector<QueryRequest> requests = {request};
+    Intelligence::IntelligenceRequest query(requests, true, false);
 
     std::string expected = "{\n"
         "    \"limit\": 20,\n"
@@ -39,7 +41,9 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
 
 TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
     QueryRequest request(Condition::EQUALS, "phase", "testing", true);
-    IntelligenceQuery<int> query(request, false);
+    vector<QueryRequest> requests = {request};
+    Intelligence::IntelligenceRequest query(requests, false, false);
+
     std::string expected = "{"
         "\"limit\":20,"
         "\"fullResponse\":true,"
@@ -54,7 +58,8 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
 
 TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) {
     QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true);
-    IntelligenceQuery<int> query(request, false);
+    vector<QueryRequest> requests = {request};
+    Intelligence::IntelligenceRequest query(requests, false, false);
     std::string expected = "{"
         "\"limit\":20,"
         "\"fullResponse\":true,"
@@ -71,7 +76,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
     QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
     QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
     std::vector<QueryRequest> requests = {request1, request2};
-    IntelligenceQuery<int> query(requests, true);
+    Intelligence::IntelligenceRequest query(requests, true, true);
 
     std::string expected = "{\n"
         "    \"queries\": [\n"
@@ -109,30 +114,30 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
     QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
     QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
     std::vector<QueryRequest> requests = {request1, request2};
-    IntelligenceQuery<int> query(requests, false);
+    Intelligence::IntelligenceRequest query(requests, false, true);
 
-std::string expected = "{"
+    std::string expected = "{"
-        "\"queries\":[{"
+            "\"queries\":[{"
-        "\"query\":{"
+            "\"query\":{"
-        "\"limit\":20,"
+            "\"limit\":20,"
-        "\"fullResponse\":true,"
+            "\"fullResponse\":true,"
-        "\"query\":{"
+            "\"query\":{"
-        "\"operator\":\"equals\","
+            "\"operator\":\"equals\","
-        "\"key\":\"mainAttributes.phase\","
+            "\"key\":\"mainAttributes.phase\","
-        "\"value\":\"testing\""
+            "\"value\":\"testing\""
-        "}},"
+            "}},"
-        "\"index\":0"
+            "\"index\":0"
-        "},{"
+            "},{"
-        "\"query\":{"
+            "\"query\":{"
-        "\"limit\":20,"
+            "\"limit\":20,"
-        "\"fullResponse\":true,"
+            "\"fullResponse\":true,"
-        "\"query\":{"
+            "\"query\":{"
-        "\"operator\":\"equals\","
+            "\"operator\":\"equals\","
-        "\"key\":\"mainAttributes.height\","
+            "\"key\":\"mainAttributes.height\","
-        "\"value\":\"testing\""
+            "\"value\":\"testing\""
-        "}},"
+            "}},"
-        "\"index\":1"
+            "\"index\":1"
-        "}]}";
+            "}]}";
 
     EXPECT_EQ(*query.genJson(), expected);
 }

core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc

@@ -172,6 +172,15 @@ TEST(QueryRequestTestV2, AttributesTest)
     EXPECT_EQ(out.str(), output_json);
 }
 
+TEST(QueryRequestTestV2, AttributeConditionExceptionTest)
+{
+    try {
+        Intelligence_IS_V2::createAttributeString("jey", (Intelligence_IS_V2::AttributeKeyType)5);
+    } catch (const IntelligenceException &e) {
+        EXPECT_THAT(string(e.what()), HasSubstr("Received illegal Attribute Type."));
+    }
+}
+
 TEST(QueryRequestTestV2, AndQueryTest)
 {
     QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc

@@ -1,5 +1,3 @@
-#include "intelligence_is_v2/query_response_v2.h"
-#include "intelligence_is_v2/asset_source_v2.h"
 #include "intelligence_is_v2/intelligence_types_v2.h"
 #include "intelligence_is_v2/data_string_v2.h"
 
@@ -31,12 +29,12 @@ public:
         try{
             ReadAttribute<DataString>("color", data).serialize(ar);
         } catch (exception &e) {
-            dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color";
+            dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color. Error:" + string(e.what());
         }
         try {
             ReadAttribute<DataString>("user", data1).serialize(ar);
         } catch (const exception &e) {
-            dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user";
+            dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user. Error:" + string(e.what());
         }
     }
 
@@ -47,9 +45,6 @@ private:
 
 TEST(QueryResponseTestV2, ReadAttributeTest)
 {
-    StrictMock<MockMainLoop> mock_ml;
-    NiceMock<MockTimeGet> time_get;
-    IntelligenceComponentV2 new_intelligence;
     DataString data;
     ReadAttribute<DataString> obj("user", data);
 
@@ -70,9 +65,6 @@ TEST(QueryResponseTestV2, ReadAttributeTest)
 
 TEST(QueryResponseTestV2, stringData1Test)
 {
-    StrictMock<MockMainLoop> mock_ml;
-    NiceMock<MockTimeGet> time_get;
-    IntelligenceComponentV2 new_intelligence;
     DataString data;
     stringData1 obj;
     string data_str(
@@ -95,11 +87,9 @@ TEST(QueryResponseTestV2, stringData1Test)
 
 TEST(QueryResponseTestV2, QueryResponseTestV2)
 {
-    StrictMock<MockMainLoop> mock_ml;
-    NiceMock<MockTimeGet> time_get;
-    IntelligenceComponentV2 new_intelligence;
     DataString data;
-    IntelligenceQueryResponse<stringData1> obj;
+    IntelligenceQueryResponseT<stringData1> obj;
+    IntelligenceQueryResponse obj2;
     string data_str(
         "{\n"
         "  \"assetCollections\": [\n"
@@ -161,12 +151,14 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
 
     stringstream ss(data_str);
     {
-        cereal::JSONInputArchive ar(ss);
+        obj.loadFromJson(ss.str());
-        obj.loadFromJson(ar);
+        obj2.loadFromJson(ss.str());
     }
 
     EXPECT_EQ(obj.getAmountOfAssets(), 2);
+    EXPECT_EQ(obj2.getAmountOfAssets(), 2);
     EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE);
+    EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE);
     EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1);
     EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip");
     EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1);
@@ -227,11 +219,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
 
 TEST(QueryResponseTestV2, MainAttributesTestV2)
 {
-    StrictMock<MockMainLoop> mock_ml;
-    NiceMock<MockTimeGet> time_get;
-    IntelligenceComponentV2 new_intelligence;
     DataString data;
-    IntelligenceQueryResponse<stringData1> obj;
+    IntelligenceQueryResponseT<stringData1> obj;
     string string_attribute(
         "{\n"
         "  \"assetCollections\": [\n"
@@ -277,7 +266,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
     stringstream ss(string_attribute);
     {
         cereal::JSONInputArchive ar(ss);
-        obj.loadFromJson(ar);
+        obj.serialize(ar);
     }
 
     map<string, vector<string>> attributes_map = obj.getData().begin()->getMainAttributes();
@@ -331,7 +320,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
     stringstream ss2(many_strings_attribute);
     {
         cereal::JSONInputArchive ar(ss2);
-        obj.loadFromJson(ar);
+        obj.serialize(ar);
     }
 
     map<string, vector<string>> attributes_map2 = obj.getData().begin()->getMainAttributes();
@@ -390,7 +379,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
     stringstream ss3(strings_vector_attribute);
     {
         cereal::JSONInputArchive ar(ss3);
-        obj.loadFromJson(ar);
+        obj.serialize(ar);
     }
 
     map<string, vector<string>> attributes_map3 = obj.getData().begin()->getMainAttributes();
@@ -403,11 +392,8 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
 
 TEST(QueryResponseTestV2, IntelligenceFailTest)
 {
-    StrictMock<MockMainLoop> mock_ml;
-    NiceMock<MockTimeGet> time_get;
-    IntelligenceComponentV2 new_intelligence;
     DataString data;
-    IntelligenceQueryResponse<stringData1> obj;
+    IntelligenceQueryResponseT<stringData1> obj;
     string status_fail_data_str(
         "{\n"
         "  \"assetCollections\": [\n"
@@ -456,7 +442,7 @@ TEST(QueryResponseTestV2, IntelligenceFailTest)
     {
         cereal::JSONInputArchive ar(ss);
         try {
-            obj.loadFromJson(ar);
+            obj.serialize(ar);
         } catch (exception &e) {
             EXPECT_EQ(e.what(), error_str);
         }

core/intelligence_is_v2/intelligence_request.cc

@@ -0,0 +1,128 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_request.h"
+#include "debug.h"
+#include "intelligence_comp_v2.h"
+#include "intelligence_is_v2/json_stream.h"
+
+
+using namespace Intelligence;
+using namespace std;
+
+USE_DEBUG_FLAG(D_INTELLIGENCE);
+
+static const unsigned int upper_assets_limit = 50;
+static const unsigned int upper_confidence_limit = 1000;
+
+Maybe<void>
+IntelligenceRequest::checkAssetsLimit() const
+{
+    for (const QueryRequest &query_request : queries) {
+        uint assets_limit = query_request.getAssetsLimit();
+        if (0 < assets_limit && assets_limit <= upper_assets_limit) continue;
+        dbgTrace(D_INTELLIGENCE)
+            << "Assets limit for request is "
+            << upper_assets_limit
+            << ", requests assets: "
+            << assets_limit;
+        return genError("Assets limit valid range is of [1, " + to_string(upper_assets_limit) + "]");
+    }
+    return Maybe<void>();
+}
+
+Maybe<void>
+IntelligenceRequest::checkMinConfidence() const
+{
+    for (const QueryRequest &query_request : queries) {
+        if (query_request.checkMinConfidence(upper_confidence_limit)) continue;
+        dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
+        return genError(
+            "Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
+        );
+    }
+    return Maybe<void>();
+}
+
+bool
+IntelligenceRequest::isPagingActivated() const
+{
+    if (!isPagingAllowed()) return false;
+    return queries.begin()->getCursorState().ok();
+}
+
+Maybe<bool>
+IntelligenceRequest::isPagingFinished() const
+{
+    if (!isPagingActivated()) return genError("Paging is not activated");
+    return queries.begin()->getCursorState().unpack() == CursorState::DONE;
+}
+
+Maybe<Intelligence_IS_V2::CursorState>
+IntelligenceRequest::getPagingStatus() const
+{
+    if (!isPagingAllowed()) return genError("Paging is not allowed");
+    return queries.begin()->getCursorState();
+}
+
+bool
+IntelligenceRequest::isPagingAllowed() const
+{
+    if (isBulk()) return false;
+    return true;
+}
+
+Maybe<std::string>
+IntelligenceRequest::genJson() const
+{
+    std::stringstream str_stream;
+    JsonStream json_stream(&str_stream, is_pretty);
+    {
+        cereal::JSONOutputArchive out_ar(json_stream);
+        if (isBulk()) {
+            out_ar.setNextName("queries");
+            out_ar.startNode();
+            out_ar.makeArray();
+            uint index = 0;
+            for (const auto &query : queries) {
+                out_ar.setNextName(nullptr);
+                out_ar.startNode();
+                out_ar.setNextName("query");
+                out_ar.startNode();
+                query.saveToJson(out_ar);
+                out_ar.finishNode();
+                out_ar(cereal::make_nvp("index", index));
+                out_ar.finishNode();
+                index++;
+            }
+            out_ar.finishNode();
+        } else {
+            queries.begin()->saveToJson(out_ar);
+        }
+    }
+
+    return str_stream.str();
+}
+
+bool
+IntelligenceRequest::loadJson(const string &json)
+{
+    response_from_fog = json;
+    return true;
+}
+
+Maybe<string>
+IntelligenceRequest::getResponseFromFog() const
+{
+    return response_from_fog;
+}

core/intelligence_is_v2/intelligence_response.cc

@@ -0,0 +1,74 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "intelligence_is_v2/intelligence_response.h"
+
+using namespace std;
+using namespace Intelligence;
+
+Maybe<void>
+Response::load()
+{
+    try {
+        stringstream in;
+        in.str(json_response);
+        cereal::JSONInputArchive in_ar(in);
+        if (is_bulk) {
+            IntelligenceQueryBulkResponse bulk_response;
+            bulk_response.serialize(in_ar);
+            unsigned int error_idx = 0;
+            unsigned int valid_idx = 0;
+            const auto &valid_response = bulk_response.getValid();
+            const auto &errors = bulk_response.getErrors();
+            responses.clear();
+            responses.reserve(size);
+            dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
+            for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
+                if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
+                    responses.push_back(valid_response[valid_idx].getResponse());
+                    dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
+                    valid_idx++;
+                } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
+                    responses.emplace_back();
+                    responses[query_idx].setFailInBulk();
+                    dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
+                    error_idx++;
+                } else {
+                    dbgWarning(D_INTELLIGENCE)
+                        << "Query index was not found neither in valid nor error responses, assuming error";
+                    responses[query_idx].setFailInBulk();
+                }
+            }
+        } else {
+            single_response.serialize(in_ar);
+        }
+    } catch(const std::exception &e) {
+        return genError("Load common data failed. Error: " + string(e.what()));
+    }
+    return {};
+}
+
+Intelligence_IS_V2::ResponseStatus
+Response::getResponseStatus() const
+{
+    if (!is_bulk) return single_response.getResponseStatus();
+
+    if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
+    for (const auto &response_itr : responses) {
+        if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
+            return ResponseStatus::IN_PROGRESS;
+        }
+    }
+
+    return ResponseStatus::DONE;
+}

core/intelligence_is_v2/intelligence_server.cc

@@ -0,0 +1,203 @@
+// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "intelligence_server.h"
+#include "config.h"
+#include "debug.h"
+#include "enum_array.h"
+#include "intelligence_comp_v2.h"
+
+USE_DEBUG_FLAG(D_INTELLIGENCE);
+
+using namespace Intelligence;
+using namespace std;
+
+static const string query_uri = "/api/v2/intelligence/assets/query";
+static const string queries_uri = "/api/v2/intelligence/assets/queries";
+static const string primary_port_setting = "local intelligence server primary port";
+static const string secondary_port_setting = "local intelligence server secondary port";
+
+Sender::Sender(IntelligenceRequest request) : request(request)
+{
+    i_message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
+    i_timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
+    i_mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
+    bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
+                                                            std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
+                                                            false;
+    if (getProfileAgentSettingWithDefault<bool>(crowdsec_enabled, "layer7AccessControl.crowdsec.enabled")) {
+        is_local_intelligence = true;
+    }
+
+    if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence")) {
+        is_local_intelligence = true;
+    }
+
+    auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
+    if (setting_server_ip.ok()) server_ip = *setting_server_ip;
+}
+
+Maybe<Response>
+Sender::sendIntelligenceRequest()
+{
+    if (server_ip.ok() && is_local_intelligence) {
+        auto response = sendQueryObjectToLocalServer(true);
+        if (response.ok()) return response;
+        dbgWarning(D_INTELLIGENCE) << "Failed to send query to primary port. Error" << response.getErr();
+        response = sendQueryObjectToLocalServer(false);
+        if (response.ok()) return response;
+        dbgWarning(D_INTELLIGENCE) << "Failed to send query to secondary port. Error" << response.getErr();
+    }
+
+    if (request.getPagingStatus().ok()) {
+        return sendMessage();
+    }
+
+    return sendQueryMessage();
+}
+
+Maybe<Response>
+Sender::sendQueryObjectToLocalServer(bool is_primary_port)
+{
+    auto local_port =  getSetting<uint>(
+        "intelligence",
+        is_primary_port ? primary_port_setting : secondary_port_setting
+    );
+
+    if (!local_port.ok()) return genError(
+        "Failed to send intelligence query to local server. Config Error number:"
+        + to_string(static_cast<uint>(local_port.getErr()))
+    );
+
+    server_port = *local_port;
+    conn_flags.reset();
+
+    auto res = sendQueryMessage();
+
+    server_port = genError("port unset after use");
+
+    return res;
+}
+
+Maybe<Response>
+Sender::sendQueryMessage()
+{
+    uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
+        20,
+        "intelligence",
+        "request overall timeout"
+    );
+
+    uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
+        5,
+        "intelligence",
+        "request lap timeout"
+    );
+
+    chrono::seconds request_overall_timeout = chrono::seconds(request_overall_timeout_conf);
+    chrono::seconds request_lap_timeout = chrono::seconds(request_lap_timeout_conf);
+
+    chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
+    chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
+    chrono::seconds seconds_since_start = chrono::seconds(0);
+    chrono::seconds seconds_since_last_lap = chrono::seconds(0);
+
+    Maybe<Response> res = genError("Uninitialized");
+    do {
+        res = sendMessage();
+
+        if (res.ok() && res->getResponseStatus() == ResponseStatus::IN_PROGRESS) {
+            i_mainloop->yield(true);
+        }
+
+        seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
+            i_timer->getMonotonicTime() - send_request_start_time
+        );
+
+        seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
+            i_timer->getMonotonicTime() - last_lap_time
+        );
+        last_lap_time = i_timer->getMonotonicTime();
+    } while (res.ok() &&
+        res->getResponseStatus() == ResponseStatus::IN_PROGRESS &&
+        seconds_since_start < request_overall_timeout &&
+        seconds_since_last_lap < request_lap_timeout
+    );
+
+    return res;
+}
+
+Maybe<Response>
+Sender::sendMessage()
+{
+        if (server_ip.ok() || server_port.ok()) {
+        if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set");
+        if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set");
+    } else if (!server_ip.ok() && !server_port.ok()) {
+        auto req_status = i_message->sendSyncMessage(
+            HTTPMethod::POST,
+            request.isBulk() ? queries_uri : query_uri,
+            request,
+            MessageCategory::INTELLIGENCE
+        );
+        if (req_status.ok()) {
+            return createResponse();
+        };
+        auto response_error = req_status.getErr().toString();
+        dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
+        return genError(
+            "Failed to send intelligence request. "
+            + req_status.getErr().getBody()
+            + " "
+            + req_status.getErr().toString()
+        );
+    }
+
+    dbgTrace(D_INTELLIGENCE)
+        << "Sending intelligence request with IP: "
+        << *server_ip
+        << " port: "
+        << *server_port
+        << " query_uri: "
+        << (request.isBulk() ? queries_uri : query_uri);
+
+    MessageMetadata req_md(*server_ip, *server_port, conn_flags);
+    auto req_status = i_message->sendSyncMessage(
+        HTTPMethod::POST,
+        query_uri,
+        request,
+        MessageCategory::INTELLIGENCE,
+        req_md
+    );
+    if (!req_status.ok()) {
+        auto response_error = req_status.getErr().toString();
+        dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
+        return genError(
+            "Failed to send intelligence request. "
+            + req_status.getErr().getBody()
+            + " "
+            + req_status.getErr().toString()
+        );
+    };
+    return createResponse();
+}
+
+Maybe<Response>
+Sender::createResponse()
+{
+    auto mb_json_body = request.getResponseFromFog();
+    if (!mb_json_body.ok()) return mb_json_body.passErr();
+    Response response(*mb_json_body, request.getSize(), request.isBulk());
+    auto load_status = response.load();
+    if (!load_status.ok()) return load_status.passErr();
+    return response;
+}

core/intelligence_is_v2/query_filter_v2.cc

@@ -146,18 +146,6 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
     }
 }
 
-Maybe<SerializableQueryCondition::ValueVariant>
-SerializableQueryFilter::getConditionValueByKey(const string &key) const
-{
-    for (const SerializableQueryCondition &condition : condition_operands) {
-        if (condition.getConditionType() == Condition::EQUALS && condition.getKey() == key) {
-            return condition.getValue();
-        }
-    }
-
-    return genError("Key not found.");
-}
-
 bool
 SerializableQueryFilter::isOperatorComp(const Operator &oper) const
 {

core/intelligence_is_v2/query_request_v2.cc

@@ -25,27 +25,6 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
 
 static const EnumArray<ObjectType, string> object_type_to_string_array{"asset", "zone", "configuration", "shortLived"};
 
-BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index)
-        :
-    request(_request),
-    index(_index)
-{}
-
-QueryRequest
-BulkQueryRequest::getQueryRequest() const
-{
-    return request;
-}
-
-void
-BulkQueryRequest::save(cereal::JSONOutputArchive &ar) const
-{
-    ar(
-        cereal::make_nvp("query", getQueryRequest()),
-        cereal::make_nvp("index",  index)
-    );
-}
-
 QueryRequest::QueryRequest(
     Condition condition_type,
     const string &key,
@@ -96,22 +75,7 @@ QueryRequest::convertObjectTypeToString() const
 void
 QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const
 {
-    ar(
+    save(ar);
-        cereal::make_nvp("limit", assets_limit),
-        cereal::make_nvp("fullResponse", full_response),
-        cereal::make_nvp("query", query)
-    );
-
-    auto objTypeString = convertObjectTypeToString();
-    if (objTypeString.ok()) {
-        ar(cereal::make_nvp("objectType", *objTypeString));
-    } else {
-        dbgTrace(D_INTELLIGENCE) << objTypeString.getErr();
-    }
-
-    if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second));
-    requested_attributes.save(ar);
-    query_types.save(ar);
 }
 
 void
@@ -214,7 +178,7 @@ QueryRequest::setAssetsLimit(uint _assets_limit)
 }
 
 bool
-QueryRequest::checkMinConfidence(uint upper_confidence_limit)
+QueryRequest::checkMinConfidence(uint upper_confidence_limit) const
 {
     return requested_attributes.checkMinConfidence(upper_confidence_limit);
 }
@@ -232,7 +196,7 @@ QueryRequest::isPagingActivated()
 }
 
 Maybe<CursorState>
-QueryRequest::getCursorState()
+QueryRequest::getCursorState() const
 {
     if (!cursor.ok()) return genError("Paging not activated");
     return cursor.unpack().first;

core/intelligence_is_v2/requested_attributes_v2.cc

@@ -54,7 +54,7 @@ SerializableAttributesMap::getAttributeByKey(const string &key) const
 }
 
 bool
-SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit)
+SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit) const
 {
     for (auto const &attribute : requested_attributes) {
         if (attribute.second == 0 || attribute.second > upper_confidence_limit) return false;

nodes/http_transaction_handler/CMakeLists.txt

@@ -15,6 +15,7 @@ target_link_libraries(cp-nano-http-transaction-handler
 	pcre2-posix
 	yajl_s
 	hiredis
+        maxminddb
 
 	-lshmem_ipc
 	-lnginx_attachment_util
@@ -38,6 +39,8 @@ target_link_libraries(cp-nano-http-transaction-handler
         ips
         keywords
 	l7_access_control
+	geo_location
+	http_geo_filter
 	-Wl,--end-group
 )
 

nodes/http_transaction_handler/main.cc

@@ -21,6 +21,8 @@
 #include "waap.h"
 #include "ips_comp.h"
 #include "keyword_comp.h"
+#include "http_geo_filter.h"
+#include "geo_location.h"
 
 int
 main(int argc, char **argv)
@@ -34,7 +36,9 @@ main(int argc, char **argv)
         RateLimit,
         WaapComponent,
         IPSComp,
-        KeywordComp
+        KeywordComp,
+        GeoLocation,
+        HttpGeoFilter
     > comps;
 
     comps.registerGlobalValue<bool>("Is Rest primary routine", true);

nodes/orchestration/package/open-appsec-ctl.sh

@@ -1027,6 +1027,10 @@ run_status() # Initials - rs
     for service in $all_services; do
         print_single_service_status "$service"
     done
+
+    if command -v getenforce &>/dev/null && [ "$(getenforce)" != "Disabled" ]; then
+        echo "SELinux has been detected, which could potentially disrupt the agent's normal operation."
+    fi
 }
 
 run_load_settings() # Initials - rls