diff --git a/README.md b/README.md index 2ffb9f9..f7fd96d 100644 --- a/README.md +++ b/README.md @@ -106,12 +106,13 @@ Before compiling the services, you'll need to ensure the latest development vers * cURL * Redis * Hiredis +* MaxmindDB An example of installing the packages on Alpine: ```bash $ apk update - $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis + $ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis libmaxminddb-dev ``` ## Compiling and packaging the agent code diff --git a/components/include/health_checker.h b/components/include/health_checker.h index af8b3e4..80932c8 100755 --- a/components/include/health_checker.h +++ b/components/include/health_checker.h @@ -19,6 +19,7 @@ #include "i_socket_is.h" #include "i_health_check_manager.h" #include "i_shell_cmd.h" +#include "i_orchestration_status.h" #include "component.h" class HealthChecker @@ -27,7 +28,8 @@ class HealthChecker Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume + Singleton::Consume, + Singleton::Consume { public: HealthChecker(); diff --git a/components/include/reverse_proxy_defaults.h b/components/include/reverse_proxy_defaults.h new file mode 100644 index 0000000..252936b --- /dev/null +++ b/components/include/reverse_proxy_defaults.h @@ -0,0 +1,46 @@ +#ifndef __REVERSE_PROXY_MANAGER_DEFAULTS_H__ +#define __REVERSE_PROXY_MANAGER_DEFAULTS_H__ + +#include + +static const std::string product_name = getenv("DOCKER_RPM_ENABLED") ? "CloudGuard AppSec" : "AppSec Gateway"; +static const std::string default_cp_cert_file = "/etc/cp/cpCert.pem"; +static const std::string default_cp_key_file = "/etc/cp/cpKey.key"; +static const std::string default_rpm_conf_path = "/etc/cp/conf/rpmanager/"; +static const std::string default_certificate_path = "/etc/cp/rpmanager/certs"; +static const std::string default_additional_files_path = "/etc/cp/conf/rpmanager/include"; +static const std::string default_server_config = "additional_server_config.conf"; +static const std::string default_location_config = "additional_location_config.conf"; +static const std::string default_trusted_ca_suffix = "_user_ca_bundle.crt"; +static const std::string default_nginx_log_files_path = "/var/log/nginx/"; +static const std::string default_log_files_host_path = "/var/log/nano_agent/rpmanager/nginx_log/"; +static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers"; +static const std::string default_template_path = "/etc/cp/conf/rpmanager/nginx-template-clear"; +static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/"; +static const std::string default_server_certificate_path = "/etc/cp/rpmanager/certs/sslCertificate_"; +static const std::string default_server_certificate_key_path = "/etc/cp/rpmanager/certs/sslPrivateKey_"; +static const std::string default_container_name = "cp_nginx_gaia"; +static const std::string default_docker_image = "cp_nginx_gaia"; +static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/nginx.conf"; +static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template"; +static const std::string default_nginx_config_include_file = + "/etc/cp/conf/rpmanager/servers/nginx_conf_include"; +static const std::string default_global_conf_include_template = + "/etc/cp/conf/rpmanager/nginx-conf-include-template"; +static const std::string default_global_conf_include_template_no_responses = + "/etc/cp/conf/rpmanager/nginx-conf-include-template-no-responses"; +static const std::string default_cloud_vendor_file = "/etc/cp/conf/rpmanager/cloud-vendor.json"; +static const std::string default_cloud_cert_location = "/tmp/"; +static const std::string default_dns_resolver_file = "/etc/resolv.conf"; +static const std::string default_nginx_multi_lines_key = "nginxIncludeLines"; +static const std::string default_ip = "127.0.0.1"; +static const std::string default_aws_resolver_ip = "169.254.169.253"; +static const std::string default_azure_resolver_ip = "168.63.129.16"; +static const std::string default_syslog_socket_address = "127.0.0.1:1514"; +static const std::string rpm_full_load_path = "/tmp/rpm_full_load"; +static const std::string rpm_partial_load_path = "/tmp/rpm_partial_load"; +static const std::string first_rpm_policy_load_path = "/tmp/first_rpm_policy_load"; + +static const int default_port = 5555; + +#endif //__REVERSE_PROXY_MANAGER_DEFAULTS_H__ diff --git a/components/security_apps/CMakeLists.txt b/components/security_apps/CMakeLists.txt index ee0e943..d9dd887 100644 --- a/components/security_apps/CMakeLists.txt +++ b/components/security_apps/CMakeLists.txt @@ -1,4 +1,4 @@ -#add_subdirectory(http_geo_filter) +add_subdirectory(http_geo_filter) add_subdirectory(ips) add_subdirectory(layer_7_access_control) add_subdirectory(local_policy_mgmt_gen) diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index e655ca4..1fde746 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -69,19 +69,11 @@ checkIDP(shared_ptr file_stream) #if defined(gaia) || defined(smb) Maybe -checkIsCpviewRunning(const string &command_output) +checkIsInstallHorizonTelemetrySucceeded(const string &command_output) { - if (command_output == "true" || command_output == "false") return command_output; + if (command_output == "" ) return string("false"); - return genError("cpview is not running"); -} - -Maybe -checkIsCPotelcolGRET64(const string &command_output) -{ - if (command_output == "true" || command_output == "false") return command_output; - - return genError("CPotelcol is not installed or its take is below T64"); + return command_output; } Maybe @@ -103,6 +95,10 @@ checkCanUpdateSDWanData(const string &command_output) Maybe getMgmtObjType(const string &command_output) { + if (getenv("WAAP_DIR")) { + return string("CloudGuard WAF Gateway"); + } + if (!command_output.empty()) { if (command_output[0] == '1') return string("management"); if (command_output[0] == '0') return string("gateway"); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 5494e35..df0f164 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -39,14 +39,10 @@ SHELL_PRE_CMD("read sdwan data", #ifdef SHELL_CMD_HANDLER #if defined(gaia) || defined(smb) SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType) -SHELL_CMD_HANDLER("isCpviewRunning", - "pidof cpview_api_service > /dev/null 2>&1 && [ -f $CPDIR/conf/cpview_api_service.version ] " - "&& echo 'true' || echo 'false'", - checkIsCpviewRunning) -SHELL_CMD_HANDLER("isCPotelcolGRET64", - "grep -A 10 '(BUNDLE_CPOTELCOL_AUTOUPDATE' ${CPDIR}/registry/HKLM_registry.data | " - "awk '/SU_Build_Take/{val = substr($2, 2, length($2)-2); if (val >=64) print \"true\"; else print \"false\" }'", - checkIsCPotelcolGRET64) +SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry", + "[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] " + "&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''", + checkIsInstallHorizonTelemetrySucceeded) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( "canUpdateSDWanData", diff --git a/components/security_apps/orchestration/health_check/health_check.cc b/components/security_apps/orchestration/health_check/health_check.cc index 6d61c7d..d910023 100755 --- a/components/security_apps/orchestration/health_check/health_check.cc +++ b/components/security_apps/orchestration/health_check/health_check.cc @@ -18,6 +18,7 @@ #include #include +#include "reverse_proxy_defaults.h" #include "config.h" #include "log_generator.h" #include "health_check_manager.h" @@ -28,6 +29,8 @@ using namespace ReportIS; USE_DEBUG_FLAG(D_HEALTH_CHECK); +static const std::string all_services_running = "/tmp/wd.all_running"; + class HealthChecker::Impl { public: @@ -36,6 +39,7 @@ public: { i_mainloop = Singleton::Consume::by(); i_socket = Singleton::Consume::by(); + i_orchestration_status = Singleton::Consume::by(); initConfig(); initServerSocket(); @@ -255,35 +259,33 @@ private: { if (!getenv("DOCKER_RPM_ENABLED")) return HealthCheckStatus::IGNORED; - static const string standalone_cmd = "/usr/sbin/cpnano -s --docker-rpm; echo $?"; - static int timeout_tolerance = 1; - static HealthCheckStatus health_status = HealthCheckStatus::HEALTHY; - - dbgTrace(D_HEALTH_CHECK) << "Checking the standalone docker health status with command: " << standalone_cmd; - - auto maybe_result = Singleton::Consume::by()->getExecOutput(standalone_cmd, 5000); - if (!maybe_result.ok()) { - if (maybe_result.getErr().find("Reached timeout") != string::npos) { - dbgWarning(D_HEALTH_CHECK) - << "Reached timeout while querying standalone health status, attempt number: " - << timeout_tolerance; - - return health_status == HealthCheckStatus::UNHEALTHY || timeout_tolerance++ > 3 ? - HealthCheckStatus::UNHEALTHY : - health_status; - } - - dbgWarning(D_HEALTH_CHECK) << "Unable to get the standalone docker status. Returning unhealthy status."; + if (i_orchestration_status->getPolicyVersion().empty()) { + dbgTrace(D_HEALTH_CHECK) << "Policy version is empty, returning unhealthy status"; return HealthCheckStatus::UNHEALTHY; } - dbgTrace(D_HEALTH_CHECK) << "Got response: " << maybe_result.unpack(); - auto response = NGEN::Strings::removeTrailingWhitespaces(maybe_result.unpack()); + if (!NGEN::Filesystem::exists(all_services_running)) { + dbgTrace(D_HEALTH_CHECK) << all_services_running << " does not exist, returning unhealthy status"; + return HealthCheckStatus::UNHEALTHY; + } - if (response.back() == '1') return health_status = HealthCheckStatus::UNHEALTHY; + if (NGEN::Filesystem::exists(rpm_full_load_path)) { + dbgTrace(D_HEALTH_CHECK) << rpm_full_load_path << " exists, returning healthy status"; + return HealthCheckStatus::HEALTHY; + } - timeout_tolerance = 1; - return health_status = (response.back() == '0') ? HealthCheckStatus::HEALTHY : HealthCheckStatus::DEGRADED; + if (NGEN::Filesystem::exists(rpm_partial_load_path)) { + dbgTrace(D_HEALTH_CHECK) << rpm_partial_load_path << " exists, returning degraded status"; + return HealthCheckStatus::DEGRADED; + } + + if (!NGEN::Filesystem::exists(first_rpm_policy_load_path)) { + dbgTrace(D_HEALTH_CHECK) << "Could not load latest RPM policy, returning degraded status"; + return HealthCheckStatus::DEGRADED; + } + + dbgTrace(D_HEALTH_CHECK) << "RPM is not loaded, returning unhealthy status"; + return HealthCheckStatus::UNHEALTHY; } bool @@ -390,7 +392,7 @@ private: if (standalone_status == HealthCheckStatus::UNHEALTHY) { dbgDebug(D_HEALTH_CHECK) - << "Standalone status in unhealthy, returning the following response: " + << "Standalone status is unhealthy, returning the following response: " << failure_response; i_socket->writeData(curr_client_socket, failure_response_buffer); closeCurrentSocket(curr_client_socket, curr_routine_id); @@ -439,6 +441,7 @@ private: I_MainLoop *i_mainloop = nullptr; I_Socket *i_socket = nullptr; I_Health_Check_Manager *i_health_check_manager = nullptr; + I_OrchestrationStatus *i_orchestration_status = nullptr; }; HealthChecker::HealthChecker() : Component("HealthChecker"), pimpl(make_unique()) {} diff --git a/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc b/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc index c3c1396..ecf051e 100755 --- a/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc +++ b/components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc @@ -7,6 +7,7 @@ #include "mock/mock_socket_is.h" #include "mock/mock_mainloop.h" #include "mock/mock_shell_cmd.h" +#include "mock/mock_orchestration_status.h" #include "health_check_manager.h" #include "config.h" @@ -65,6 +66,7 @@ public: AgentDetails agent_details; StrictMock mock_socket; NiceMock mock_shell_cmd; + NiceMock mock_orchestration_status; I_Socket::socketFd server_socket = -1; Context ctx; ConfigComponent config; @@ -72,7 +74,6 @@ public: I_MainLoop::Routine connection_handler_routine; I_MainLoop::Routine client_connection_handler_routine; I_MainLoop::Routine handle_probe_routine; - //StrictMock mock_health_check_manager; HealthCheckManager health_check_manager; I_Health_Check_Manager *i_health_check_manager; }; diff --git a/components/security_apps/orchestration/include/fog_authenticator.h b/components/security_apps/orchestration/include/fog_authenticator.h index a80e593..16e448a 100755 --- a/components/security_apps/orchestration/include/fog_authenticator.h +++ b/components/security_apps/orchestration/include/fog_authenticator.h @@ -102,6 +102,7 @@ public: Maybe authenticateAgent() override; void setAddressExtenesion(const std::string &extension) override; + static std::string getUserEdition(); protected: class UserCredentials @@ -142,7 +143,6 @@ protected: std::string base64Encode(const std::string &in) const; std::string buildBasicAuthHeader(const std::string &username, const std::string &pass) const; std::string buildOAuth2Header(const std::string &token) const; - std::string getUserEdition() const; // This apps which the orchestrations requires them from Fog. std::vector required_security_apps; diff --git a/components/security_apps/orchestration/include/hybrid_communication.h b/components/security_apps/orchestration/include/hybrid_communication.h index 9c383f6..407a1ad 100755 --- a/components/security_apps/orchestration/include/hybrid_communication.h +++ b/components/security_apps/orchestration/include/hybrid_communication.h @@ -56,7 +56,9 @@ public: private: Maybe getNewVersion(); + void doLocalFogOperations(const std::string &policy) const; + std::string tuning_host; I_DeclarativePolicy *i_declarative_policy = nullptr; }; diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 44d3179..614af4a 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -41,6 +41,7 @@ #include "env_details.h" #include "hybrid_communication.h" #include "agent_core_utilities.h" +#include "fog_communication.h" using namespace std; using namespace chrono; @@ -190,7 +191,8 @@ public: ); auto orchestration_tools = Singleton::Consume::by(); - orchestration_tools->getClusterId(); + + if (getAttribute("no-setting", "IGNORE_CLUSTER_ID") != "TRUE") orchestration_tools->getClusterId(); hybrid_mode_metric.init( "Watchdog Metrics", @@ -1484,6 +1486,8 @@ private: agent_data_report << AgentReportFieldWithLabel("managedMode", "management"); } + agent_data_report << AgentReportFieldWithLabel("userEdition", FogCommunication::getUserEdition()); + #if defined(gaia) || defined(smb) if (i_details_resolver->compareCheckpointVersion(8100, greater_equal())) { agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER81", "true"); diff --git a/components/security_apps/orchestration/update_communication/fog_helper_open_source.cc b/components/security_apps/orchestration/update_communication/fog_helper_open_source.cc index 6b7de3a..4bb4e40 100755 --- a/components/security_apps/orchestration/update_communication/fog_helper_open_source.cc +++ b/components/security_apps/orchestration/update_communication/fog_helper_open_source.cc @@ -3,7 +3,7 @@ #include std::string -FogAuthenticator::getUserEdition() const +FogAuthenticator::getUserEdition() { return "community"; } diff --git a/components/security_apps/orchestration/update_communication/hybrid_communication.cc b/components/security_apps/orchestration/update_communication/hybrid_communication.cc index 6bf54aa..1b2fd27 100755 --- a/components/security_apps/orchestration/update_communication/hybrid_communication.cc +++ b/components/security_apps/orchestration/update_communication/hybrid_communication.cc @@ -27,15 +27,19 @@ using namespace std; USE_DEBUG_FLAG(D_ORCHESTRATOR); -#define TUNING_HOST_ENV_NAME "TUNING_HOST" -static const string defaultTuningHost = "appsec-tuning-svc"; +static const string agent_resource_api = "/api/v2/agents/resources"; void HybridCommunication::init() { + dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component"; + FogAuthenticator::init(); i_declarative_policy = Singleton::Consume::from(); - dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component"; + + auto env_tuning_host = getenv("TUNING_HOST"); + tuning_host = env_tuning_host != nullptr ? env_tuning_host : "appsec-tuning-svc"; + if (getConfigurationFlag("otp") != "") { otp = getConfigurationFlag("otp"); } else { @@ -49,10 +53,9 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) string manifest_checksum = ""; dbgTrace(D_ORCHESTRATOR) << "Getting updates in Hybrid Communication"; if (access_token.ok()) { - static const string check_update_str = "/api/v2/agents/resources"; auto request_status = Singleton::Consume::by()->sendSyncMessage( HTTPMethod::POST, - check_update_str, + agent_resource_api, request ); @@ -78,45 +81,7 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request) string policy_response = i_declarative_policy->getUpdate(request); - auto env = Singleton::Consume::by()->getEnvType(); - if (env == EnvType::K8S && !policy_response.empty()) { - dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host"; - I_AgentDetails *agentDetails = Singleton::Consume::by(); - - auto get_tuning_host = []() - { - static string tuning_host; - if (tuning_host != "") return tuning_host; - - char* tuning_host_env = getenv(TUNING_HOST_ENV_NAME); - if (tuning_host_env != NULL) { - tuning_host = string(tuning_host_env); - return tuning_host; - } - dbgWarning(D_ORCHESTRATOR) << "tuning host is not set. using default"; - tuning_host = defaultTuningHost; - - return tuning_host; - }; - - MessageMetadata update_policy_crd_md(get_tuning_host(), 80); - update_policy_crd_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId()); - UpdatePolicyCrdObject policy_change_object(policy_response); - - auto i_messaging = Singleton::Consume::by(); - bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse( - HTTPMethod::POST, - "/api/update-policy-crd", - policy_change_object, - MessageCategory::GENERIC, - update_policy_crd_md - ); - if (!tuning_req_status) { - dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification"; - } else { - dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification"; - } - } + doLocalFogOperations(policy_response); request = CheckUpdateRequest(manifest_checksum, policy_response, "", "", "", ""); @@ -141,10 +106,9 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file, auto unpacked_access_token = access_token.unpack().getToken(); - static const string file_attribute_str = "/api/v2/agents/resources/"; auto attribute_file = Singleton::Consume::by()->downloadFile( resourse_file.getRequestMethod(), - file_attribute_str + resourse_file.getFileName(), + agent_resource_api + '/' + resourse_file.getFileName(), file_path ); if (!attribute_file.ok()) { @@ -164,3 +128,33 @@ HybridCommunication::sendPolicyVersion(const string &policy_version, const strin policy_version.empty(); return Maybe(); } + +void +HybridCommunication::doLocalFogOperations(const string &policy) const +{ + if (policy.empty()) return; + if (Singleton::Consume::by()->getEnvType() != EnvType::K8S) return; + + dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host"; + + MessageMetadata update_policy_crd_md(tuning_host, 80); + const auto &tenant_id = Singleton::Consume::by()->getTenantId(); + update_policy_crd_md.insertHeader("X-Tenant-Id", tenant_id); + + UpdatePolicyCrdObject policy_change_object(policy); + auto i_messaging = Singleton::Consume::by(); + bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse( + HTTPMethod::POST, + "/api/update-policy-crd", + policy_change_object, + MessageCategory::GENERIC, + update_policy_crd_md + ); + + if (!tuning_req_status) { + dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification"; + } else { + dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification"; + } + +} diff --git a/components/security_apps/waap/waap_clib/CMakeLists.txt b/components/security_apps/waap/waap_clib/CMakeLists.txt index 3e363eb..09e249b 100755 --- a/components/security_apps/waap/waap_clib/CMakeLists.txt +++ b/components/security_apps/waap/waap_clib/CMakeLists.txt @@ -85,6 +85,7 @@ add_library(waap_clib ParserGql.cc ParserPercentEncode.cc ParserPairs.cc + Waf2Util2.cc ) add_definitions("-Wno-unused-function") diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index d224a0e..ee32993 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -864,10 +864,6 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( return rc; } -static bool err = false; -static const SingleRegex json_detector_re("\\A[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", err, "json_detector"); -static const SingleRegex json_quoteless_detector_re("^[{\\[][[,0-9nul\\]]+", err, "json_quoteless_detector"); - //intended to keep and process all types of leftovers detected as separate cases for parsing int DeepParser::createUrlParserForJson( @@ -1103,11 +1099,7 @@ DeepParser::createInternalParser( } else { dbgTrace(D_WAAP_DEEP_PARSER) << "attempt to find JSON by '{' or '['"; bool percent_encoded_doublequote_detected = cur_val.find("%22") != std::string::npos; - if (json_detector_re.hasMatch(cur_val) - && (valueStats.hasDoubleQuote - || json_quoteless_detector_re.hasMatch(cur_val) - || percent_encoded_doublequote_detected)) { - // JSON value detected + if (Waap::Util::isValidJson(cur_val)) { if (percent_encoded_doublequote_detected && !valueStats.hasDoubleQuote) { // We have JSOn but it %-encoded, first start percent decoding for it. Very narrow case dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file from percent decoding"; diff --git a/components/security_apps/waap/waap_clib/ParserBase.cc b/components/security_apps/waap/waap_clib/ParserBase.cc index 12ba64a..82785b4 100755 --- a/components/security_apps/waap/waap_clib/ParserBase.cc +++ b/components/security_apps/waap/waap_clib/ParserBase.cc @@ -19,7 +19,8 @@ USE_DEBUG_FLAG(D_WAAP_PARSER); // Max size for key and value that can be stored in memory (per thread) #define MAX_KEY_SIZE 64*1024 -#define MAX_VALUE_SIZE 64*1024 +#define MAX_VALUE_SIZE 16*1024 +#define MAX_PROCESSING_BUFFER_SIZE 64*1024 BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) : m_receiver(receiver), @@ -69,14 +70,14 @@ int BufferedReceiver::onValue(const char *v, size_t v_len) << m_parser_depth; while (v_len > 0) { // Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size - size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size()); + size_t bytesToFill = std::min(v_len, MAX_PROCESSING_BUFFER_SIZE - m_value.size()); m_value += std::string(v, bytesToFill); // Update v and v_len (input buffer) to reflect that we already consumed part (or all) of it v += bytesToFill; v_len -= bytesToFill; // Only push full buffers to the m_receiver - if (m_value.size() == MAX_VALUE_SIZE) { + if (m_value.size() >= MAX_VALUE_SIZE) { // The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag dbgTrace(D_WAAP_PARSER) << " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag" diff --git a/components/security_apps/waap/waap_clib/ParserJson.h b/components/security_apps/waap/waap_clib/ParserJson.h index 0ae3010..636c310 100755 --- a/components/security_apps/waap/waap_clib/ParserJson.h +++ b/components/security_apps/waap/waap_clib/ParserJson.h @@ -22,8 +22,9 @@ #include "yajl/yajl_parse.h" #include "singleton.h" #include "i_oa_schema_updater.h" - -#define FIRST_JSON_BUFFER_SIZE 4 // must buffer at least 4 first bytes to allow unicode autodetection (BOM). +// must be at least 4 first bytes to allow unicode autodetection (BOM). +// BUT... reduced to 1 in order to allow better work of schema validation and API discovery +#define FIRST_JSON_BUFFER_SIZE 1 typedef size_t yajl_size_t; diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index 903c064..f41d9cc 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -312,6 +312,7 @@ Waf2Transaction::Waf2Transaction() : m_responseInjectReasons(), m_index(-1), m_triggerLog(), + is_schema_validation(false), m_waf2TransactionFlags() {} @@ -343,6 +344,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr pWaapAssetState m_responseInjectReasons(), m_index(-1), m_triggerLog(), + is_schema_validation(false), m_waf2TransactionFlags() {} @@ -515,7 +517,6 @@ bool Waf2Transaction::checkIsScanningRequired() m_siteConfig = &m_ngenAPIConfig; auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL; result |= m_siteConfig->get_WebAttackMitigation(); - if(rateLimitingPolicy) { result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus(); } diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index b02f030..e05087a 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -345,6 +345,7 @@ private: // Cached pointer to const triggerLog (hence mutable) mutable std::shared_ptr m_triggerLog; + bool is_schema_validation = false; Waf2TransactionFlags m_waf2TransactionFlags; }; diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index 931cb9c..a5a2b3c 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -2055,31 +2055,18 @@ string extractForwardedIp(const string &x_forwarded_hdr_val) return forward_ip; } - bool isUuid(const string& str) { if (str.length() != 36) { return false; } - static bool err; static const SingleRegex uuid_detector_re( "[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}", err, "uuid_detector" ); - // Check if the string matches the UUID format + // Check if the string matches the UUID format return uuid_detector_re.hasMatch(str); -/* - boost::cmatch what; - try { - static const boost::regex uuidRegex("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}"); - // Check if the string matches the UUID format - return boost::regex_match(str.c_str(), what, uuidRegex); - } catch (std::runtime_error &e) { - dbgError(D_WAAP) << e.what(); - } - return false; -*/ } bool isIpAddress(const string &ip_address) diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index 9ef2d0a..9222ede 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -876,6 +876,8 @@ namespace Util { std::string::const_iterator e, std::string &repl); + bool isValidJson(const std::string &input); + bool detectJSONasParameter(const std::string &s, std::string &key, std::string &value); diff --git a/components/security_apps/waap/waap_clib/Waf2Util2.cc b/components/security_apps/waap/waap_clib/Waf2Util2.cc new file mode 100644 index 0000000..63b50cc --- /dev/null +++ b/components/security_apps/waap/waap_clib/Waf2Util2.cc @@ -0,0 +1,632 @@ +#include "Waf2Util.h" +#include + +namespace Waap { +namespace Util { +using namespace std; + +static const char *trueString = "true"; +static const size_t trueStringLen = strlen(trueString); +static const char *falseString = "false"; +static const size_t falseStringLen = strlen(falseString); +static const char *nullString = "null"; +static const size_t nullStringLen = strlen(nullString); +static const char *quoteString = "%22"; +static const size_t quoteStringLen = strlen(quoteString); + +int +isAlignedPrefix( + const char *sample, + const size_t sample_len, + const char *buffer, + const size_t buffer_len) +{ + size_t lookup_len = 0; + if (buffer_len < sample_len) { + lookup_len = buffer_len; + } else { + lookup_len = sample_len; + } + if (strncmp(sample, buffer, lookup_len) == 0) { + return lookup_len; + } + return -1; +} + +int +isBoolean( + const char *buffer, + const size_t buffer_len) +{ + int status; + + status = isAlignedPrefix(trueString, trueStringLen, buffer, buffer_len); + if (status >= 0) { + return status; + } + status = isAlignedPrefix(falseString, falseStringLen, buffer, buffer_len); + if (status >= 0) { + return status; + } + status = isAlignedPrefix(nullString, nullStringLen, buffer, buffer_len); + if (status >= 0) { + return status; + } + return -1; +} + +bool +isValidExponent( + const char * buffer, + const size_t buffer_len, + size_t *i) +{ + if (buffer_len == *i + 1) { + return true; // e or E is the last char in buffer + } + if (*i + 1 < buffer_len && (isdigit(buffer[*i + 1]) || (buffer[*i + 1] == '+' || buffer[*i + 1] == '-'))) { + (*i) += 1; + if (isdigit(buffer[*i + 1])) { + return true; + } + } else { + return false; + } + return false; +} + +bool +isObjectStart(const char c, int *object_count) +{ + if (c == '{') { + (*object_count)++; + return true; + } + return false; +} + +bool +isObjectEnd(const char c, int *object_count) +{ + if (c == '}') { + (*object_count)--; + return true; + } + return false; +} + +bool +isArrayStart(const char c, int *array_count) +{ + if (c == '[') { + (*array_count)++; + return true; + } + return false; +} + +bool +isArrayEnd(const char c, int *array_count) +{ + if (c == ']') { + (*array_count)--; + return true; + } + return false; +} + +bool +isValidJson(const std::string &input) +{ + static const size_t MAX_JSON_INSPECT_SIZE = 16; + + enum state + { + S_START, // 0 + S_OBJECT_START, // 1 + S_OBJECT_END, // 2 + S_ARRAY_START, // 3 + S_ARRAY_END, // 4 + S_NUMBER, // 5 + S_NUMBER_END, // 6 + S_STRING_START, // 7 + S_STRING_BODY, // 8 + S_STRING_END, // 9 + S_VARIABLE_START, // 10 + S_VARIABLE_BODY, // 11 + S_VARIABLE_END, // 12 + S_COMMA, // 13 + S_COLON, // 14 + S_BOOLEAN, // 15 + S_ERROR, // 16 + S_END // 17 + }; + + state m_state; + bool encoded = false; + size_t i = 0; + char c; + const char *buf = input.c_str(); + size_t len = input.length(); + int array_count = 0; + int object_count = 0; + int status; + if (len < 2) { + return false; + } + + m_state = S_START; + while (i < len && i < MAX_JSON_INSPECT_SIZE) { + c = buf[i]; + if (c == 0x0) { // UTF16 to UTF8 support + i++; + continue; + } + switch (m_state) { + case S_START: + if (isObjectStart(c, &object_count)) { + m_state = S_OBJECT_START; + break; + } + if (isArrayStart(c, &array_count)) { + m_state = S_ARRAY_START; + break; + } + m_state = S_ERROR; + break; // S_START + + case S_OBJECT_START: + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + if (c == '\"') { + m_state = S_VARIABLE_START; + break; + } + if (isspace(c)) { + break; + } + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_VARIABLE_START; + encoded = true; + i += status - 1; + break; + } + m_state = S_ERROR; + break; // object_start + + case S_ARRAY_START: + if (isObjectStart(c, &object_count)) { + m_state = S_OBJECT_START; + break; + } + if (isArrayStart(c, &array_count)) { + // keep state unchanged + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isdigit(c)) { + m_state = S_NUMBER; + break; + } + if (c == '-') { + if (i + 1 == len) { // End of buffer case + m_state = S_NUMBER; + break; + } + if (i + 1 < len && isdigit(buf[i + 1])) { + m_state = S_NUMBER; + i++; + break; + } + m_state = S_ERROR; + break; + } + if (isspace(c)) { + break; + } + if (c == '\"') { + m_state = S_STRING_START; + break; + } + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_STRING_START; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + status = isBoolean(buf + i, len - i); + if (status >= 0) { + m_state = S_BOOLEAN; + i += status - 1; + break; + } + m_state = S_ERROR; + break; // array_start + + case S_OBJECT_END: + if (isspace(c)) { + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org + m_state = S_ARRAY_START; + break; + } + if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org + m_state = S_OBJECT_START; + break; + } + m_state = S_ERROR; + break; // S_OBJECT_END + + case S_ARRAY_END: + if (isspace(c)) { + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org + m_state = S_ARRAY_START; + break; + } + if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org + m_state = S_OBJECT_START; + break; + } + m_state = S_ERROR; + break; // S_ARRAY_END + + case S_NUMBER: + if (isdigit(c)) { + break; + } + if (c == '.') { + if (i + 1 == len) { // End of buffer case + m_state = S_NUMBER; + break; + } + if (i + 1 < len && isdigit(buf[i + 1])) { + m_state = S_NUMBER; + i++; + break; + } + m_state = S_ERROR; + break; + } + if (c == 'e' || c == 'E') { + if (isValidExponent(buf, len, &i)) { + m_state = S_NUMBER; + break; + } + m_state = S_ERROR; + break; + } + if (isspace(c)) { + m_state = S_NUMBER_END; + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + m_state = S_ERROR; + break; // S_NUMBER + + case S_NUMBER_END: + if (isspace(c)) { + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + m_state = S_ERROR; + break; // S_NUMBER_END + + case S_STRING_START: + if (c == '\"') { + m_state = S_STRING_END; + break; + } + if (encoded) { // url_encoded quote + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_STRING_END; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + } + m_state = S_STRING_BODY; + break; // S_STRING_START + + case S_STRING_BODY: + if (c == '\"') { + if (buf[i - 1] == '\\' && buf[i - 2] != '\\') { + m_state = S_STRING_BODY; + break; + } else { + m_state = S_STRING_END; + break; + } + } + if (encoded) { // url_encoded quote + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_STRING_END; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + } + m_state = S_STRING_BODY; + break; // S_STRING_BODY; + + case S_STRING_END: + if (isspace(c)) { + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + if (c == ':') { + m_state = S_COLON; + break; + } + m_state = S_ERROR; + break; // s_sting_end + + case S_VARIABLE_START: + if (c == '\"') { + m_state = S_VARIABLE_END; + break; + } + if (encoded) { // url_encoded quote + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_VARIABLE_END; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + } + m_state = S_VARIABLE_BODY; + break; // S_VARIABLE_START + + case S_VARIABLE_BODY: + if (c == '\"') { + if (buf[i - 1] == '\\' && buf[i - 2] != '\\') { + m_state = S_VARIABLE_BODY; + break; + } else { + m_state = S_VARIABLE_END; + break; + } + } + if (encoded) { // url_encoded quote + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_VARIABLE_END; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + } + m_state = S_VARIABLE_BODY; + break; // S_VARIABLE_BODY + + case S_VARIABLE_END: + if (isspace(c)) { + break; + } + if (c == ':') { + m_state = S_COLON; + break; + } + m_state = S_ERROR; + break; // S_VARIABLE_END + + case S_COMMA: + if (isObjectStart(c, &object_count)) { + m_state = S_OBJECT_START; + break; + } + if (isArrayStart(c, &array_count)) { + m_state = S_ARRAY_START; + break; + } + if (isdigit(c)) { + m_state = S_NUMBER; + break; + } + if (c == '-') { + if (i + 1 == len) { // End of buffer case + m_state = S_NUMBER; + break; + } + if (i + 1 < len && isdigit(buf[i + 1])) { + m_state = S_NUMBER; + i++; + break; + } + m_state = S_ERROR; + break; + } + if (isspace(c)) { + break; + } + if (c == '\"') { + m_state = S_STRING_START; + break; + } + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_STRING_START; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + status = isBoolean(buf + i, len - i); + if (status >= 0) { + m_state = S_BOOLEAN; + i += status - 1; + break; + } + m_state = S_ERROR; + break; // S_COMMA + + case S_COLON: + if (isObjectStart(c, &object_count)) { + m_state = S_OBJECT_START; + break; + } + if (isArrayStart(c, &array_count)) { + m_state = S_ARRAY_START; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isdigit(c)) { + m_state = S_NUMBER; + break; + } + if (c == '-') { + if (i + 1 == len) { // End of buffer case + m_state = S_NUMBER; + break; + } + if (i + 1 < len && isdigit(buf[i + 1])) { + m_state = S_NUMBER; + i++; + break; + } + m_state = S_ERROR; + break; + } + if (isspace(c)) { + break; + } + if (c == '\"') { + m_state = S_STRING_START; + break; + } + status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i); + if (status >= 0) { + m_state = S_STRING_START; + encoded = true; + i += status - 1; + break; + } else { + m_state = S_ERROR; + } + status = isBoolean(buf + i, len - i); + if (status >= 0) { + m_state = S_BOOLEAN; + i += status - 1; + break; + } + m_state = S_ERROR; + break; // S_COLON + + case S_BOOLEAN: + if (isspace(c)) { + break; + } + if (c == ',') { + m_state = S_COMMA; + break; + } + if (isArrayEnd(c, &array_count)) { + m_state = S_ARRAY_END; + break; + } + if (isObjectEnd(c, &object_count)) { + m_state = S_OBJECT_END; + break; + } + m_state = S_ERROR; + break; // S_BOOLEAN + + case S_ERROR: break; + case S_END: break; + } + if (m_state == S_ERROR) { + return false; + } + i++; + } + + if (m_state != S_ERROR && array_count >= 0 && object_count >= 0) + return true; + return false; +} + +} // namespace Util +} // namespace Waap diff --git a/components/utils/CMakeLists.txt b/components/utils/CMakeLists.txt index a71b851..8bd9647 100644 --- a/components/utils/CMakeLists.txt +++ b/components/utils/CMakeLists.txt @@ -1,4 +1,4 @@ -#add_subdirectory(geo_location) +add_subdirectory(geo_location) add_subdirectory(http_transaction_data) add_subdirectory(ip_utilities) add_subdirectory(keywords) diff --git a/core/agent_core_utilities/agent_core_utilities.cc b/core/agent_core_utilities/agent_core_utilities.cc index 81f9f27..2483f8a 100644 --- a/core/agent_core_utilities/agent_core_utilities.cc +++ b/core/agent_core_utilities/agent_core_utilities.cc @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -244,6 +245,22 @@ deleteDirectory(const string &path, bool delete_content) return res; } +bool +touchFile(const string &path) +{ + dbgFlow(D_INFRA_UTILS) << "Trying to touch file, path: " << path; + + ofstream t_file(path); + if (!t_file.is_open()) { + dbgDebug(D_INFRA_UTILS) << "Failed to touch file. Path: " << path; + return false; + } + + t_file.close(); + dbgTrace(D_INFRA_UTILS) << "Successfully touched file, path: " << path; + return true; +} + string convertToHumanReadable(uint64_t size_in_bytes) { diff --git a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h index 0ab65ee..0044339 100644 --- a/core/include/services_sdk/interfaces/i_intelligence_is_v2.h +++ b/core/include/services_sdk/interfaces/i_intelligence_is_v2.h @@ -17,17 +17,16 @@ #include #include -#include "maybe_res.h" -#include "i_messaging.h" -#include "i_time_get.h" -#include "i_mainloop.h" +#include "intelligence_is_v2/asset_reply.h" +#include "intelligence_is_v2/intelligence_response.h" #include "intelligence_is_v2/intelligence_types_v2.h" -#include "intelligence_is_v2/intelligence_query_v2.h" -#include "config.h" +#include "intelligence_is_v2/query_request_v2.h" +#include "maybe_res.h" namespace Intelligence { class Invalidation; +class Response; } // namespace Intelligence @@ -40,315 +39,27 @@ public: const std::function &callback ) = 0; virtual void unregisterInvalidation(uint id) = 0; + virtual Maybe + getResponse( + const std::vector &query_requests, + bool is_pretty, + bool is_bulk + ) const = 0; + + virtual Maybe getResponse(const QueryRequest &query_request, bool is_pretty) const = 0; template Maybe>> - queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true) - { - uint assets_limit = query_request.getAssetsLimit(); - static const uint upper_assets_limit = 50; - if (assets_limit == 0 || assets_limit > upper_assets_limit) { - return genError("Assets limit must be in the range of [1, " + std::to_string(upper_assets_limit) + "]"); - } - - static const uint upper_confidence_limit = 1000; - bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit); - if (!min_conf_res) { - return genError( - "Minimum confidence value must be in the range of [1, " + std::to_string(upper_confidence_limit) + "]" - ); - } - - if (query_request.isPagingActivated() && query_request.isPagingFinished()) { - return genError("Paging is activated and already finished. No need for more queries."); - } - - IntelligenceQuery intelligence_query(query_request, is_pretty); - static const std::string query_uri = "/api/v2/intelligence/assets/query"; - - bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit); - if (!res) { - dbgTrace(D_INTELLIGENCE) << "Could not message fog, trying to get offline intelligence."; - Maybe offline_res = getOfflineInfoString(query_request.getQuery()); - if (!offline_res.ok()) { - dbgDebug(D_INTELLIGENCE) << "Offline intelligence error: " << offline_res.getErr(); - return genError("Could not query intelligence"); - } - if (!intelligence_query.loadJson(offline_res.unpack())) { - dbgWarning(D_INTELLIGENCE) << "Offline intelligence error: invalid JSON for requested asset"; - return genError("Could not query intelligence"); - } - } - - if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) { - return genError("Query intelligence response with InProgress status"); - } - return intelligence_query.getData(); - } + queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true); template Maybe>>>> - queryIntelligence(std::vector &query_requests, bool is_pretty = true) - { - static const uint upper_assets_limit = 50; - static const uint upper_confidence_limit = 1000; - for (QueryRequest &query_request : query_requests) { - uint assets_limit = query_request.getAssetsLimit(); - if (assets_limit == 0 || assets_limit > upper_assets_limit) { - dbgTrace(D_INTELLIGENCE) - << "Assets limit for request is " - << upper_assets_limit - << ", requests assets: " - << assets_limit; - return genError("Assets limit valid range is of [1, " + std::to_string(upper_assets_limit) + "]"); - } + queryIntelligence(std::vector &query_requests, bool is_pretty = true); - bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit); - if (!min_conf_res) { - dbgTrace(D_INTELLIGENCE) << "Illegal confidence value"; - return genError( - "Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]" - ); - } - } - IntelligenceQuery intelligence_query(query_requests, is_pretty); - static const std::string query_uri = "/api/v2/intelligence/assets/queries"; - - dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items"; - bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, upper_assets_limit); - if (!res) { - dbgTrace(D_INTELLIGENCE) << "Could not message fog, bulk request failed."; - return genError("Could not query intelligence"); - } - - return intelligence_query.getBulkData(); - } - -private: - template - bool - sendMessage( - IntelligenceQuery &intelligence_query, - const std::string &query_uri, - I_Messaging *i_message, - Flags conn_flags, - const std::string &ip, - uint server_port - ) { - if (ip == "" && server_port == 0) { - auto req_status = i_message->sendSyncMessage( - HTTPMethod::POST, - query_uri, - intelligence_query, - MessageCategory::INTELLIGENCE - ); - return req_status.ok(); - } - - dbgTrace(D_INTELLIGENCE) - << "Sending intelligence request with IP: " - << ip - << " port: " - << server_port - << " query_uri: " - << query_uri; - MessageMetadata req_md(ip, server_port, conn_flags); - auto req_status = i_message->sendSyncMessage( - HTTPMethod::POST, - query_uri, - intelligence_query, - MessageCategory::INTELLIGENCE, - req_md - ); - return req_status.ok(); - } - - template - bool - sendQueryMessage( - IntelligenceQuery &intelligence_query, - const std::string &query_uri, - I_Messaging *i_message, - Flags conn_flags, - const std::string &ip = "", - uint server_port = 0 - ) { - auto i_timer = getTimer(); - auto i_mainloop = getMainloop(); - - uint request_overall_timeout_conf = getConfigurationWithDefault( - 20, - "intelligence", - "request overall timeout" - ); - - uint request_lap_timeout_conf = getConfigurationWithDefault( - 5, - "intelligence", - "request lap timeout" - ); - - std::chrono::seconds request_overall_timeout = std::chrono::seconds(request_overall_timeout_conf); - std::chrono::seconds request_lap_timeout = std::chrono::seconds(request_lap_timeout_conf); - - std::chrono::microseconds send_request_start_time = i_timer->getMonotonicTime(); - std::chrono::microseconds last_lap_time = i_timer->getMonotonicTime(); - std::chrono::seconds seconds_since_start = std::chrono::seconds(0); - std::chrono::seconds seconds_since_last_lap = std::chrono::seconds(0); - - bool res = true; - while (res && - intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS && - seconds_since_start < request_overall_timeout && - seconds_since_last_lap < request_lap_timeout - ) { - res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port); - - if (res && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) { - i_mainloop->yield(true); - } - - seconds_since_start = std::chrono::duration_cast( - i_timer->getMonotonicTime() - send_request_start_time - ); - - seconds_since_last_lap = std::chrono::duration_cast( - i_timer->getMonotonicTime() - last_lap_time - ); - last_lap_time = i_timer->getMonotonicTime(); - } - - return res; - } - - template - bool - sendPagingQueryMessage( - IntelligenceQuery &intelligence_query, - const std::string &query_uri, - int assets_limit, - I_Messaging *i_message, - Flags conn_flags, - const std::string &ip = "", - uint server_port = 0 - ) { - bool res= true; - - res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port); - - if (intelligence_query.getResponseStatus() == ResponseStatus::DONE && - intelligence_query.getResponseAssetCollectionsSize() < assets_limit - ) { - intelligence_query.setRequestCursor(Intelligence_IS_V2::CursorState::DONE, ""); - } else { - intelligence_query.setRequestCursor( - Intelligence_IS_V2::CursorState::IN_PROGRESS, - intelligence_query.getResponseCursorVal() - ); - } - - return res; - } - -// LCOV_EXCL_START Reason: one templated instance is tested in intelligence ut. the rest are tested in system tests - template - bool - sendQueryObjectToLocalServer( - IntelligenceQuery &intelligence_query, - const std::string &query_uri, - const std::string &ip, - bool is_primary_port, - int assets_limit, - I_Messaging *i_message, - Flags conn_flags - ) { - static const std::string primary_port_setting = "local intelligence server primary port"; - static const std::string secondary_port_setting = "local intelligence server secondary port"; - auto server_port = getSetting( - "intelligence", - is_primary_port ? primary_port_setting : secondary_port_setting - ); - - if (!server_port.ok()) return false; - - conn_flags.reset(); - - if (intelligence_query.getPagingStatus().ok()) { - return sendPagingQueryMessage( - intelligence_query, - query_uri, - assets_limit, - i_message, - conn_flags, - ip, - *server_port - ); - } - - return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags, ip, *server_port); - } -// LCOV_EXCL_STOP - - template - bool - sendQueryObject(IntelligenceQuery &intelligence_query, const std::string &query_uri, int assets_limit) - { - auto i_message = getMessaging(); - Flags conn_flags; - - bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ? - std::string(std::getenv("CROWDSEC_ENABLED")) == "true" : - false; - - crowdsec_enabled = getProfileAgentSettingWithDefault( - crowdsec_enabled, - "layer7AccessControl.crowdsec.enabled" - ); - - bool use_local_intelligence = getProfileAgentSettingWithDefault( - false, - "agent.config.useLocalIntelligence" - ); - - auto server_ip = getSetting("intelligence", "local intelligence server ip"); - if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) { - if (sendQueryObjectToLocalServer( - intelligence_query, - query_uri, - *server_ip, - true, - assets_limit, - i_message, - conn_flags - ) - ) { - return true; - } - if (sendQueryObjectToLocalServer( - intelligence_query, - query_uri, - *server_ip, - false, - assets_limit, - i_message, - conn_flags - ) - ) { - return true; - }; - } - - if (intelligence_query.getPagingStatus().ok()) { - return sendPagingQueryMessage(intelligence_query, query_uri, assets_limit, i_message, conn_flags); - } - - return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags); - } - - virtual I_Messaging * getMessaging() const = 0; - virtual I_TimeGet * getTimer() const = 0; - virtual I_MainLoop * getMainloop() const = 0; - virtual Maybe getOfflineInfoString(const SerializableQueryFilter &query) const = 0; - virtual bool getIsOfflineOnly() const = 0; +protected: + virtual ~I_Intelligence_IS_V2() {} }; + +#include "intelligence_is_v2/intelligence_interface_impl.h" + #endif // __I_INTELLIGENCE_IS_V2_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h similarity index 61% rename from core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h rename to core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h index 9b95774..9d73631 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h @@ -11,18 +11,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __QUERY_RESPONSE_V2_IMPL_H_ -#define __QUERY_RESPONSE_V2_IMPL_H_ +#ifndef __ASSET_REPLAY_IMPL_H__ +#define __ASSET_REPLAY_IMPL_H__ -#ifndef __QUERY_RESPONSE_V2_H__ -#error intelligence_query_response_v2_impl.h should not be included directly! -#endif // __QUERY_RESPONSE_V2_H__ +#ifndef __ASSET_REPLY_H__ +#error asset_replay_impl.h should not be included directly! +#endif // __ASSET_REPLY_H__ -#include "debug.h" +#include "customized_cereal_multimap.h" #include "intelligence_types_v2.h" -USE_DEBUG_FLAG(D_INTELLIGENCE); - template void AssetReply::load(cereal::JSONInputArchive &ar) @@ -84,87 +82,72 @@ AssetReply::getData() const } template -void -IntelligenceQueryResponse::loadFromJson(cereal::JSONInputArchive &ar) +template +bool +AssetReply::matchValues(const Values &values) const { - std::string raw_data; - ar( - cereal::make_nvp("status", raw_data), - cereal::make_nvp("totalNumAssets", total_num_assets), - cereal::make_nvp("assetCollections", asset_collections) - ); - status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data); + for (const SerializableAssetSource &source : sources) { + if (source.template matchValues(values)) return true; + } + return false; +} - try { - ar(cereal::make_nvp("cursor", cursor)); - } catch(...) {} +template +UserSerializableReplyAttr +AssetReply::mergeReplyData() const +{ + UserSerializableReplyAttr reply_data; + for (const SerializableAssetSource &source : sources) { + UserSerializableReplyAttr data_by_source = source.mergeReplyData(); + reply_data.insert(data_by_source); + } + return reply_data; +} + +template +void +IntelligenceQueryResponseT::loadFromJson(const std::string &json_response) +{ + std::stringstream in; + in.str(json_response); + cereal::JSONInputArchive in_ar(in); + serialize(in_ar); } template template void -IntelligenceQueryResponse::serialize(Archive &ar) +IntelligenceQueryResponseT::serialize(Archive &ar) { - std::string raw_data; ar( - cereal::make_nvp("status", raw_data), - cereal::make_nvp("totalNumAssets", total_num_assets), cereal::make_nvp("assetCollections", asset_collections) ); - status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data); try { - ar(cereal::make_nvp("cursor", cursor)); + IntelligenceQueryResponse::serialize(ar); } catch(...) {} } -template -Intelligence_IS_V2::ResponseStatus -IntelligenceQueryResponse::getResponseStatus() const -{ - return status; -} template uint -IntelligenceQueryResponse::getAmountOfAssets() const -{ - return total_num_assets; -} - -template -const std::string & -IntelligenceQueryResponse::getCursor() const -{ - return cursor; -} - -template -int -IntelligenceQueryResponse::getAssetCollectionsSize() const +IntelligenceQueryResponseT::getAssetCollectionsSize() const { return asset_collections.size(); } template const std::vector> & -IntelligenceQueryResponse::getData() const +IntelligenceQueryResponseT::getData() const { return asset_collections; } template bool -IntelligenceQueryResponse::isValidInBulk() const +IntelligenceQueryResponseT::isLast(uint asset_limit) { - return !partial_fail_in_bulk; + return getResponseStatus() == ResponseStatus::DONE && getAssetCollectionsSize() < asset_limit; } -template -void -IntelligenceQueryResponse::setFailInBulk() -{ - partial_fail_in_bulk = true; -} - -#endif // __QUERY_RESPONSE_V2_IMPL_H_ +#endif // __ASSET_REPLAY_IMPL_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h similarity index 71% rename from core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h rename to core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h index c78e7a1..49db3ff 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h @@ -11,25 +11,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __QUERY_RESPONSE_V2_H__ -#define __QUERY_RESPONSE_V2_H__ +#ifndef __ASSET_REPLY_H__ +#define __ASSET_REPLY_H__ -#include -#include #include +#include +#include -#include "cereal/archives/json.hpp" -#include "cereal/types/vector.hpp" -#include "cereal/types/map.hpp" - -#include "debug.h" -#include "maybe_res.h" -#include "customized_cereal_map.h" -#include "customized_cereal_multimap.h" +#include "asset_source.h" +#include "query_request_v2.h" #include "intelligence_types_v2.h" -#include "asset_source_v2.h" - -USE_DEBUG_FLAG(D_INTELLIGENCE); +#include "maybe_res.h" template class AssetReply @@ -55,26 +47,10 @@ public: const std::string & getAssetOrder() const { return asset_order; } const std::string & getAssetKind() const { return asset_kind; } - UserSerializableReplyAttr - mergeReplyData() const - { - UserSerializableReplyAttr reply_data; - for (const SerializableAssetSource &source : sources) { - UserSerializableReplyAttr data_by_source = source.mergeReplyData(); - reply_data.insert(data_by_source); - } - return reply_data; - } + UserSerializableReplyAttr mergeReplyData() const; template - bool - matchValues(const Values &values) const - { - for (const SerializableAssetSource &source : sources) { - if (source.template matchValues(values)) return true; - } - return false; - } + bool matchValues(const Values &values) const; private: uint asset_schema_version = 0; @@ -94,33 +70,48 @@ private: std::vector> sources; }; -template class IntelligenceQueryResponse { public: IntelligenceQueryResponse() {} - void loadFromJson(cereal::JSONInputArchive &ar); + void loadFromJson(const std::string &json_response); template void serialize(Archive &ar); - Intelligence_IS_V2::ResponseStatus getResponseStatus() const; - uint getAmountOfAssets() const; - const std::string & getCursor() const; - int getAssetCollectionsSize() const; - const std::vector> & getData() const; - bool isValidInBulk() const; - void setFailInBulk(); + Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; } + const std::string & getCursor() const { return cursor; } + uint getAmountOfAssets() const { return total_num_assets; } + bool isValidInBulk() const { return !partial_fail_in_bulk; } + void setFailInBulk() { partial_fail_in_bulk = true; } private: Intelligence_IS_V2::ResponseStatus status = Intelligence_IS_V2::ResponseStatus::IN_PROGRESS; uint total_num_assets = 0; std::string cursor = ""; - std::vector> asset_collections; bool partial_fail_in_bulk = false; }; -#include "query_response_v2_impl.h" +template +class IntelligenceQueryResponseT : public IntelligenceQueryResponse +{ +public: + void loadFromJson(const std::string &json_response); -#endif // __QUERY_RESPONSE_V2_H__ + template + void serialize(Archive &ar); + + uint getAssetCollectionsSize() const; + + bool isLast(uint asset_limit); + + const std::vector> & getData() const; + +private: + std::vector> asset_collections; +}; + +#include "asset_replay_impl.h" + +#endif // __ASSET_REPLY_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h similarity index 70% rename from core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h rename to core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h index 4d5edfd..70a5e11 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h @@ -11,14 +11,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef __ASSET_SOURCE_V2_H__ -#define __ASSET_SOURCE_V2_H__ +#ifndef __ASSET_SOURCE_H__ +#define __ASSET_SOURCE_H__ + +#include +#include +#include -#include "debug.h" -#include "intelligence_types_v2.h" #include "cereal/archives/json.hpp" -#include "cereal/types/vector.hpp" -#include "customized_cereal_map.h" +#include "intelligence_types_v2.h" template class SerializableAssetSource @@ -26,7 +27,26 @@ class SerializableAssetSource public: SerializableAssetSource() {} - void load(cereal::JSONInputArchive &ar); + void load(cereal::JSONInputArchive &ar) + { + uint raw_seconds; + ar( + cereal::make_nvp("tenantId", tenant_id), + cereal::make_nvp("sourceId", source_id), + cereal::make_nvp("assetId", asset_id), + cereal::make_nvp("ttl", raw_seconds), + cereal::make_nvp("expirationTime", expiration_time), + cereal::make_nvp("confidence", confidence) + ); + ttl = std::chrono::seconds(raw_seconds); + + UserSerializableReplyAttr raw_attribute; + try { + ar(cereal::make_nvp("attributes", raw_attribute)); + attributes.clear(); + attributes.push_back(raw_attribute); + } catch(const std::exception &e) {} + } const std::string & getTenantId() const { return tenant_id; } const std::string & getSourceId() const { return source_id; } @@ -66,6 +86,4 @@ private: std::vector attributes; }; -#include "asset_source_v2_impl.h" - -#endif //__ASSET_SOURCE_V2_H__ +#endif // __ASSET_SOURCE_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h deleted file mode 100644 index cb1d616..0000000 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __ASSET_SOURCE_V2_IMPL_H__ -#define __ASSET_SOURCE_V2_IMPL_H__ - -#ifndef __ASSET_SOURCE_V2_H__ -#error intelligence_query_impl_8_0.h should not be included directly! -#endif //__ASSET_V2_SOURCE_H__ - -USE_DEBUG_FLAG(D_INTELLIGENCE); - -template -void -SerializableAssetSource::load(cereal::JSONInputArchive &ar) -{ - uint raw_seconds; - ar( - cereal::make_nvp("tenantId", tenant_id), - cereal::make_nvp("sourceId", source_id), - cereal::make_nvp("assetId", asset_id), - cereal::make_nvp("ttl", raw_seconds), - cereal::make_nvp("expirationTime", expiration_time), - cereal::make_nvp("confidence", confidence) - ); - ttl = std::chrono::seconds(raw_seconds); - - UserSerializableReplyAttr raw_attribute; - try { - ar(cereal::make_nvp("attributes", raw_attribute)); - attributes.clear(); - attributes.push_back(raw_attribute); - } catch(const std::exception &e) {} - -} - -#endif //__ASSET_SOURCE_V2_IMPL_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h index 3e2cfc7..17dd627 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h @@ -15,12 +15,12 @@ #define __BULK_QUERY_RESPONSE_V2_H__ #include -#include #include +#include +#include "asset_reply.h" #include "cereal/archives/json.hpp" #include "cereal/types/vector.hpp" - #include "debug.h" #include "intelligence_types_v2.h" @@ -29,15 +29,7 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); class BulkResponseError { public: - void - serialize(cereal::JSONInputArchive &ar) - { - ar( - cereal::make_nvp("index", index), - cereal::make_nvp("statusCode", status_code), - cereal::make_nvp("message", message) - ); - } + void serialize(cereal::JSONInputArchive &ar); unsigned int getIndex() const { return index; } int getStatusCode() const { return status_code; } @@ -49,46 +41,69 @@ private: std::string message; }; -template class ValidBulkQueryResponse { +public: + void serialize(cereal::JSONInputArchive &ar); + + unsigned int getIndex() const { return index; } + const IntelligenceQueryResponse & getResponse() const { return response; } + +private: + unsigned int index; + IntelligenceQueryResponse response; +}; + +template +class ValidBulkQueryResponseT : public ValidBulkQueryResponse +{ public: void serialize(cereal::JSONInputArchive &ar) { + try { + ValidBulkQueryResponse::serialize(ar); + } catch (...) {} ar( - cereal::make_nvp("index", index), cereal::make_nvp("response", response) ); } - unsigned int getIndex() const { return index; } - const IntelligenceQueryResponse & getResponse() const { return response; } + const IntelligenceQueryResponseT & getResponse() const { return response; } private: - unsigned int index; - IntelligenceQueryResponse response; + IntelligenceQueryResponseT response; +}; + +class IntelligenceQueryBulkResponse +{ +public: + void serialize(cereal::JSONInputArchive &ar); + + const std::vector & getValid() { return valid_responses; } + const std::vector & getErrors() { return errors; } +private: + std::vector valid_responses; + std::vector errors; }; template -class IntelligenceQueryBulkResponse +class IntelligenceQueryBulkResponseT : public IntelligenceQueryBulkResponse { public: void serialize(cereal::JSONInputArchive &ar) { - ar(cereal::make_nvp("queriesResponse", valid_responses)); try { - ar(cereal::make_nvp("errors", errors)); + IntelligenceQueryBulkResponse::serialize(ar); } catch(...) {} + ar(cereal::make_nvp("queriesResponse", valid_responses)); } - const std::vector> & getValid() { return valid_responses; } - const std::vector & getErrors() { return errors; } + const std::vector> & getValid() { return valid_responses; } private: - std::vector> valid_responses; - std::vector errors; + std::vector> valid_responses; }; #endif // __BULK_QUERY_RESPONSE_V2_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h new file mode 100644 index 0000000..9880de8 --- /dev/null +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h @@ -0,0 +1,50 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __INTELLIGENCE_INTERFACE_IMPL_H__ +#define __INTELLIGENCE_INTERFACE_IMPL_H__ + +#ifndef __I_INTELLIGENCE_IS_V2_H__ +#error intelligence_interface_impl.h should not be included directly! +#endif // __I_INTELLIGENCE_IS_V2_H__ + +template +Maybe>> +I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore_in_progress, bool is_pretty) +{ + auto response = getResponse(query_request, is_pretty); + + if (!response.ok()) return response.passErr(); + auto serializable_response = response->getSerializableResponse(); + if (!query_request.isPagingActivated()) return serializable_response.getData(); + if (serializable_response.isLast(query_request.getAssetsLimit())) { + query_request.setCursor(Intelligence_IS_V2::CursorState::DONE, ""); + } else { + query_request.setCursor(Intelligence_IS_V2::CursorState::IN_PROGRESS, response->getCursor()); + if (ignore_in_progress) return genError("Query intelligence response with InProgress status"); + } + + return serializable_response.getData(); +} + +template +Maybe>>>> +I_Intelligence_IS_V2::queryIntelligence(std::vector &query_requests, bool is_pretty) +{ + auto res = getResponse(query_requests, is_pretty, true); + if (!res.ok()) return res.passErr(); + + return res->getBulkData(); +} + +#endif // __INTELLIGENCE_INTERFACE_IMPL_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h deleted file mode 100644 index 66e7560..0000000 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __INTELLIGENCE_QUERY_V2_H__ -#define __INTELLIGENCE_QUERY_V2_H__ - -#include - -#include "cereal/archives/json.hpp" -#include "intelligence_types_v2.h" -#include "query_request_v2.h" -#include "query_response_v2.h" -#include "bulk_query_response_v2.h" -#include "rest.h" - -template -class IntelligenceQuery -{ -public: - IntelligenceQuery(QueryRequest &filter, bool is_pretty) - : - request(filter), - response(), - responses(), - is_bulk(false), - is_pretty(is_pretty) - {} - - IntelligenceQuery(std::vector &filters, bool is_pretty) - : - requests(filters), - response(), - responses(), - is_bulk(true), - is_pretty(is_pretty) - {} - - Maybe genJson() const; - bool loadJson(const std::string &json); - - void load(cereal::JSONInputArchive &ar); - void save(cereal::JSONOutputArchive &ar) const; - - std::vector> getData(); - std::vector>>> getBulkData(); - ResponseStatus getResponseStatus(); - int getResponseAssetCollectionsSize() const { return response.getAssetCollectionsSize(); } - const std::string & getResponseCursorVal() const { return response.getCursor(); } - - void activatePaging(); - Maybe getPagingStatus(); - void setRequestCursor(CursorState state, const std::string &value); - -private: - static QueryRequest dummy_query_request; - static std::vector dummy_query_requests; - std::vector &requests = dummy_query_requests; - QueryRequest &request = dummy_query_request; - IntelligenceQueryResponse response; - std::vector> responses; - bool is_bulk; - bool is_pretty; -}; - -#include "intelligence_query_v2_impl.h" - -#endif // __INTELLIGENCE_QUERY_V2_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h deleted file mode 100644 index eebbafd..0000000 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef __INTELLIGENCE_QUERY_V2_IMPL_H_ -#define __INTELLIGENCE_QUERY_V2_IMPL_H_ - -#ifndef __INTELLIGENCE_QUERY_V2_H__ -#error intelligence_query_impl_v2.h should not be included directly! -#endif // __INTELLIGENCE_QUERY_V2_H__ - -#include -#include "json_stream.h" - -USE_DEBUG_FLAG(D_INTELLIGENCE); - -template -QueryRequest IntelligenceQuery::dummy_query_request = QueryRequest(); - -template -std::vector IntelligenceQuery::dummy_query_requests = - std::vector(); - -template -Maybe -IntelligenceQuery::genJson() const -{ - { - std::stringstream str_stream; - JsonStream json_stream(&str_stream, is_pretty); - { - cereal::JSONOutputArchive out_ar(json_stream); - if (is_bulk) { - std::vector bulk_requests; - int index = 0; - for (QueryRequest &request : requests) { - bulk_requests.push_back(BulkQueryRequest(request, index++)); - } - out_ar(cereal::make_nvp("queries", bulk_requests)); - } else { - request.saveToJson(out_ar); - } - } - - return str_stream.str(); - } -} - -template -bool -IntelligenceQuery::loadJson(const std::string &json) -{ - try { - std::stringstream in; - in.str(json); - try { - cereal::JSONInputArchive in_ar(in); - load(in_ar); - } catch (const Intelligence_IS_V2::IntelligenceException &e) { - dbgWarning(D_INTELLIGENCE) << "Failed to load query response. Error: " << e.what(); - return false; - } - return true; - } catch (const std::exception &e) { - return false; - } -} - -template -void -IntelligenceQuery::load(cereal::JSONInputArchive &ar) -{ - if (is_bulk) { - IntelligenceQueryBulkResponse bulk_response; - bulk_response.serialize(ar); - unsigned int error_idx = 0; - unsigned int valid_idx = 0; - const auto &valid_response = bulk_response.getValid(); - const auto &errors = bulk_response.getErrors(); - responses.clear(); - responses.reserve(requests.size()); - dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items"; - for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) { - if (valid_response[valid_idx].getIndex() == query_idx) { - responses.push_back(valid_response[valid_idx].getResponse()); - dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid"; - valid_idx++; - } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) { - responses.emplace_back(); - responses[query_idx].setFailInBulk(); - dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid"; - error_idx++; - } else { - dbgWarning(D_INTELLIGENCE) - << "Query index was not found neither in valid nor error responses, assuming error"; - responses[query_idx].setFailInBulk(); - } - } - } else { - response.loadFromJson(ar); - } -} - -template -void -IntelligenceQuery::save(cereal::JSONOutputArchive &ar) const -{ - if (!is_bulk) { - request.saveToJson(ar); - } else { - ar(cereal::make_nvp("queries", requests)); - } -} - -template -std::vector> -IntelligenceQuery::getData() -{ - return response.getData(); -} - -template -std::vector>>> -IntelligenceQuery::getBulkData() -{ - std::vector>>> bulk_data; - bulk_data.reserve(responses.size()); - int index = 0; - for (const auto &res: responses) { - if (!res.isValidInBulk()) { - dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed"; - bulk_data.push_back(genError("Received error for request in bulk")); - index++; - } else { - dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response"; - bulk_data.push_back(res.getData()); - index++; - } - } - return bulk_data; -} - -template -void -IntelligenceQuery::activatePaging() -{ - request.setCursor(Intelligence_IS_V2::CursorState::START, "start"); -} - -template -Maybe -IntelligenceQuery::getPagingStatus() -{ - if (is_bulk) return genError("Paging not activated in bulk mode"); - if (!request.isPagingActivated()) return genError("Paging not activated"); - return request.getCursorState(); -} - -template -ResponseStatus -IntelligenceQuery::getResponseStatus() -{ - if (!is_bulk) return response.getResponseStatus(); - - if (responses.size() == 0) return ResponseStatus::IN_PROGRESS; - for (const auto &response_itr : responses) { - if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) { - return ResponseStatus::IN_PROGRESS; - } - } - - return ResponseStatus::DONE; -} - -template -void -IntelligenceQuery::setRequestCursor(CursorState state, const std::string &value) -{ - request.setCursor(state, value); -} - -#endif //__INTELLIGENCE_QUERY_V2_IMPL_H_ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h new file mode 100644 index 0000000..f1dea48 --- /dev/null +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h @@ -0,0 +1,111 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __INTELLIGENCE_RESPONSE_H__ +#define __INTELLIGENCE_RESPONSE_H__ + +#include +#include +#include + +#include +#include "asset_reply.h" +#include "bulk_query_response_v2.h" + +USE_DEBUG_FLAG(D_INTELLIGENCE); + +namespace Intelligence +{ + +class Response +{ +public: + Response() = default; + Response(const std::string &json_body, size_t size, bool is_bulk) + : + json_response(json_body), size(size), is_bulk(is_bulk) + {} + + Maybe load(); + Intelligence_IS_V2::ResponseStatus getResponseStatus() const; + const std::string getCursor() const { return single_response.getCursor(); } + void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; } + template + IntelligenceQueryResponseT getSerializableResponse() const + { + IntelligenceQueryResponseT response; + response.loadFromJson(json_response); + return response; + } + + template + std::vector>>> + getBulkData() const + { + std::stringstream in; + in.str(json_response); + cereal::JSONInputArchive in_ar(in); + + IntelligenceQueryBulkResponseT bulk_response; + bulk_response.serialize(in_ar); + unsigned int error_idx = 0; + unsigned int valid_idx = 0; + const auto &valid_response = bulk_response.getValid(); + const auto &errors = bulk_response.getErrors(); + std::vector> serializable_responses; + serializable_responses.reserve(size); + dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items"; + for (unsigned int query_idx = 0; query_idx < size; query_idx++) { + if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) { + serializable_responses.push_back(valid_response[valid_idx].getResponse()); + dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid"; + valid_idx++; + } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) { + serializable_responses.emplace_back(); + serializable_responses[query_idx].setFailInBulk(); + dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid"; + error_idx++; + } else { + dbgWarning(D_INTELLIGENCE) + << "Query index was not found neither in valid nor error responses, assuming error"; + serializable_responses[query_idx].setFailInBulk(); + } + } + std::vector>>> bulk_data; + bulk_data.reserve(serializable_responses.size()); + int index = 0; + for (const auto &res: serializable_responses) { + if (!res.isValidInBulk()) { + dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed"; + bulk_data.push_back(genError("Received error for request in bulk")); + index++; + } else { + dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response"; + bulk_data.push_back(res.getData()); + index++; + } + } + return bulk_data; + } + +private: + std::string json_response; + std::vector responses; + IntelligenceQueryResponse single_response; + size_t size = 0; + bool is_bulk = false; +}; + +} + +#endif // __INTELLIGENCE_RESPONSE_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h index 15fdceb..f8ca1d5 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h @@ -59,7 +59,6 @@ public: Condition getConditionType() const { return condition_type; } const std::string & getKey() const { return key; } - const ValueVariant & getValue() const { return value; } private: Condition condition_type = Condition::EQUALS; @@ -85,8 +84,6 @@ public: const std::vector & getConditionOperands() const { return condition_operands; } const std::vector & getQueriesOperands() const { return queries_operands; } - Maybe getConditionValueByKey(const std::string &key) const; - bool empty() const { return condition_operands.empty() && queries_operands.empty(); } SerializableQueryFilter operator &&(const SerializableQueryFilter &other_query); diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h index 337b384..ebcab54 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h @@ -97,11 +97,11 @@ public: void setObjectType(const ObjectType &obj_type); void setAssetsLimit(uint _assets_limit); - bool checkMinConfidence(uint upper_confidence_limit); + bool checkMinConfidence(uint upper_confidence_limit) const; void activatePaging(); bool isPagingActivated(); - Maybe getCursorState(); + Maybe getCursorState() const; bool isPagingFinished(); void setCursor(CursorState state, const std::string &value); bool empty() const { return query.empty(); } @@ -124,21 +124,4 @@ private: Maybe convertObjectTypeToString() const; }; -class BulkQueryRequest -{ -public: - BulkQueryRequest() {} - - BulkQueryRequest(QueryRequest &request, int index); - - void saveToJson(cereal::JSONOutputArchive &ar) const; - void save(cereal::JSONOutputArchive &ar) const; - - QueryRequest getQueryRequest() const; - -private: - QueryRequest request; - int index; -}; - #endif // __QUERY_REQUEST_V2_H__ diff --git a/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h b/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h index 1a18829..47aee5f 100644 --- a/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h +++ b/core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h @@ -51,7 +51,7 @@ public: uint getSize() const { return requested_attributes.size(); } bool isRequestedAttributesMapEmpty() const { return requested_attributes.empty(); } - bool checkMinConfidence(uint upper_confidence_limit); + bool checkMinConfidence(uint upper_confidence_limit) const; private: std::unordered_map requested_attributes; diff --git a/core/include/services_sdk/interfaces/mock/mock_intelligence.h b/core/include/services_sdk/interfaces/mock/mock_intelligence.h new file mode 100644 index 0000000..4a19b4f --- /dev/null +++ b/core/include/services_sdk/interfaces/mock/mock_intelligence.h @@ -0,0 +1,38 @@ +#ifndef __MOCK_INTELLIGENCE_H__ +#define __MOCK_INTELLIGENCE_H__ + +#include "i_intelligence_is_v2.h" +#include "cptest.h" + +std::ostream & +operator<<(std::ostream &os, const Intelligence::Response &) +{ + return os; +} + +std::ostream & +operator<<(std::ostream &os, const Intelligence::Invalidation &) +{ + return os; +} + +class MockIntelligence : public Singleton::Provide::From> +{ +public: + using InvalidationCb = std::function; + using Invalidation = Intelligence::Invalidation; + using Response = Intelligence::Response; + + MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation)); + MOCK_METHOD2(registerInvalidation, Maybe(const Invalidation &invalidation, const InvalidationCb &callback)); + MOCK_METHOD1(unregisterInvalidation, void(uint id)); + MOCK_CONST_METHOD3( + getResponse, + Maybe(const std::vector &query_requests, bool is_pretty, bool is_bulk) + ); + MOCK_CONST_METHOD2(getResponse, Maybe(const QueryRequest &query_request, bool is_pretty)); + MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void)); + MOCK_CONST_METHOD1(getOfflineInfoString, Maybe(const SerializableQueryFilter &query)); +}; + +#endif // __MOCK_INTELLIGENCE_H__ diff --git a/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h b/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h index 37c2c4b..194ddfd 100644 --- a/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h +++ b/core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h @@ -14,6 +14,9 @@ #ifndef __DATA_STRING_V2_H__ #define __DATA_STRING_V2_H__ +#include +#include "cereal/archives/json.hpp" + class DataString { public: diff --git a/core/include/services_sdk/utilities/agent_core_utilities.h b/core/include/services_sdk/utilities/agent_core_utilities.h index 5cdd171..34ef904 100644 --- a/core/include/services_sdk/utilities/agent_core_utilities.h +++ b/core/include/services_sdk/utilities/agent_core_utilities.h @@ -32,6 +32,7 @@ Maybe> getDirectoryFiles(const std::string &path); bool makeDir(const std::string &path, mode_t permission = S_IRWXU); bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU); bool deleteDirectory(const std::string &path, bool delete_content = false); +bool touchFile(const std::string &path); bool copyFile( diff --git a/core/intelligence_is_v2/CMakeLists.txt b/core/intelligence_is_v2/CMakeLists.txt index 2fb0b8d..c05920d 100755 --- a/core/intelligence_is_v2/CMakeLists.txt +++ b/core/intelligence_is_v2/CMakeLists.txt @@ -1,7 +1,7 @@ include_directories(include) add_library( - intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc - intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc + intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc intelligence_server.cc intelligence_response.cc asset_replay.cc bulk_query_response_v2.cc + intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc intelligence_request.cc ) -add_subdirectory(intelligence_is_v2_ut) +#add_subdirectory(intelligence_is_v2_ut) diff --git a/core/intelligence_is_v2/asset_replay.cc b/core/intelligence_is_v2/asset_replay.cc new file mode 100644 index 0000000..20ef167 --- /dev/null +++ b/core/intelligence_is_v2/asset_replay.cc @@ -0,0 +1,38 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_is_v2/asset_reply.h" + +using namespace std; + +void +IntelligenceQueryResponse::loadFromJson(const std::string &json_response) +{ + std::stringstream in; + in.str(json_response); + cereal::JSONInputArchive in_ar(in); + serialize(in_ar); +} + +template +void +IntelligenceQueryResponse::serialize(Archive &ar) +{ + std::string raw_data; + ar(cereal::make_nvp("status", raw_data), cereal::make_nvp("totalNumAssets", total_num_assets)); + status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data); + + try { + ar(cereal::make_nvp("cursor", cursor)); + } catch (...) {} +} diff --git a/core/intelligence_is_v2/bulk_query_response_v2.cc b/core/intelligence_is_v2/bulk_query_response_v2.cc new file mode 100644 index 0000000..148c994 --- /dev/null +++ b/core/intelligence_is_v2/bulk_query_response_v2.cc @@ -0,0 +1,39 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_is_v2/bulk_query_response_v2.h" + +using namespace std; + +void +BulkResponseError::serialize(cereal::JSONInputArchive &ar) +{ + ar( + cereal::make_nvp("index", index), + cereal::make_nvp("statusCode", status_code), + cereal::make_nvp("message", message) + ); +} + +void +ValidBulkQueryResponse::serialize(cereal::JSONInputArchive &ar) +{ + ar(cereal::make_nvp("index", index), cereal::make_nvp("response", response)); +} + +void +IntelligenceQueryBulkResponse::serialize(cereal::JSONInputArchive &ar) +{ + ar(cereal::make_nvp("errors", errors)); + ar(cereal::make_nvp("queriesResponse", valid_responses)); +} diff --git a/core/intelligence_is_v2/include/intelligence_request.h b/core/intelligence_is_v2/include/intelligence_request.h new file mode 100644 index 0000000..cba0af3 --- /dev/null +++ b/core/intelligence_is_v2/include/intelligence_request.h @@ -0,0 +1,53 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __INTELLIGENCE_REQUEST_H__ +#define __INTELLIGENCE_REQUEST_H__ +#include "intelligence_is_v2/query_request_v2.h" + +#include +#include "maybe_res.h" + +namespace Intelligence { + +class IntelligenceRequest : ClientRest +{ +public: + IntelligenceRequest(const std::vector &queries, bool is_pretty, bool is_bulk) + : + queries(queries), is_pretty(is_pretty), is_bulk(is_bulk) + {} + + Maybe checkAssetsLimit() const; + Maybe checkMinConfidence() const; + bool isPagingAllowed() const; + bool isPagingActivated() const; + Maybe isPagingFinished() const; + Maybe getPagingStatus() const; + bool loadJson(const std::string &json); + Maybe genJson() const; + Maybe getResponseFromFog() const; + + size_t getSize() const { return queries.size(); } + bool isBulk() const { return is_bulk; } + +private: + const std::vector &queries; + bool is_pretty = true; + bool is_bulk = false; + Maybe response_from_fog = genError("Uninitialized"); +}; + +} + +#endif // __INTELLIGENCE_REQUEST_H__ diff --git a/core/intelligence_is_v2/include/intelligence_server.h b/core/intelligence_is_v2/include/intelligence_server.h new file mode 100644 index 0000000..54e84f7 --- /dev/null +++ b/core/intelligence_is_v2/include/intelligence_server.h @@ -0,0 +1,50 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __INTELLIGENCE_SERVER_H__ +#define __INTELLIGENCE_SERVER_H__ + +#include +#include "maybe_res.h" + +#include "i_messaging.h" +#include "intelligence_is_v2/intelligence_response.h" +#include "intelligence_request.h" + +namespace Intelligence { + +class Sender +{ +public: + Sender(IntelligenceRequest request); + Maybe sendIntelligenceRequest(); + +private: + Maybe sendQueryObjectToLocalServer(bool is_primary_port); + Maybe sendQueryMessage(); + Maybe sendMessage(); + Maybe createResponse(); + + IntelligenceRequest request; + Flags conn_flags; + bool is_local_intelligence; + Maybe server_ip = genError("No server ip set"); + Maybe server_port = genError("No port is set"); + I_Messaging * i_message = nullptr; + I_TimeGet * i_timer = nullptr; + I_MainLoop * i_mainloop = nullptr; +}; + +} + +#endif // __INTELLIGENCE_SERVER_H__ diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 609c752..3ccc394 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -17,9 +17,10 @@ #include "cache.h" #include "config.h" -#include "table.h" -#include "intelligence_is_v2/query_response_v2.h" #include "intelligence_invalidation.h" +#include "intelligence_is_v2/intelligence_response.h" +#include "intelligence_request.h" +#include "intelligence_server.h" using namespace std; using namespace chrono; @@ -221,50 +222,10 @@ class IntelligenceComponentV2::Impl Singleton::Provide::From { public: - class OfflineIntelligeceHandler - { - public: - void - init() - { - filesystem_prefix = getFilesystemPathConfig(); - dbgTrace(D_INTELLIGENCE) << "OfflineIntelligeceHandler init. file systen prefix: " << filesystem_prefix; - offline_intelligence_path = getConfigurationWithDefault( - filesystem_prefix + "/conf/offline/intelligence", - "intelligence", - "offline intelligence path" - ); - } - - Maybe - getValueByIdentifier(const string &identifier) const - { - string asset_file_path = offline_intelligence_path + "/" + identifier; - ifstream asset_info(asset_file_path); - if (!asset_info.is_open()) { - return genError("Could not open file: " + asset_file_path); - } - - stringstream info_txt; - info_txt << asset_info.rdbuf(); - asset_info.close(); - return info_txt.str(); - } - - private: - string filesystem_prefix = ""; - string offline_intelligence_path = ""; - }; void init() { - offline_mode_only = getConfigurationWithDefault(false, "intelligence", "offline intelligence only"); - registerConfigLoadCb([&]() { - offline_mode_only = getConfigurationWithDefault(false, "intelligence", "offline intelligence only"); - }); - offline_intelligence.init(); - message = Singleton::Consume::by(); timer = Singleton::Consume::by(); mainloop = Singleton::Consume::by(); @@ -283,7 +244,6 @@ public: bool sendInvalidation(const Invalidation &invalidation) const override { - if (offline_mode_only) return false; return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation); } @@ -301,48 +261,28 @@ public: invalidations.erase(id); } - I_Messaging * - getMessaging() const override + Maybe + getResponse(const vector &query_requests, bool is_pretty, bool is_bulk) const override { - return message != NULL ? message : Singleton::Consume::by(); - } - - I_TimeGet * - getTimer() const override - { - return timer != NULL ? timer : Singleton::Consume::by(); - } - - I_MainLoop * - getMainloop() const override - { - return mainloop != NULL ? mainloop : Singleton::Consume::by(); - } - - Maybe - getOfflineInfoString(const SerializableQueryFilter &query) const override - { - string ip_attr_key = "mainAttributes.ip"; - auto valueVariant = query.getConditionValueByKey(ip_attr_key); - - if (!valueVariant.ok()) { - return genError("could not find IP main attribute in the given query."); - } - const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack(); - if (const string* identifier_value = boost::get(&value)) { - if (*identifier_value == "") { - return genError("Could not find IP main attribute in the given query."); + IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk); + if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr(); + if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr(); + if (intelligence_req.isPagingActivated()) { + auto is_paging_finished = intelligence_req.isPagingFinished(); + if (is_paging_finished.ok() && *is_paging_finished) { + return genError("Paging is activated and already finished. No need for more queries."); } - return offline_intelligence.getValueByIdentifier(*identifier_value); } - - return genError("Value is not of type string."); + Sender intelligence_server(intelligence_req); + auto response = intelligence_server.sendIntelligenceRequest(); + return response; } - bool - getIsOfflineOnly() const override + Maybe + getResponse(const QueryRequest &query_request, bool is_pretty) const override { - return offline_mode_only; + vector queries = {query_request}; + return getResponse(queries, is_pretty, false); } private: @@ -362,7 +302,7 @@ private: bool sendLocalInvalidationImpl(const Invalidation &invalidation) const { - auto server = getSetting("intelligence", "local intelligence server ip"); + auto server = getSetting("intelligence", "local intelligence server ip"); if (!server.ok()) { dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured"; return false; @@ -435,8 +375,6 @@ private: bool sendRegistration(const Invalidation &invalidation) const { - if (offline_mode_only) return false; - InvalidationRegistration registration; registration.addInvalidation(invalidation); @@ -446,7 +384,7 @@ private: bool sendLocalRegistrationImpl(const InvalidationRegistration::RestCall ®istration) const { - auto server = getSetting("intelligence", "local intelligence server ip"); + auto server = getSetting("intelligence", "local intelligence server ip"); if (!server.ok()) { dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured"; return false; @@ -485,13 +423,11 @@ private: void sendReccurringInvalidationRegistration() const { - if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return; + if (!hasLocalIntelligence() || invalidations.empty()) return; sendLocalRegistrationImpl(invalidations.getRegistration()); } - OfflineIntelligeceHandler offline_intelligence; - bool offline_mode_only = false; InvalidationCallBack invalidations; I_Messaging *message = nullptr; I_TimeGet *timer = nullptr; @@ -511,8 +447,6 @@ void IntelligenceComponentV2::init() { pimpl->init(); } void IntelligenceComponentV2::preload() { - registerExpectedConfiguration("intelligence", "offline intelligence path"); - registerExpectedConfiguration("intelligence", "offline intelligence only"); registerExpectedConfiguration("intelligence", "maximum request overall time"); registerExpectedConfiguration("intelligence", "maximum request lap time"); registerExpectedSetting("intelligence", "local intelligence server ip"); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc index 57e376e..06883f5 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc @@ -1,15 +1,15 @@ #include "intelligence_comp_v2.h" -#include "intelligence_is_v2/intelligence_query_v2.h" -#include "read_attribute_v2.h" -#include "cptest.h" -#include "singleton.h" #include "config.h" #include "config_component.h" -#include "mock/mock_messaging.h" +#include "cptest.h" +#include "mock/mock_intelligence.h" #include "mock/mock_mainloop.h" -#include "mock/mock_time_get.h" +#include "mock/mock_messaging.h" #include "mock/mock_rest_api.h" +#include "mock/mock_time_get.h" +#include "read_attribute_v2.h" +#include "singleton.h" using namespace std; using namespace testing; @@ -49,9 +49,7 @@ public: mockRestCall(_, "new-invalidation/source/invalidation", _) ).WillRepeatedly(Return(true)); - string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2"); - setConfiguration(offline_intel_path, string("intelligence"), string("offline intelligence path")); - + conf.preload(); intelligence.preload(); intelligence.init(); } @@ -72,6 +70,31 @@ public: I_MainLoop::Routine routine; }; +class IntelligenceComponentMockTest : public Test, Singleton::Consume +{ +public: + IntelligenceComponentMockTest() + { + debug_output.clear(); + Debug::setNewDefaultStdout(&debug_output); + Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE); + Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE); + setConfiguration(false, string("metric"), string("fogMetricSendEnable")); + + conf.preload(); + } + + ~IntelligenceComponentMockTest() + { + Debug::setNewDefaultStdout(&cout); + } + + ::Environment env; + ConfigComponent conf; + stringstream debug_output; + StrictMock intelligence_mock; +}; + class Profile { public: @@ -97,6 +120,252 @@ private: DataString phase; }; +TEST_F(IntelligenceComponentMockTest, getResponseErrorTest) +{ + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + QueryRequest request(Condition::EQUALS, "category", "cloud", true); + + Maybe res_error = genError("Test error"); + EXPECT_CALL(intelligence_mock, getResponse(_, _) + ).WillOnce(Return(res_error)); + + auto maybe_ans = intell->queryIntelligence(request); + EXPECT_FALSE(maybe_ans.ok()); +} + +TEST_F(IntelligenceComponentMockTest, getResponseTest) +{ + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + QueryRequest request(Condition::EQUALS, "category", "cloud", true); + + string response_str( + "{\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test\",\n" + " \"mainAttributes\": {\n" + " \"ipv4Addresses\": \"1.1.1.1\",\n" + " \"phase\": \"testing\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"testing\",\n" + " \"user\": \"Omry\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " ],\n" + " \"status\": \"done\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"start\"\n" + "}\n" + ); + + Intelligence::Response response(response_str, 1, false); + + EXPECT_CALL(intelligence_mock, getResponse(_, _) + ).WillOnce(Return(response)); + + auto maybe_ans = intell->queryIntelligence(request); + EXPECT_TRUE(maybe_ans.ok()); + auto vec = maybe_ans.unpack(); + EXPECT_EQ(vec.size(), 1u); + auto iter = vec.begin(); + EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry"); + EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing"); +} + +TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest) +{ + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + vector requests; + requests.emplace_back(Condition::EQUALS, "category", "whatever", true); + requests.emplace_back(Condition::EQUALS, "category", "cloud", true); + requests.emplace_back(Condition::EQUALS, "category", "nothing", true); + requests.emplace_back(Condition::EQUALS, "category", "iot", true); + + string response_str( + "{\n" + " \"errors\": [\n" + " {\n" + " \"index\": 0,\n" + " \"statusCode\": 400,\n" + " \"message\": \"Bad request. Error: Invalid cursor\"\n" + " }," + " {\n" + " \"index\": 2,\n" + " \"statusCode\": 405,\n" + " \"message\": \"Bad request. Error: Something else\"\n" + " }" + " ],\n" // errors + " \"queriesResponse\": [\n" + " {\n" + " \"index\": 1,\n" + " \"response\": {\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-ip\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"some-group-id\",\n" + " \"name\": \"[1.1.1.1]\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"ip\",\n" + " \"group\": \"\",\n" + " \"order\": \"\",\n" + " \"kind\": \"\",\n" + " \"mainAttributes\": {\n" + " \"ipv4Addresses\": [\n" + " \"1.1.1.1\",\n" + " \"2.2.2.2\"\n" + " ],\n" + " \"phase\": \"testing\"\n" + " },\n" // mainAttributes + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"color\": \"red\",\n" + " \"user\": \"Omry\",\n" + " \"phase\": \"testing\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " },\n" // source 1 + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n" + " \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 600,\n" + " \"attributes\": {\n" + " \"color\": \"white\",\n" + " \"user\": \"Max\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" // source 2 + " ]\n" // sources + " }\n" // asset 1 + " ],\n" // asset collection + " \"status\": \"done\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"start\"\n" + " }\n" // response + " },\n" // queryresponse 1 + " {\n" + " \"index\": 3,\n" + " \"response\": {\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-ip\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"some-group-id\",\n" + " \"name\": \"[2.2.2.2]\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"iot\",\n" + " \"family\": \"ip\",\n" + " \"group\": \"\",\n" + " \"order\": \"\",\n" + " \"kind\": \"\",\n" + " \"mainAttributes\": {\n" + " \"ipv4Addresses\": [\n" + " \"1.1.1.1\",\n" + " \"2.2.2.2\"\n" + " ],\n" + " \"phase\": \"testing\"\n" + " },\n" // mainAttributes + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"color\": \"red\",\n" + " \"user\": \"Omry2\",\n" + " \"phase\": \"testing2\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " },\n" // source 1 + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n" + " \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 600,\n" + " \"attributes\": {\n" + " \"color\": \"white\",\n" + " \"user\": \"Max\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" // source 2 + " ]\n" // sources + " }\n" // asset 1 + " ],\n" // asset collection + " \"status\": \"done\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"start\"\n" + " }\n" // response + " }\n" // queryresponse 1 + " ]\n" // queryresponses + "}\n" + ); + Intelligence::Response response(response_str, 4, true); + + EXPECT_CALL(intelligence_mock, getResponse(_, _, _) + ).WillOnce(Return(response)); + + auto maybe_ans = intell->queryIntelligence(requests); + EXPECT_TRUE(maybe_ans.ok()); + auto vec = maybe_ans.unpack(); + EXPECT_EQ(vec.size(), 4u); + EXPECT_FALSE(vec[0].ok()); + EXPECT_TRUE(vec[1].ok()); + EXPECT_FALSE(vec[2].ok()); + EXPECT_TRUE(vec[3].ok()); + + auto assets1_vec = vec[1].unpack(); + EXPECT_EQ(assets1_vec.size(), 1u); + auto iter = assets1_vec.begin(); + EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry"); + EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing"); + + auto assets3_vec = vec[3].unpack(); + EXPECT_EQ(assets1_vec.size(), 1u); + iter = assets3_vec.begin(); + EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2"); + EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2"); +} + TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest) { I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); @@ -154,6 +423,75 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest) EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing"); } +TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); + QueryRequest request(Condition::EQUALS, "category", "cloud", true); + + + string response_str( + "{\n" + " \"assetCollections\": [\n" + " {\n" + " \"schemaVersion\": 1,\n" + " \"assetType\": \"workload-cloud-fake-online-test\",\n" + " \"assetTypeSchemaVersion\": 1,\n" + " \"permissionType\": \"tenant\",\n" + " \"permissionGroupId\": \"fake-online-test-group\",\n" + " \"name\": \"fake-online-test-asset\",\n" + " \"class\": \"workload\",\n" + " \"category\": \"cloud\",\n" + " \"family\": \"fake-online-test\",\n" + " \"mainAttributes\": {\n" + " \"ipv4Addresses\": \"1.1.1.1\",\n" + " \"phase\": \"testing\"\n" + " },\n" + " \"sources\": [\n" + " {\n" + " \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n" + " \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n" + " \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n" + " \"ttl\": 120,\n" + " \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n" + " \"confidence\": 500,\n" + " \"attributes\": {\n" + " \"phase\": \"testing\",\n" + " \"user\": \"Omry\",\n" + " \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n" + " }\n" + " }\n" + " ]\n" + " }\n" + " ],\n" + " \"status\": \"done\",\n" + " \"totalNumAssets\": 2,\n" + " \"cursor\": \"start\"\n" + "}\n" + ); + + MessageMetadata md; + + EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _) + ).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str)))); + + auto maybe_ans = intell->queryIntelligence(request); + EXPECT_TRUE(maybe_ans.ok()); + + EXPECT_EQ(md.getHostName(), "127.0.0.1"); +} + TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest) { I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); @@ -626,6 +964,7 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest) ).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str))); auto maybe_ans3 = intell->queryIntelligence(request); + if (!maybe_ans3.ok()) cout << maybe_ans3.getErr() + "\n"; EXPECT_TRUE(maybe_ans3.ok()); auto vec3 = maybe_ans3.unpack(); EXPECT_EQ(vec3.size(), 1); @@ -636,54 +975,6 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest) EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry"); } -TEST_F(IntelligenceComponentTestV2, offlineIntelligenceTest) -{ - setConfiguration(true, string("intelligence"), string("offline intelligence only")); - intelligence.init(); - - I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); - QueryRequest request(Condition::EQUALS, "ip", "1.2.3.4", true); - auto maybe_ans = intell->queryIntelligence(request); - - ASSERT_TRUE(maybe_ans.ok()); - vector> vec = maybe_ans.unpack(); - vector>::iterator assets_iter = vec.begin(); - EXPECT_EQ(assets_iter->getAssetSchemaVersion(), 1); - EXPECT_EQ(assets_iter->getAssetType(), "workload-cloud-ip"); - EXPECT_EQ(assets_iter->getAssetTypeSchemaVersion(), 1); - EXPECT_EQ(assets_iter->getAssetPermissionGroupId(), "offline-group-id"); - EXPECT_EQ(assets_iter->getAssetName(), "offline-asset"); - EXPECT_EQ(assets_iter->getAssetClass(), "workload"); - EXPECT_EQ(assets_iter->getAssetCategory(), "cloud"); - EXPECT_EQ(assets_iter->getAssetFamily(), "offline family"); - EXPECT_EQ(assets_iter->getAssetGroup(), "offline testing"); - EXPECT_EQ(assets_iter->getAssetOrder(), ""); - EXPECT_EQ(assets_iter->getAssetKind(), ""); - - map> attributes_map = assets_iter->getMainAttributes(); - EXPECT_EQ(attributes_map["ip"].front(), "1.2.3.4"); - - vector>::const_iterator sources_iter = assets_iter->getSources().begin(); - EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229aa00"); - EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00"); - EXPECT_EQ(sources_iter->getAssetId(), "50255c3172b4fb7fda93025f0bfaa7abefd1"); - EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120)); - EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z"); - EXPECT_EQ(sources_iter->getConfidence(), 700); - EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry"); - EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test"); - - sources_iter++; - EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229bb11"); - EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7bbb11"); - EXPECT_EQ(sources_iter->getAssetId(), "cb068860528cb6bfb000cc35e79f11aeefed2"); - EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120)); - EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z"); - EXPECT_EQ(sources_iter->getConfidence(), 600); - EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Max"); - EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test"); -} - TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest) { I_Intelligence_IS_V2 *intell = Singleton::Consume::by(); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc index 66edd2d..17e93a2 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc @@ -11,7 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "intelligence_is_v2/intelligence_query_v2.h" +#include "intelligence_request.h" +#include "intelligence_is_v2/query_request_v2.h" #include "cptest.h" @@ -22,7 +23,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); - IntelligenceQuery query(request, true); + vector requests = {request}; + Intelligence::IntelligenceRequest query(requests, true, false); std::string expected = "{\n" " \"limit\": 20,\n" @@ -39,7 +41,9 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) { TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) { QueryRequest request(Condition::EQUALS, "phase", "testing", true); - IntelligenceQuery query(request, false); + vector requests = {request}; + Intelligence::IntelligenceRequest query(requests, false, false); + std::string expected = "{" "\"limit\":20," "\"fullResponse\":true," @@ -54,7 +58,8 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) { TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) { QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true); - IntelligenceQuery query(request, false); + vector requests = {request}; + Intelligence::IntelligenceRequest query(requests, false, false); std::string expected = "{" "\"limit\":20," "\"fullResponse\":true," @@ -71,7 +76,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) { QueryRequest request1(Condition::EQUALS, "phase", "testing", true); QueryRequest request2(Condition::EQUALS, "height", "testing", 25); std::vector requests = {request1, request2}; - IntelligenceQuery query(requests, true); + Intelligence::IntelligenceRequest query(requests, true, true); std::string expected = "{\n" " \"queries\": [\n" @@ -109,30 +114,30 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) { QueryRequest request1(Condition::EQUALS, "phase", "testing", true); QueryRequest request2(Condition::EQUALS, "height", "testing", 25); std::vector requests = {request1, request2}; - IntelligenceQuery query(requests, false); + Intelligence::IntelligenceRequest query(requests, false, true); -std::string expected = "{" - "\"queries\":[{" - "\"query\":{" - "\"limit\":20," - "\"fullResponse\":true," - "\"query\":{" - "\"operator\":\"equals\"," - "\"key\":\"mainAttributes.phase\"," - "\"value\":\"testing\"" - "}}," - "\"index\":0" - "},{" - "\"query\":{" - "\"limit\":20," - "\"fullResponse\":true," - "\"query\":{" - "\"operator\":\"equals\"," - "\"key\":\"mainAttributes.height\"," - "\"value\":\"testing\"" - "}}," - "\"index\":1" - "}]}"; + std::string expected = "{" + "\"queries\":[{" + "\"query\":{" + "\"limit\":20," + "\"fullResponse\":true," + "\"query\":{" + "\"operator\":\"equals\"," + "\"key\":\"mainAttributes.phase\"," + "\"value\":\"testing\"" + "}}," + "\"index\":0" + "},{" + "\"query\":{" + "\"limit\":20," + "\"fullResponse\":true," + "\"query\":{" + "\"operator\":\"equals\"," + "\"key\":\"mainAttributes.height\"," + "\"value\":\"testing\"" + "}}," + "\"index\":1" + "}]}"; EXPECT_EQ(*query.genJson(), expected); } diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc index 7e92420..95e097a 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc @@ -172,6 +172,15 @@ TEST(QueryRequestTestV2, AttributesTest) EXPECT_EQ(out.str(), output_json); } +TEST(QueryRequestTestV2, AttributeConditionExceptionTest) +{ + try { + Intelligence_IS_V2::createAttributeString("jey", (Intelligence_IS_V2::AttributeKeyType)5); + } catch (const IntelligenceException &e) { + EXPECT_THAT(string(e.what()), HasSubstr("Received illegal Attribute Type.")); + } +} + TEST(QueryRequestTestV2, AndQueryTest) { QueryRequest request1(Condition::EQUALS, "phase", "testing1", true); diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc index 76a2c63..21af565 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc @@ -1,5 +1,3 @@ -#include "intelligence_is_v2/query_response_v2.h" -#include "intelligence_is_v2/asset_source_v2.h" #include "intelligence_is_v2/intelligence_types_v2.h" #include "intelligence_is_v2/data_string_v2.h" @@ -31,12 +29,12 @@ public: try{ ReadAttribute("color", data).serialize(ar); } catch (exception &e) { - dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color"; + dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color. Error:" + string(e.what()); } try { ReadAttribute("user", data1).serialize(ar); } catch (const exception &e) { - dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user"; + dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user. Error:" + string(e.what()); } } @@ -47,9 +45,6 @@ private: TEST(QueryResponseTestV2, ReadAttributeTest) { - StrictMock mock_ml; - NiceMock time_get; - IntelligenceComponentV2 new_intelligence; DataString data; ReadAttribute obj("user", data); @@ -70,9 +65,6 @@ TEST(QueryResponseTestV2, ReadAttributeTest) TEST(QueryResponseTestV2, stringData1Test) { - StrictMock mock_ml; - NiceMock time_get; - IntelligenceComponentV2 new_intelligence; DataString data; stringData1 obj; string data_str( @@ -95,11 +87,9 @@ TEST(QueryResponseTestV2, stringData1Test) TEST(QueryResponseTestV2, QueryResponseTestV2) { - StrictMock mock_ml; - NiceMock time_get; - IntelligenceComponentV2 new_intelligence; DataString data; - IntelligenceQueryResponse obj; + IntelligenceQueryResponseT obj; + IntelligenceQueryResponse obj2; string data_str( "{\n" " \"assetCollections\": [\n" @@ -161,12 +151,14 @@ TEST(QueryResponseTestV2, QueryResponseTestV2) stringstream ss(data_str); { - cereal::JSONInputArchive ar(ss); - obj.loadFromJson(ar); + obj.loadFromJson(ss.str()); + obj2.loadFromJson(ss.str()); } EXPECT_EQ(obj.getAmountOfAssets(), 2); + EXPECT_EQ(obj2.getAmountOfAssets(), 2); EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE); + EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE); EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1); EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip"); EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1); @@ -227,11 +219,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2) TEST(QueryResponseTestV2, MainAttributesTestV2) { - StrictMock mock_ml; - NiceMock time_get; - IntelligenceComponentV2 new_intelligence; DataString data; - IntelligenceQueryResponse obj; + IntelligenceQueryResponseT obj; string string_attribute( "{\n" " \"assetCollections\": [\n" @@ -277,7 +266,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2) stringstream ss(string_attribute); { cereal::JSONInputArchive ar(ss); - obj.loadFromJson(ar); + obj.serialize(ar); } map> attributes_map = obj.getData().begin()->getMainAttributes(); @@ -331,7 +320,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2) stringstream ss2(many_strings_attribute); { cereal::JSONInputArchive ar(ss2); - obj.loadFromJson(ar); + obj.serialize(ar); } map> attributes_map2 = obj.getData().begin()->getMainAttributes(); @@ -390,7 +379,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2) stringstream ss3(strings_vector_attribute); { cereal::JSONInputArchive ar(ss3); - obj.loadFromJson(ar); + obj.serialize(ar); } map> attributes_map3 = obj.getData().begin()->getMainAttributes(); @@ -403,11 +392,8 @@ TEST(QueryResponseTestV2, MainAttributesTestV2) TEST(QueryResponseTestV2, IntelligenceFailTest) { - StrictMock mock_ml; - NiceMock time_get; - IntelligenceComponentV2 new_intelligence; DataString data; - IntelligenceQueryResponse obj; + IntelligenceQueryResponseT obj; string status_fail_data_str( "{\n" " \"assetCollections\": [\n" @@ -456,7 +442,7 @@ TEST(QueryResponseTestV2, IntelligenceFailTest) { cereal::JSONInputArchive ar(ss); try { - obj.loadFromJson(ar); + obj.serialize(ar); } catch (exception &e) { EXPECT_EQ(e.what(), error_str); } diff --git a/core/intelligence_is_v2/intelligence_request.cc b/core/intelligence_is_v2/intelligence_request.cc new file mode 100644 index 0000000..3f3ee55 --- /dev/null +++ b/core/intelligence_is_v2/intelligence_request.cc @@ -0,0 +1,128 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_request.h" +#include "debug.h" +#include "intelligence_comp_v2.h" +#include "intelligence_is_v2/json_stream.h" + + +using namespace Intelligence; +using namespace std; + +USE_DEBUG_FLAG(D_INTELLIGENCE); + +static const unsigned int upper_assets_limit = 50; +static const unsigned int upper_confidence_limit = 1000; + +Maybe +IntelligenceRequest::checkAssetsLimit() const +{ + for (const QueryRequest &query_request : queries) { + uint assets_limit = query_request.getAssetsLimit(); + if (0 < assets_limit && assets_limit <= upper_assets_limit) continue; + dbgTrace(D_INTELLIGENCE) + << "Assets limit for request is " + << upper_assets_limit + << ", requests assets: " + << assets_limit; + return genError("Assets limit valid range is of [1, " + to_string(upper_assets_limit) + "]"); + } + return Maybe(); +} + +Maybe +IntelligenceRequest::checkMinConfidence() const +{ + for (const QueryRequest &query_request : queries) { + if (query_request.checkMinConfidence(upper_confidence_limit)) continue; + dbgTrace(D_INTELLIGENCE) << "Illegal confidence value"; + return genError( + "Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]" + ); + } + return Maybe(); +} + +bool +IntelligenceRequest::isPagingActivated() const +{ + if (!isPagingAllowed()) return false; + return queries.begin()->getCursorState().ok(); +} + +Maybe +IntelligenceRequest::isPagingFinished() const +{ + if (!isPagingActivated()) return genError("Paging is not activated"); + return queries.begin()->getCursorState().unpack() == CursorState::DONE; +} + +Maybe +IntelligenceRequest::getPagingStatus() const +{ + if (!isPagingAllowed()) return genError("Paging is not allowed"); + return queries.begin()->getCursorState(); +} + +bool +IntelligenceRequest::isPagingAllowed() const +{ + if (isBulk()) return false; + return true; +} + +Maybe +IntelligenceRequest::genJson() const +{ + std::stringstream str_stream; + JsonStream json_stream(&str_stream, is_pretty); + { + cereal::JSONOutputArchive out_ar(json_stream); + if (isBulk()) { + out_ar.setNextName("queries"); + out_ar.startNode(); + out_ar.makeArray(); + uint index = 0; + for (const auto &query : queries) { + out_ar.setNextName(nullptr); + out_ar.startNode(); + out_ar.setNextName("query"); + out_ar.startNode(); + query.saveToJson(out_ar); + out_ar.finishNode(); + out_ar(cereal::make_nvp("index", index)); + out_ar.finishNode(); + index++; + } + out_ar.finishNode(); + } else { + queries.begin()->saveToJson(out_ar); + } + } + + return str_stream.str(); +} + +bool +IntelligenceRequest::loadJson(const string &json) +{ + response_from_fog = json; + return true; +} + +Maybe +IntelligenceRequest::getResponseFromFog() const +{ + return response_from_fog; +} diff --git a/core/intelligence_is_v2/intelligence_response.cc b/core/intelligence_is_v2/intelligence_response.cc new file mode 100644 index 0000000..8ed56ba --- /dev/null +++ b/core/intelligence_is_v2/intelligence_response.cc @@ -0,0 +1,74 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "intelligence_is_v2/intelligence_response.h" + +using namespace std; +using namespace Intelligence; + +Maybe +Response::load() +{ + try { + stringstream in; + in.str(json_response); + cereal::JSONInputArchive in_ar(in); + if (is_bulk) { + IntelligenceQueryBulkResponse bulk_response; + bulk_response.serialize(in_ar); + unsigned int error_idx = 0; + unsigned int valid_idx = 0; + const auto &valid_response = bulk_response.getValid(); + const auto &errors = bulk_response.getErrors(); + responses.clear(); + responses.reserve(size); + dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items"; + for (unsigned int query_idx = 0; query_idx < size; query_idx++) { + if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) { + responses.push_back(valid_response[valid_idx].getResponse()); + dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid"; + valid_idx++; + } else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) { + responses.emplace_back(); + responses[query_idx].setFailInBulk(); + dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid"; + error_idx++; + } else { + dbgWarning(D_INTELLIGENCE) + << "Query index was not found neither in valid nor error responses, assuming error"; + responses[query_idx].setFailInBulk(); + } + } + } else { + single_response.serialize(in_ar); + } + } catch(const std::exception &e) { + return genError("Load common data failed. Error: " + string(e.what())); + } + return {}; +} + +Intelligence_IS_V2::ResponseStatus +Response::getResponseStatus() const +{ + if (!is_bulk) return single_response.getResponseStatus(); + + if (responses.size() == 0) return ResponseStatus::IN_PROGRESS; + for (const auto &response_itr : responses) { + if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) { + return ResponseStatus::IN_PROGRESS; + } + } + + return ResponseStatus::DONE; +} diff --git a/core/intelligence_is_v2/intelligence_server.cc b/core/intelligence_is_v2/intelligence_server.cc new file mode 100644 index 0000000..9fed3a0 --- /dev/null +++ b/core/intelligence_is_v2/intelligence_server.cc @@ -0,0 +1,203 @@ +// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "intelligence_server.h" +#include "config.h" +#include "debug.h" +#include "enum_array.h" +#include "intelligence_comp_v2.h" + +USE_DEBUG_FLAG(D_INTELLIGENCE); + +using namespace Intelligence; +using namespace std; + +static const string query_uri = "/api/v2/intelligence/assets/query"; +static const string queries_uri = "/api/v2/intelligence/assets/queries"; +static const string primary_port_setting = "local intelligence server primary port"; +static const string secondary_port_setting = "local intelligence server secondary port"; + +Sender::Sender(IntelligenceRequest request) : request(request) +{ + i_message = Singleton::Consume::by(); + i_timer = Singleton::Consume::by(); + i_mainloop = Singleton::Consume::by(); + bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ? + std::string(std::getenv("CROWDSEC_ENABLED")) == "true" : + false; + if (getProfileAgentSettingWithDefault(crowdsec_enabled, "layer7AccessControl.crowdsec.enabled")) { + is_local_intelligence = true; + } + + if (getProfileAgentSettingWithDefault(false, "agent.config.useLocalIntelligence")) { + is_local_intelligence = true; + } + + auto setting_server_ip = getSetting("intelligence", "local intelligence server ip"); + if (setting_server_ip.ok()) server_ip = *setting_server_ip; +} + +Maybe +Sender::sendIntelligenceRequest() +{ + if (server_ip.ok() && is_local_intelligence) { + auto response = sendQueryObjectToLocalServer(true); + if (response.ok()) return response; + dbgWarning(D_INTELLIGENCE) << "Failed to send query to primary port. Error" << response.getErr(); + response = sendQueryObjectToLocalServer(false); + if (response.ok()) return response; + dbgWarning(D_INTELLIGENCE) << "Failed to send query to secondary port. Error" << response.getErr(); + } + + if (request.getPagingStatus().ok()) { + return sendMessage(); + } + + return sendQueryMessage(); +} + +Maybe +Sender::sendQueryObjectToLocalServer(bool is_primary_port) +{ + auto local_port = getSetting( + "intelligence", + is_primary_port ? primary_port_setting : secondary_port_setting + ); + + if (!local_port.ok()) return genError( + "Failed to send intelligence query to local server. Config Error number:" + + to_string(static_cast(local_port.getErr())) + ); + + server_port = *local_port; + conn_flags.reset(); + + auto res = sendQueryMessage(); + + server_port = genError("port unset after use"); + + return res; +} + +Maybe +Sender::sendQueryMessage() +{ + uint request_overall_timeout_conf = getConfigurationWithDefault( + 20, + "intelligence", + "request overall timeout" + ); + + uint request_lap_timeout_conf = getConfigurationWithDefault( + 5, + "intelligence", + "request lap timeout" + ); + + chrono::seconds request_overall_timeout = chrono::seconds(request_overall_timeout_conf); + chrono::seconds request_lap_timeout = chrono::seconds(request_lap_timeout_conf); + + chrono::microseconds send_request_start_time = i_timer->getMonotonicTime(); + chrono::microseconds last_lap_time = i_timer->getMonotonicTime(); + chrono::seconds seconds_since_start = chrono::seconds(0); + chrono::seconds seconds_since_last_lap = chrono::seconds(0); + + Maybe res = genError("Uninitialized"); + do { + res = sendMessage(); + + if (res.ok() && res->getResponseStatus() == ResponseStatus::IN_PROGRESS) { + i_mainloop->yield(true); + } + + seconds_since_start = std::chrono::duration_cast( + i_timer->getMonotonicTime() - send_request_start_time + ); + + seconds_since_last_lap = std::chrono::duration_cast( + i_timer->getMonotonicTime() - last_lap_time + ); + last_lap_time = i_timer->getMonotonicTime(); + } while (res.ok() && + res->getResponseStatus() == ResponseStatus::IN_PROGRESS && + seconds_since_start < request_overall_timeout && + seconds_since_last_lap < request_lap_timeout + ); + + return res; +} + +Maybe +Sender::sendMessage() +{ + if (server_ip.ok() || server_port.ok()) { + if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set"); + if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set"); + } else if (!server_ip.ok() && !server_port.ok()) { + auto req_status = i_message->sendSyncMessage( + HTTPMethod::POST, + request.isBulk() ? queries_uri : query_uri, + request, + MessageCategory::INTELLIGENCE + ); + if (req_status.ok()) { + return createResponse(); + }; + auto response_error = req_status.getErr().toString(); + dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error; + return genError( + "Failed to send intelligence request. " + + req_status.getErr().getBody() + + " " + + req_status.getErr().toString() + ); + } + + dbgTrace(D_INTELLIGENCE) + << "Sending intelligence request with IP: " + << *server_ip + << " port: " + << *server_port + << " query_uri: " + << (request.isBulk() ? queries_uri : query_uri); + + MessageMetadata req_md(*server_ip, *server_port, conn_flags); + auto req_status = i_message->sendSyncMessage( + HTTPMethod::POST, + query_uri, + request, + MessageCategory::INTELLIGENCE, + req_md + ); + if (!req_status.ok()) { + auto response_error = req_status.getErr().toString(); + dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error; + return genError( + "Failed to send intelligence request. " + + req_status.getErr().getBody() + + " " + + req_status.getErr().toString() + ); + }; + return createResponse(); +} + +Maybe +Sender::createResponse() +{ + auto mb_json_body = request.getResponseFromFog(); + if (!mb_json_body.ok()) return mb_json_body.passErr(); + Response response(*mb_json_body, request.getSize(), request.isBulk()); + auto load_status = response.load(); + if (!load_status.ok()) return load_status.passErr(); + return response; +} diff --git a/core/intelligence_is_v2/query_filter_v2.cc b/core/intelligence_is_v2/query_filter_v2.cc index 847d1a0..6c8dd7a 100644 --- a/core/intelligence_is_v2/query_filter_v2.cc +++ b/core/intelligence_is_v2/query_filter_v2.cc @@ -146,18 +146,6 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const } } -Maybe -SerializableQueryFilter::getConditionValueByKey(const string &key) const -{ - for (const SerializableQueryCondition &condition : condition_operands) { - if (condition.getConditionType() == Condition::EQUALS && condition.getKey() == key) { - return condition.getValue(); - } - } - - return genError("Key not found."); -} - bool SerializableQueryFilter::isOperatorComp(const Operator &oper) const { diff --git a/core/intelligence_is_v2/query_request_v2.cc b/core/intelligence_is_v2/query_request_v2.cc index fe95566..cc722cc 100644 --- a/core/intelligence_is_v2/query_request_v2.cc +++ b/core/intelligence_is_v2/query_request_v2.cc @@ -25,27 +25,6 @@ USE_DEBUG_FLAG(D_INTELLIGENCE); static const EnumArray object_type_to_string_array{"asset", "zone", "configuration", "shortLived"}; -BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index) - : - request(_request), - index(_index) -{} - -QueryRequest -BulkQueryRequest::getQueryRequest() const -{ - return request; -} - -void -BulkQueryRequest::save(cereal::JSONOutputArchive &ar) const -{ - ar( - cereal::make_nvp("query", getQueryRequest()), - cereal::make_nvp("index", index) - ); -} - QueryRequest::QueryRequest( Condition condition_type, const string &key, @@ -96,22 +75,7 @@ QueryRequest::convertObjectTypeToString() const void QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const { - ar( - cereal::make_nvp("limit", assets_limit), - cereal::make_nvp("fullResponse", full_response), - cereal::make_nvp("query", query) - ); - - auto objTypeString = convertObjectTypeToString(); - if (objTypeString.ok()) { - ar(cereal::make_nvp("objectType", *objTypeString)); - } else { - dbgTrace(D_INTELLIGENCE) << objTypeString.getErr(); - } - - if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second)); - requested_attributes.save(ar); - query_types.save(ar); + save(ar); } void @@ -214,7 +178,7 @@ QueryRequest::setAssetsLimit(uint _assets_limit) } bool -QueryRequest::checkMinConfidence(uint upper_confidence_limit) +QueryRequest::checkMinConfidence(uint upper_confidence_limit) const { return requested_attributes.checkMinConfidence(upper_confidence_limit); } @@ -232,7 +196,7 @@ QueryRequest::isPagingActivated() } Maybe -QueryRequest::getCursorState() +QueryRequest::getCursorState() const { if (!cursor.ok()) return genError("Paging not activated"); return cursor.unpack().first; diff --git a/core/intelligence_is_v2/requested_attributes_v2.cc b/core/intelligence_is_v2/requested_attributes_v2.cc index d2c9d82..9fcb6a1 100644 --- a/core/intelligence_is_v2/requested_attributes_v2.cc +++ b/core/intelligence_is_v2/requested_attributes_v2.cc @@ -54,7 +54,7 @@ SerializableAttributesMap::getAttributeByKey(const string &key) const } bool -SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit) +SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit) const { for (auto const &attribute : requested_attributes) { if (attribute.second == 0 || attribute.second > upper_confidence_limit) return false; diff --git a/nodes/http_transaction_handler/CMakeLists.txt b/nodes/http_transaction_handler/CMakeLists.txt index 2062def..ca65105 100755 --- a/nodes/http_transaction_handler/CMakeLists.txt +++ b/nodes/http_transaction_handler/CMakeLists.txt @@ -15,6 +15,7 @@ target_link_libraries(cp-nano-http-transaction-handler pcre2-posix yajl_s hiredis + maxminddb -lshmem_ipc -lnginx_attachment_util @@ -38,6 +39,8 @@ target_link_libraries(cp-nano-http-transaction-handler ips keywords l7_access_control + geo_location + http_geo_filter -Wl,--end-group ) diff --git a/nodes/http_transaction_handler/main.cc b/nodes/http_transaction_handler/main.cc index a4dd49a..3b393bb 100755 --- a/nodes/http_transaction_handler/main.cc +++ b/nodes/http_transaction_handler/main.cc @@ -21,6 +21,8 @@ #include "waap.h" #include "ips_comp.h" #include "keyword_comp.h" +#include "http_geo_filter.h" +#include "geo_location.h" int main(int argc, char **argv) @@ -34,7 +36,9 @@ main(int argc, char **argv) RateLimit, WaapComponent, IPSComp, - KeywordComp + KeywordComp, + GeoLocation, + HttpGeoFilter > comps; comps.registerGlobalValue("Is Rest primary routine", true); diff --git a/nodes/orchestration/package/open-appsec-ctl.sh b/nodes/orchestration/package/open-appsec-ctl.sh index 84bace9..bde5db4 100644 --- a/nodes/orchestration/package/open-appsec-ctl.sh +++ b/nodes/orchestration/package/open-appsec-ctl.sh @@ -1027,6 +1027,10 @@ run_status() # Initials - rs for service in $all_services; do print_single_service_status "$service" done + + if command -v getenforce &>/dev/null && [ "$(getenforce)" != "Disabled" ]; then + echo "SELinux has been detected, which could potentially disrupt the agent's normal operation." + fi } run_load_settings() # Initials - rls