github_mirrors/openappsec
3
0
Fork 1
mirror of https://github.com/openappsec/openappsec.git synced 2025-06-28 16:41:02 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feb_06_2024-Dev

Browse Source
This commit is contained in:
Ned Wright 2024-02-06 16:41:40 +00:00
parent 9f8535c0f7
commit 623951a2f0
59 changed files with 2207 additions and 1168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -106,12 +106,13 @@ Before compiling the services, you'll need to ensure the latest development vers
* cURL
* Redis
* Hiredis
* MaxmindDB
An example of installing the packages on Alpine:
```bash
$ apk update
$ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis
$ apk add boost-dev openssl-dev pcre2-dev libxml2-dev gtest-dev curl-dev hiredis-dev redis libmaxminddb-dev
```
## Compiling and packaging the agent code

4
components/include/health_checker.h
View File

@ -19,6 +19,7 @@
#include "i_socket_is.h"
#include "i_health_check_manager.h"
#include "i_shell_cmd.h"
#include "i_orchestration_status.h"
#include "component.h"
class HealthChecker
@ -27,7 +28,8 @@ class HealthChecker
Singleton::Consume<I_MainLoop>,
Singleton::Consume<I_Socket>,
Singleton::Consume<I_Health_Check_Manager>,
Singleton::Consume<I_ShellCmd>
Singleton::Consume<I_ShellCmd>,
Singleton::Consume<I_OrchestrationStatus>
{
public:
HealthChecker();

46
components/include/reverse_proxy_defaults.h Normal file
View File

@ -0,0 +1,46 @@
#ifndef __REVERSE_PROXY_MANAGER_DEFAULTS_H__
#define __REVERSE_PROXY_MANAGER_DEFAULTS_H__
#include <string>
static const std::string product_name = getenv("DOCKER_RPM_ENABLED") ? "CloudGuard AppSec" : "AppSec Gateway";
static const std::string default_cp_cert_file = "/etc/cp/cpCert.pem";
static const std::string default_cp_key_file = "/etc/cp/cpKey.key";
static const std::string default_rpm_conf_path = "/etc/cp/conf/rpmanager/";
static const std::string default_certificate_path = "/etc/cp/rpmanager/certs";
static const std::string default_additional_files_path = "/etc/cp/conf/rpmanager/include";
static const std::string default_server_config = "additional_server_config.conf";
static const std::string default_location_config = "additional_location_config.conf";
static const std::string default_trusted_ca_suffix = "_user_ca_bundle.crt";
static const std::string default_nginx_log_files_path = "/var/log/nginx/";
static const std::string default_log_files_host_path = "/var/log/nano_agent/rpmanager/nginx_log/";
static const std::string default_config_path = "/etc/cp/conf/rpmanager/servers";
static const std::string default_template_path = "/etc/cp/conf/rpmanager/nginx-template-clear";
static const std::string default_manual_certs_path = "/etc/cp/rpmanager/manualCerts/";
static const std::string default_server_certificate_path = "/etc/cp/rpmanager/certs/sslCertificate_";
static const std::string default_server_certificate_key_path = "/etc/cp/rpmanager/certs/sslPrivateKey_";
static const std::string default_container_name = "cp_nginx_gaia";
static const std::string default_docker_image = "cp_nginx_gaia";
static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/nginx.conf";
static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template";
static const std::string default_nginx_config_include_file =
"/etc/cp/conf/rpmanager/servers/nginx_conf_include";
static const std::string default_global_conf_include_template =
"/etc/cp/conf/rpmanager/nginx-conf-include-template";
static const std::string default_global_conf_include_template_no_responses =
"/etc/cp/conf/rpmanager/nginx-conf-include-template-no-responses";
static const std::string default_cloud_vendor_file = "/etc/cp/conf/rpmanager/cloud-vendor.json";
static const std::string default_cloud_cert_location = "/tmp/";
static const std::string default_dns_resolver_file = "/etc/resolv.conf";
static const std::string default_nginx_multi_lines_key = "nginxIncludeLines";
static const std::string default_ip = "127.0.0.1";
static const std::string default_aws_resolver_ip = "169.254.169.253";
static const std::string default_azure_resolver_ip = "168.63.129.16";
static const std::string default_syslog_socket_address = "127.0.0.1:1514";
static const std::string rpm_full_load_path = "/tmp/rpm_full_load";
static const std::string rpm_partial_load_path = "/tmp/rpm_partial_load";
static const std::string first_rpm_policy_load_path = "/tmp/first_rpm_policy_load";
static const int default_port = 5555;
#endif //__REVERSE_PROXY_MANAGER_DEFAULTS_H__

2
components/security_apps/CMakeLists.txt
View File

@ -1,4 +1,4 @@
#add_subdirectory(http_geo_filter)
add_subdirectory(http_geo_filter)
add_subdirectory(ips)
add_subdirectory(layer_7_access_control)
add_subdirectory(local_policy_mgmt_gen)

18
components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h
View File

@ -69,19 +69,11 @@ checkIDP(shared_ptr<istream> file_stream)
#if defined(gaia) || defined(smb)
Maybe<string>
checkIsCpviewRunning(const string &command_output)
checkIsInstallHorizonTelemetrySucceeded(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
if (command_output == "" ) return string("false");
return genError("cpview is not running");
}
Maybe<string>
checkIsCPotelcolGRET64(const string &command_output)
{
if (command_output == "true" || command_output == "false") return command_output;
return genError("CPotelcol is not installed or its take is below T64");
return command_output;
}
Maybe<string>
@ -103,6 +95,10 @@ checkCanUpdateSDWanData(const string &command_output)
Maybe<string>
getMgmtObjType(const string &command_output)
{
if (getenv("WAAP_DIR")) {
return string("CloudGuard WAF Gateway");
}
if (!command_output.empty()) {
if (command_output[0] == '1') return string("management");
if (command_output[0] == '0') return string("gateway");

12
components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h
View File

@ -39,14 +39,10 @@ SHELL_PRE_CMD("read sdwan data",
#ifdef SHELL_CMD_HANDLER
#if defined(gaia) || defined(smb)
SHELL_CMD_HANDLER("cpProductIntegrationMgmtObjectType", "cpprod_util CPPROD_IsMgmtMachine", getMgmtObjType)
SHELL_CMD_HANDLER("isCpviewRunning",
"pidof cpview_api_service > /dev/null 2>&1 && [ -f $CPDIR/conf/cpview_api_service.version ] "
"&& echo 'true' || echo 'false'",
checkIsCpviewRunning)
SHELL_CMD_HANDLER("isCPotelcolGRET64",
"grep -A 10 '(BUNDLE_CPOTELCOL_AUTOUPDATE' ${CPDIR}/registry/HKLM_registry.data | "
"awk '/SU_Build_Take/{val = substr($2, 2, length($2)-2); if (val >=64) print \"true\"; else print \"false\" }'",
checkIsCPotelcolGRET64)
SHELL_CMD_HANDLER("prerequisitesForHorizonTelemetry",
"[ -f /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log ] "
"&& head -1 /var/log/nano_agent/cp-nano-horizon-telemetry-prerequisites.log || echo ''",
checkIsInstallHorizonTelemetrySucceeded)
SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan)
SHELL_CMD_HANDLER(
"canUpdateSDWanData",

53
components/security_apps/orchestration/health_check/health_check.cc
View File

@ -18,6 +18,7 @@
#include <sys/socket.h>
#include <unordered_map>
#include "reverse_proxy_defaults.h"
#include "config.h"
#include "log_generator.h"
#include "health_check_manager.h"
@ -28,6 +29,8 @@ using namespace ReportIS;
USE_DEBUG_FLAG(D_HEALTH_CHECK);
static const std::string all_services_running = "/tmp/wd.all_running";
class HealthChecker::Impl
{
public:
@ -36,6 +39,7 @@ public:
{
i_mainloop = Singleton::Consume<I_MainLoop>::by<HealthChecker>();
i_socket = Singleton::Consume<I_Socket>::by<HealthChecker>();
i_orchestration_status = Singleton::Consume<I_OrchestrationStatus>::by<HealthChecker>();
initConfig();
initServerSocket();
@ -255,35 +259,33 @@ private:
{
if (!getenv("DOCKER_RPM_ENABLED")) return HealthCheckStatus::IGNORED;
static const string standalone_cmd = "/usr/sbin/cpnano -s --docker-rpm; echo $?";
static int timeout_tolerance = 1;
static HealthCheckStatus health_status = HealthCheckStatus::HEALTHY;
dbgTrace(D_HEALTH_CHECK) << "Checking the standalone docker health status with command: " << standalone_cmd;
auto maybe_result = Singleton::Consume<I_ShellCmd>::by<HealthChecker>()->getExecOutput(standalone_cmd, 5000);
if (!maybe_result.ok()) {
if (maybe_result.getErr().find("Reached timeout") != string::npos) {
dbgWarning(D_HEALTH_CHECK)
<< "Reached timeout while querying standalone health status, attempt number: "
<< timeout_tolerance;
return health_status == HealthCheckStatus::UNHEALTHY || timeout_tolerance++ > 3 ?
HealthCheckStatus::UNHEALTHY :
health_status;
}
dbgWarning(D_HEALTH_CHECK) << "Unable to get the standalone docker status. Returning unhealthy status.";
if (i_orchestration_status->getPolicyVersion().empty()) {
dbgTrace(D_HEALTH_CHECK) << "Policy version is empty, returning unhealthy status";
return HealthCheckStatus::UNHEALTHY;
}
dbgTrace(D_HEALTH_CHECK) << "Got response: " << maybe_result.unpack();
auto response = NGEN::Strings::removeTrailingWhitespaces(maybe_result.unpack());
if (!NGEN::Filesystem::exists(all_services_running)) {
dbgTrace(D_HEALTH_CHECK) << all_services_running << " does not exist, returning unhealthy status";
return HealthCheckStatus::UNHEALTHY;
}
if (response.back() == '1') return health_status = HealthCheckStatus::UNHEALTHY;
if (NGEN::Filesystem::exists(rpm_full_load_path)) {
dbgTrace(D_HEALTH_CHECK) << rpm_full_load_path << " exists, returning healthy status";
return HealthCheckStatus::HEALTHY;
}
timeout_tolerance = 1;
return health_status = (response.back() == '0') ? HealthCheckStatus::HEALTHY : HealthCheckStatus::DEGRADED;
if (NGEN::Filesystem::exists(rpm_partial_load_path)) {
dbgTrace(D_HEALTH_CHECK) << rpm_partial_load_path << " exists, returning degraded status";
return HealthCheckStatus::DEGRADED;
}
if (!NGEN::Filesystem::exists(first_rpm_policy_load_path)) {
dbgTrace(D_HEALTH_CHECK) << "Could not load latest RPM policy, returning degraded status";
return HealthCheckStatus::DEGRADED;
}
dbgTrace(D_HEALTH_CHECK) << "RPM is not loaded, returning unhealthy status";
return HealthCheckStatus::UNHEALTHY;
}
bool
@ -390,7 +392,7 @@ private:
if (standalone_status == HealthCheckStatus::UNHEALTHY) {
dbgDebug(D_HEALTH_CHECK)
<< "Standalone status in unhealthy, returning the following response: "
<< "Standalone status is unhealthy, returning the following response: "
<< failure_response;
i_socket->writeData(curr_client_socket, failure_response_buffer);
closeCurrentSocket(curr_client_socket, curr_routine_id);
@ -439,6 +441,7 @@ private:
I_MainLoop *i_mainloop = nullptr;
I_Socket *i_socket = nullptr;
I_Health_Check_Manager *i_health_check_manager = nullptr;
I_OrchestrationStatus *i_orchestration_status = nullptr;
};
HealthChecker::HealthChecker() : Component("HealthChecker"), pimpl(make_unique<Impl>()) {}

3
components/security_apps/orchestration/health_check/health_check_ut/health_check_ut.cc
View File

@ -7,6 +7,7 @@
#include "mock/mock_socket_is.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_shell_cmd.h"
#include "mock/mock_orchestration_status.h"
#include "health_check_manager.h"
#include "config.h"
@ -65,6 +66,7 @@ public:
AgentDetails agent_details;
StrictMock<MockSocketIS> mock_socket;
NiceMock<MockShellCmd> mock_shell_cmd;
NiceMock<MockOrchestrationStatus> mock_orchestration_status;
I_Socket::socketFd server_socket = -1;
Context ctx;
ConfigComponent config;
@ -72,7 +74,6 @@ public:
I_MainLoop::Routine connection_handler_routine;
I_MainLoop::Routine client_connection_handler_routine;
I_MainLoop::Routine handle_probe_routine;
//StrictMock<MockHealthCheckManager> mock_health_check_manager;
HealthCheckManager health_check_manager;
I_Health_Check_Manager *i_health_check_manager;
};

2
components/security_apps/orchestration/include/fog_authenticator.h
View File

@ -102,6 +102,7 @@ public:
Maybe<void> authenticateAgent() override;
void setAddressExtenesion(const std::string &extension) override;
static std::string getUserEdition();
protected:
class UserCredentials
@ -142,7 +143,6 @@ protected:
std::string base64Encode(const std::string &in) const;
std::string buildBasicAuthHeader(const std::string &username, const std::string &pass) const;
std::string buildOAuth2Header(const std::string &token) const;
std::string getUserEdition() const;
// This apps which the orchestrations requires them from Fog.
std::vector<std::string> required_security_apps;

2
components/security_apps/orchestration/include/hybrid_communication.h
View File

@ -56,7 +56,9 @@ public:
private:
Maybe<std::string> getNewVersion();
void doLocalFogOperations(const std::string &policy) const;
std::string tuning_host;
I_DeclarativePolicy *i_declarative_policy = nullptr;
};

6
components/security_apps/orchestration/orchestration_comp.cc
View File

@ -41,6 +41,7 @@
#include "env_details.h"
#include "hybrid_communication.h"
#include "agent_core_utilities.h"
#include "fog_communication.h"
using namespace std;
using namespace chrono;
@ -190,7 +191,8 @@ public:
);
auto orchestration_tools = Singleton::Consume<I_OrchestrationTools>::by<OrchestrationComp>();
orchestration_tools->getClusterId();
if (getAttribute("no-setting", "IGNORE_CLUSTER_ID") != "TRUE") orchestration_tools->getClusterId();
hybrid_mode_metric.init(
"Watchdog Metrics",
@ -1484,6 +1486,8 @@ private:
agent_data_report << AgentReportFieldWithLabel("managedMode", "management");
}
agent_data_report << AgentReportFieldWithLabel("userEdition", FogCommunication::getUserEdition());
#if defined(gaia) || defined(smb)
if (i_details_resolver->compareCheckpointVersion(8100, greater_equal<int>())) {
agent_data_report << AgentReportFieldWithLabel("isCheckpointVersionGER81", "true");

2
components/security_apps/orchestration/update_communication/fog_helper_open_source.cc
View File

@ -3,7 +3,7 @@
#include <string>
std::string
FogAuthenticator::getUserEdition() const
FogAuthenticator::getUserEdition()
{
return "community";
}

86
components/security_apps/orchestration/update_communication/hybrid_communication.cc
View File

@ -27,15 +27,19 @@ using namespace std;
USE_DEBUG_FLAG(D_ORCHESTRATOR);
#define TUNING_HOST_ENV_NAME "TUNING_HOST"
static const string defaultTuningHost = "appsec-tuning-svc";
static const string agent_resource_api = "/api/v2/agents/resources";
void
HybridCommunication::init()
{
dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component";
FogAuthenticator::init();
i_declarative_policy = Singleton::Consume<I_DeclarativePolicy>::from<DeclarativePolicyUtils>();
dbgTrace(D_ORCHESTRATOR) << "Initializing the Hybrid Communication Component";
auto env_tuning_host = getenv("TUNING_HOST");
tuning_host = env_tuning_host != nullptr ? env_tuning_host : "appsec-tuning-svc";
if (getConfigurationFlag("otp") != "") {
otp = getConfigurationFlag("otp");
} else {
@ -49,10 +53,9 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request)
string manifest_checksum = "";
dbgTrace(D_ORCHESTRATOR) << "Getting updates in Hybrid Communication";
if (access_token.ok()) {
static const string check_update_str = "/api/v2/agents/resources";
auto request_status = Singleton::Consume<I_Messaging>::by<HybridCommunication>()->sendSyncMessage(
HTTPMethod::POST,
check_update_str,
agent_resource_api,
request
);
@ -78,45 +81,7 @@ HybridCommunication::getUpdate(CheckUpdateRequest &request)
string policy_response = i_declarative_policy->getUpdate(request);
auto env = Singleton::Consume<I_EnvDetails>::by<HybridCommunication>()->getEnvType();
if (env == EnvType::K8S && !policy_response.empty()) {
dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host";
I_AgentDetails *agentDetails = Singleton::Consume<I_AgentDetails>::by<HybridCommunication>();
auto get_tuning_host = []()
{
static string tuning_host;
if (tuning_host != "") return tuning_host;
char* tuning_host_env = getenv(TUNING_HOST_ENV_NAME);
if (tuning_host_env != NULL) {
tuning_host = string(tuning_host_env);
return tuning_host;
}
dbgWarning(D_ORCHESTRATOR) << "tuning host is not set. using default";
tuning_host = defaultTuningHost;
return tuning_host;
};
MessageMetadata update_policy_crd_md(get_tuning_host(), 80);
update_policy_crd_md.insertHeader("X-Tenant-Id", agentDetails->getTenantId());
UpdatePolicyCrdObject policy_change_object(policy_response);
auto i_messaging = Singleton::Consume<I_Messaging>::by<HybridCommunication>();
bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
"/api/update-policy-crd",
policy_change_object,
MessageCategory::GENERIC,
update_policy_crd_md
);
if (!tuning_req_status) {
dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification";
} else {
dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification";
}
}
doLocalFogOperations(policy_response);
request = CheckUpdateRequest(manifest_checksum, policy_response, "", "", "", "");
@ -141,10 +106,9 @@ HybridCommunication::downloadAttributeFile(const GetResourceFile &resourse_file,
auto unpacked_access_token = access_token.unpack().getToken();
static const string file_attribute_str = "/api/v2/agents/resources/";
auto attribute_file = Singleton::Consume<I_Messaging>::by<HybridCommunication>()->downloadFile(
resourse_file.getRequestMethod(),
file_attribute_str + resourse_file.getFileName(),
agent_resource_api + '/' + resourse_file.getFileName(),
file_path
);
if (!attribute_file.ok()) {
@ -164,3 +128,33 @@ HybridCommunication::sendPolicyVersion(const string &policy_version, const strin
policy_version.empty();
return Maybe<void>();
}
void
HybridCommunication::doLocalFogOperations(const string &policy) const
{
if (policy.empty()) return;
if (Singleton::Consume<I_EnvDetails>::by<HybridCommunication>()->getEnvType() != EnvType::K8S) return;
dbgDebug(D_ORCHESTRATOR) << "Policy has changes, sending notification to tuning host";
MessageMetadata update_policy_crd_md(tuning_host, 80);
const auto &tenant_id = Singleton::Consume<I_AgentDetails>::by<HybridCommunication>()->getTenantId();
update_policy_crd_md.insertHeader("X-Tenant-Id", tenant_id);
UpdatePolicyCrdObject policy_change_object(policy);
auto i_messaging = Singleton::Consume<I_Messaging>::by<HybridCommunication>();
bool tuning_req_status = i_messaging->sendSyncMessageWithoutResponse(
HTTPMethod::POST,
"/api/update-policy-crd",
policy_change_object,
MessageCategory::GENERIC,
update_policy_crd_md
);
if (!tuning_req_status) {
dbgWarning(D_ORCHESTRATOR) << "Failed to send tuning notification";
} else {
dbgDebug(D_ORCHESTRATOR) << "Successfully sent tuning policy update notification";
}
}

1
components/security_apps/waap/waap_clib/CMakeLists.txt
View File

@ -85,6 +85,7 @@ add_library(waap_clib
ParserGql.cc
ParserPercentEncode.cc
ParserPairs.cc
Waf2Util2.cc
)
add_definitions("-Wno-unused-function")

10
components/security_apps/waap/waap_clib/DeepParser.cc
View File

@ -864,10 +864,6 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned(
return rc;
}
static bool err = false;
static const SingleRegex json_detector_re("\\A[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", err, "json_detector");
static const SingleRegex json_quoteless_detector_re("^[{\\[][[,0-9nul\\]]+", err, "json_quoteless_detector");
//intended to keep and process all types of leftovers detected as separate cases for parsing
int
DeepParser::createUrlParserForJson(
@ -1103,11 +1099,7 @@ DeepParser::createInternalParser(
} else {
dbgTrace(D_WAAP_DEEP_PARSER) << "attempt to find JSON by '{' or '['";
bool percent_encoded_doublequote_detected = cur_val.find("%22") != std::string::npos;
if (json_detector_re.hasMatch(cur_val)
&& (valueStats.hasDoubleQuote
|| json_quoteless_detector_re.hasMatch(cur_val)
|| percent_encoded_doublequote_detected)) {
// JSON value detected
if (Waap::Util::isValidJson(cur_val)) {
if (percent_encoded_doublequote_detected && !valueStats.hasDoubleQuote) {
// We have JSOn but it %-encoded, first start percent decoding for it. Very narrow case
dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse a JSON file from percent decoding";

7
components/security_apps/waap/waap_clib/ParserBase.cc
View File

@ -19,7 +19,8 @@ USE_DEBUG_FLAG(D_WAAP_PARSER);
// Max size for key and value that can be stored in memory (per thread)
#define MAX_KEY_SIZE 64*1024
#define MAX_VALUE_SIZE 64*1024
#define MAX_VALUE_SIZE 16*1024
#define MAX_PROCESSING_BUFFER_SIZE 64*1024
BufferedReceiver::BufferedReceiver(IParserReceiver &receiver, size_t parser_depth) :
m_receiver(receiver),
@ -69,14 +70,14 @@ int BufferedReceiver::onValue(const char *v, size_t v_len)
<< m_parser_depth;
while (v_len > 0) {
// Move data from buffer v to accumulated m_value string in an attempt to fill m_value to its max size
size_t bytesToFill = std::min(v_len, MAX_VALUE_SIZE - m_value.size());
size_t bytesToFill = std::min(v_len, MAX_PROCESSING_BUFFER_SIZE - m_value.size());
m_value += std::string(v, bytesToFill);
// Update v and v_len (input buffer) to reflect that we already consumed part (or all) of it
v += bytesToFill;
v_len -= bytesToFill;
// Only push full buffers to the m_receiver
if (m_value.size() == MAX_VALUE_SIZE) {
if (m_value.size() >= MAX_VALUE_SIZE) {
// The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag
dbgTrace(D_WAAP_PARSER)
<< " The first full-size buffer will be pushed with BUFFERED_RECEIVER_F_FIRST flag"

5
components/security_apps/waap/waap_clib/ParserJson.h
View File

@ -22,8 +22,9 @@
#include "yajl/yajl_parse.h"
#include "singleton.h"
#include "i_oa_schema_updater.h"
#define FIRST_JSON_BUFFER_SIZE 4 // must buffer at least 4 first bytes to allow unicode autodetection (BOM).
// must be at least 4 first bytes to allow unicode autodetection (BOM).
// BUT... reduced to 1 in order to allow better work of schema validation and API discovery
#define FIRST_JSON_BUFFER_SIZE 1
typedef size_t yajl_size_t;

3
components/security_apps/waap/waap_clib/Waf2Engine.cc
View File

@ -312,6 +312,7 @@ Waf2Transaction::Waf2Transaction() :
m_responseInjectReasons(),
m_index(-1),
m_triggerLog(),
is_schema_validation(false),
m_waf2TransactionFlags()
{}
@ -343,6 +344,7 @@ Waf2Transaction::Waf2Transaction(std::shared_ptr<WaapAssetState> pWaapAssetState
m_responseInjectReasons(),
m_index(-1),
m_triggerLog(),
is_schema_validation(false),
m_waf2TransactionFlags()
{}
@ -515,7 +517,6 @@ bool Waf2Transaction::checkIsScanningRequired()
m_siteConfig = &m_ngenAPIConfig;
auto rateLimitingPolicy = m_siteConfig ? m_siteConfig->get_RateLimitingPolicy() : NULL;
result |= m_siteConfig->get_WebAttackMitigation();
if(rateLimitingPolicy) {
result |= m_siteConfig->get_RateLimitingPolicy()->getRateLimitingEnforcementStatus();
}

1
components/security_apps/waap/waap_clib/Waf2Engine.h
View File

@ -345,6 +345,7 @@ private:
// Cached pointer to const triggerLog (hence mutable)
mutable std::shared_ptr<Waap::Trigger::Log> m_triggerLog;
bool is_schema_validation = false;
Waf2TransactionFlags m_waf2TransactionFlags;
};

13
components/security_apps/waap/waap_clib/Waf2Util.cc
View File

@ -2055,12 +2055,10 @@ string extractForwardedIp(const string &x_forwarded_hdr_val)
return forward_ip;
}
bool isUuid(const string& str) {
if (str.length() != 36) {
return false;
}
static bool err;
static const SingleRegex uuid_detector_re(
"[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}",
@ -2069,17 +2067,6 @@ bool isUuid(const string& str) {
);
// Check if the string matches the UUID format
return uuid_detector_re.hasMatch(str);
/*
boost::cmatch what;
try {
static const boost::regex uuidRegex("[0-9a-fA-F]{8}-([0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}");
// Check if the string matches the UUID format
return boost::regex_match(str.c_str(), what, uuidRegex);
} catch (std::runtime_error &e) {
dbgError(D_WAAP) << e.what();
}
return false;
*/
}
bool isIpAddress(const string &ip_address)

2
components/security_apps/waap/waap_clib/Waf2Util.h
View File

@ -876,6 +876,8 @@ namespace Util {
std::string::const_iterator e,
std::string &repl);
bool isValidJson(const std::string &input);
bool detectJSONasParameter(const std::string &s,
std::string &key,
std::string &value);

632
components/security_apps/waap/waap_clib/Waf2Util2.cc Normal file
View File

@ -0,0 +1,632 @@
#include "Waf2Util.h"
#include <string>
namespace Waap {
namespace Util {
using namespace std;
static const char *trueString = "true";
static const size_t trueStringLen = strlen(trueString);
static const char *falseString = "false";
static const size_t falseStringLen = strlen(falseString);
static const char *nullString = "null";
static const size_t nullStringLen = strlen(nullString);
static const char *quoteString = "%22";
static const size_t quoteStringLen = strlen(quoteString);
int
isAlignedPrefix(
const char *sample,
const size_t sample_len,
const char *buffer,
const size_t buffer_len)
{
size_t lookup_len = 0;
if (buffer_len < sample_len) {
lookup_len = buffer_len;
} else {
lookup_len = sample_len;
}
if (strncmp(sample, buffer, lookup_len) == 0) {
return lookup_len;
}
return -1;
}
int
isBoolean(
const char *buffer,
const size_t buffer_len)
{
int status;
status = isAlignedPrefix(trueString, trueStringLen, buffer, buffer_len);
if (status >= 0) {
return status;
}
status = isAlignedPrefix(falseString, falseStringLen, buffer, buffer_len);
if (status >= 0) {
return status;
}
status = isAlignedPrefix(nullString, nullStringLen, buffer, buffer_len);
if (status >= 0) {
return status;
}
return -1;
}
bool
isValidExponent(
const char * buffer,
const size_t buffer_len,
size_t *i)
{
if (buffer_len == *i + 1) {
return true; // e or E is the last char in buffer
}
if (*i + 1 < buffer_len && (isdigit(buffer[*i + 1]) || (buffer[*i + 1] == '+' || buffer[*i + 1] == '-'))) {
(*i) += 1;
if (isdigit(buffer[*i + 1])) {
return true;
}
} else {
return false;
}
return false;
}
bool
isObjectStart(const char c, int *object_count)
{
if (c == '{') {
(*object_count)++;
return true;
}
return false;
}
bool
isObjectEnd(const char c, int *object_count)
{
if (c == '}') {
(*object_count)--;
return true;
}
return false;
}
bool
isArrayStart(const char c, int *array_count)
{
if (c == '[') {
(*array_count)++;
return true;
}
return false;
}
bool
isArrayEnd(const char c, int *array_count)
{
if (c == ']') {
(*array_count)--;
return true;
}
return false;
}
bool
isValidJson(const std::string &input)
{
static const size_t MAX_JSON_INSPECT_SIZE = 16;
enum state
{
S_START, // 0
S_OBJECT_START, // 1
S_OBJECT_END, // 2
S_ARRAY_START, // 3
S_ARRAY_END, // 4
S_NUMBER, // 5
S_NUMBER_END, // 6
S_STRING_START, // 7
S_STRING_BODY, // 8
S_STRING_END, // 9
S_VARIABLE_START, // 10
S_VARIABLE_BODY, // 11
S_VARIABLE_END, // 12
S_COMMA, // 13
S_COLON, // 14
S_BOOLEAN, // 15
S_ERROR, // 16
S_END // 17
};
state m_state;
bool encoded = false;
size_t i = 0;
char c;
const char *buf = input.c_str();
size_t len = input.length();
int array_count = 0;
int object_count = 0;
int status;
if (len < 2) {
return false;
}
m_state = S_START;
while (i < len && i < MAX_JSON_INSPECT_SIZE) {
c = buf[i];
if (c == 0x0) { // UTF16 to UTF8 support
i++;
continue;
}
switch (m_state) {
case S_START:
if (isObjectStart(c, &object_count)) {
m_state = S_OBJECT_START;
break;
}
if (isArrayStart(c, &array_count)) {
m_state = S_ARRAY_START;
break;
}
m_state = S_ERROR;
break; // S_START
case S_OBJECT_START:
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
if (c == '\"') {
m_state = S_VARIABLE_START;
break;
}
if (isspace(c)) {
break;
}
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_VARIABLE_START;
encoded = true;
i += status - 1;
break;
}
m_state = S_ERROR;
break; // object_start
case S_ARRAY_START:
if (isObjectStart(c, &object_count)) {
m_state = S_OBJECT_START;
break;
}
if (isArrayStart(c, &array_count)) {
// keep state unchanged
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isdigit(c)) {
m_state = S_NUMBER;
break;
}
if (c == '-') {
if (i + 1 == len) { // End of buffer case
m_state = S_NUMBER;
break;
}
if (i + 1 < len && isdigit(buf[i + 1])) {
m_state = S_NUMBER;
i++;
break;
}
m_state = S_ERROR;
break;
}
if (isspace(c)) {
break;
}
if (c == '\"') {
m_state = S_STRING_START;
break;
}
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_STRING_START;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
status = isBoolean(buf + i, len - i);
if (status >= 0) {
m_state = S_BOOLEAN;
i += status - 1;
break;
}
m_state = S_ERROR;
break; // array_start
case S_OBJECT_END:
if (isspace(c)) {
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org
m_state = S_ARRAY_START;
break;
}
if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org
m_state = S_OBJECT_START;
break;
}
m_state = S_ERROR;
break; // S_OBJECT_END
case S_ARRAY_END:
if (isspace(c)) {
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
if (isArrayStart(c, &array_count)) { // nJSON support but contradicts to definition of json.org
m_state = S_ARRAY_START;
break;
}
if (isObjectStart(c, &object_count)) { // nJSON support but contradicts to definition of json.org
m_state = S_OBJECT_START;
break;
}
m_state = S_ERROR;
break; // S_ARRAY_END
case S_NUMBER:
if (isdigit(c)) {
break;
}
if (c == '.') {
if (i + 1 == len) { // End of buffer case
m_state = S_NUMBER;
break;
}
if (i + 1 < len && isdigit(buf[i + 1])) {
m_state = S_NUMBER;
i++;
break;
}
m_state = S_ERROR;
break;
}
if (c == 'e' || c == 'E') {
if (isValidExponent(buf, len, &i)) {
m_state = S_NUMBER;
break;
}
m_state = S_ERROR;
break;
}
if (isspace(c)) {
m_state = S_NUMBER_END;
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
m_state = S_ERROR;
break; // S_NUMBER
case S_NUMBER_END:
if (isspace(c)) {
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
m_state = S_ERROR;
break; // S_NUMBER_END
case S_STRING_START:
if (c == '\"') {
m_state = S_STRING_END;
break;
}
if (encoded) { // url_encoded quote
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_STRING_END;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
}
m_state = S_STRING_BODY;
break; // S_STRING_START
case S_STRING_BODY:
if (c == '\"') {
if (buf[i - 1] == '\\' && buf[i - 2] != '\\') {
m_state = S_STRING_BODY;
break;
} else {
m_state = S_STRING_END;
break;
}
}
if (encoded) { // url_encoded quote
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_STRING_END;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
}
m_state = S_STRING_BODY;
break; // S_STRING_BODY;
case S_STRING_END:
if (isspace(c)) {
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
if (c == ':') {
m_state = S_COLON;
break;
}
m_state = S_ERROR;
break; // s_sting_end
case S_VARIABLE_START:
if (c == '\"') {
m_state = S_VARIABLE_END;
break;
}
if (encoded) { // url_encoded quote
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_VARIABLE_END;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
}
m_state = S_VARIABLE_BODY;
break; // S_VARIABLE_START
case S_VARIABLE_BODY:
if (c == '\"') {
if (buf[i - 1] == '\\' && buf[i - 2] != '\\') {
m_state = S_VARIABLE_BODY;
break;
} else {
m_state = S_VARIABLE_END;
break;
}
}
if (encoded) { // url_encoded quote
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_VARIABLE_END;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
}
m_state = S_VARIABLE_BODY;
break; // S_VARIABLE_BODY
case S_VARIABLE_END:
if (isspace(c)) {
break;
}
if (c == ':') {
m_state = S_COLON;
break;
}
m_state = S_ERROR;
break; // S_VARIABLE_END
case S_COMMA:
if (isObjectStart(c, &object_count)) {
m_state = S_OBJECT_START;
break;
}
if (isArrayStart(c, &array_count)) {
m_state = S_ARRAY_START;
break;
}
if (isdigit(c)) {
m_state = S_NUMBER;
break;
}
if (c == '-') {
if (i + 1 == len) { // End of buffer case
m_state = S_NUMBER;
break;
}
if (i + 1 < len && isdigit(buf[i + 1])) {
m_state = S_NUMBER;
i++;
break;
}
m_state = S_ERROR;
break;
}
if (isspace(c)) {
break;
}
if (c == '\"') {
m_state = S_STRING_START;
break;
}
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_STRING_START;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
status = isBoolean(buf + i, len - i);
if (status >= 0) {
m_state = S_BOOLEAN;
i += status - 1;
break;
}
m_state = S_ERROR;
break; // S_COMMA
case S_COLON:
if (isObjectStart(c, &object_count)) {
m_state = S_OBJECT_START;
break;
}
if (isArrayStart(c, &array_count)) {
m_state = S_ARRAY_START;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isdigit(c)) {
m_state = S_NUMBER;
break;
}
if (c == '-') {
if (i + 1 == len) { // End of buffer case
m_state = S_NUMBER;
break;
}
if (i + 1 < len && isdigit(buf[i + 1])) {
m_state = S_NUMBER;
i++;
break;
}
m_state = S_ERROR;
break;
}
if (isspace(c)) {
break;
}
if (c == '\"') {
m_state = S_STRING_START;
break;
}
status = isAlignedPrefix(quoteString, quoteStringLen, buf + i, len - i);
if (status >= 0) {
m_state = S_STRING_START;
encoded = true;
i += status - 1;
break;
} else {
m_state = S_ERROR;
}
status = isBoolean(buf + i, len - i);
if (status >= 0) {
m_state = S_BOOLEAN;
i += status - 1;
break;
}
m_state = S_ERROR;
break; // S_COLON
case S_BOOLEAN:
if (isspace(c)) {
break;
}
if (c == ',') {
m_state = S_COMMA;
break;
}
if (isArrayEnd(c, &array_count)) {
m_state = S_ARRAY_END;
break;
}
if (isObjectEnd(c, &object_count)) {
m_state = S_OBJECT_END;
break;
}
m_state = S_ERROR;
break; // S_BOOLEAN
case S_ERROR: break;
case S_END: break;
}
if (m_state == S_ERROR) {
return false;
}
i++;
}
if (m_state != S_ERROR && array_count >= 0 && object_count >= 0)
return true;
return false;
}
} // namespace Util
} // namespace Waap

2
components/utils/CMakeLists.txt
View File

@ -1,4 +1,4 @@
#add_subdirectory(geo_location)
add_subdirectory(geo_location)
add_subdirectory(http_transaction_data)
add_subdirectory(ip_utilities)
add_subdirectory(keywords)

17
core/agent_core_utilities/agent_core_utilities.cc
View File

@ -20,6 +20,7 @@
#include <unistd.h>
#include <sys/syscall.h>
#include <dirent.h>
#include <fstream>
#include <sstream>
#include <algorithm>
@ -244,6 +245,22 @@ deleteDirectory(const string &path, bool delete_content)
return res;
}
bool
touchFile(const string &path)
{
dbgFlow(D_INFRA_UTILS) << "Trying to touch file, path: " << path;
ofstream t_file(path);
if (!t_file.is_open()) {
dbgDebug(D_INFRA_UTILS) << "Failed to touch file. Path: " << path;
return false;
}
t_file.close();
dbgTrace(D_INFRA_UTILS) << "Successfully touched file, path: " << path;
return true;
}
string
convertToHumanReadable(uint64_t size_in_bytes)
{

325
core/include/services_sdk/interfaces/i_intelligence_is_v2.h
View File

@ -17,17 +17,16 @@
#include <chrono>
#include <string>
#include "maybe_res.h"
#include "i_messaging.h"
#include "i_time_get.h"
#include "i_mainloop.h"
#include "intelligence_is_v2/asset_reply.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_is_v2/intelligence_types_v2.h"
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "config.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "maybe_res.h"
namespace Intelligence {
class Invalidation;
class Response;
} // namespace Intelligence
@ -40,315 +39,27 @@ public:
const std::function<void(const Intelligence::Invalidation &)> &callback
) = 0;
virtual void unregisterInvalidation(uint id) = 0;
virtual Maybe<Intelligence::Response>
getResponse(
const std::vector<QueryRequest> &query_requests,
bool is_pretty,
bool is_bulk
) const = 0;
virtual Maybe<Intelligence::Response> getResponse(const QueryRequest &query_request, bool is_pretty) const = 0;
template<typename Data>
Maybe<std::vector<AssetReply<Data>>>
queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true)
{
uint assets_limit = query_request.getAssetsLimit();
static const uint upper_assets_limit = 50;
if (assets_limit == 0 || assets_limit > upper_assets_limit) {
return genError("Assets limit must be in the range of [1, " + std::to_string(upper_assets_limit) + "]");
}
static const uint upper_confidence_limit = 1000;
bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
if (!min_conf_res) {
return genError(
"Minimum confidence value must be in the range of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
if (query_request.isPagingActivated() && query_request.isPagingFinished()) {
return genError("Paging is activated and already finished. No need for more queries.");
}
IntelligenceQuery<Data> intelligence_query(query_request, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/query";
bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, assets_limit);
if (!res) {
dbgTrace(D_INTELLIGENCE) << "Could not message fog, trying to get offline intelligence.";
Maybe<std::string> offline_res = getOfflineInfoString(query_request.getQuery());
if (!offline_res.ok()) {
dbgDebug(D_INTELLIGENCE) << "Offline intelligence error: " << offline_res.getErr();
return genError("Could not query intelligence");
}
if (!intelligence_query.loadJson(offline_res.unpack())) {
dbgWarning(D_INTELLIGENCE) << "Offline intelligence error: invalid JSON for requested asset";
return genError("Could not query intelligence");
}
}
if (ignore_in_progress && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return genError("Query intelligence response with InProgress status");
}
return intelligence_query.getData();
}
queryIntelligence(QueryRequest &query_request, bool ignore_in_progress = false, bool is_pretty = true);
template<typename Data>
Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true)
{
static const uint upper_assets_limit = 50;
static const uint upper_confidence_limit = 1000;
for (QueryRequest &query_request : query_requests) {
uint assets_limit = query_request.getAssetsLimit();
if (assets_limit == 0 || assets_limit > upper_assets_limit) {
dbgTrace(D_INTELLIGENCE)
<< "Assets limit for request is "
<< upper_assets_limit
<< ", requests assets: "
<< assets_limit;
return genError("Assets limit valid range is of [1, " + std::to_string(upper_assets_limit) + "]");
}
queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty = true);
bool min_conf_res = query_request.checkMinConfidence(upper_confidence_limit);
if (!min_conf_res) {
dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
return genError(
"Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
}
IntelligenceQuery<Data> intelligence_query(query_requests, is_pretty);
static const std::string query_uri = "/api/v2/intelligence/assets/queries";
dbgTrace(D_INTELLIGENCE) << "Sending intelligence bulk request with " << query_requests.size() << " items";
bool res = getIsOfflineOnly() ? false : sendQueryObject(intelligence_query, query_uri, upper_assets_limit);
if (!res) {
dbgTrace(D_INTELLIGENCE) << "Could not message fog, bulk request failed.";
return genError("Could not query intelligence");
}
return intelligence_query.getBulkData();
}
private:
template<typename Data>
bool
sendMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip,
uint server_port
) {
if (ip == "" && server_port == 0) {
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
intelligence_query,
MessageCategory::INTELLIGENCE
);
return req_status.ok();
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< ip
<< " port: "
<< server_port
<< " query_uri: "
<< query_uri;
MessageMetadata req_md(ip, server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
intelligence_query,
MessageCategory::INTELLIGENCE,
req_md
);
return req_status.ok();
}
template<typename Data>
bool
sendQueryMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip = "",
uint server_port = 0
) {
auto i_timer = getTimer();
auto i_mainloop = getMainloop();
uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
20,
"intelligence",
"request overall timeout"
);
uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
5,
"intelligence",
"request lap timeout"
);
std::chrono::seconds request_overall_timeout = std::chrono::seconds(request_overall_timeout_conf);
std::chrono::seconds request_lap_timeout = std::chrono::seconds(request_lap_timeout_conf);
std::chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
std::chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
std::chrono::seconds seconds_since_start = std::chrono::seconds(0);
std::chrono::seconds seconds_since_last_lap = std::chrono::seconds(0);
bool res = true;
while (res &&
intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS &&
seconds_since_start < request_overall_timeout &&
seconds_since_last_lap < request_lap_timeout
) {
res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
if (res && intelligence_query.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
i_mainloop->yield(true);
}
seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - send_request_start_time
);
seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - last_lap_time
);
last_lap_time = i_timer->getMonotonicTime();
}
return res;
}
template<typename Data>
bool
sendPagingQueryMessage(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
int assets_limit,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags,
const std::string &ip = "",
uint server_port = 0
) {
bool res= true;
res = sendMessage(intelligence_query, query_uri, i_message, conn_flags, ip, server_port);
if (intelligence_query.getResponseStatus() == ResponseStatus::DONE &&
intelligence_query.getResponseAssetCollectionsSize() < assets_limit
) {
intelligence_query.setRequestCursor(Intelligence_IS_V2::CursorState::DONE, "");
} else {
intelligence_query.setRequestCursor(
Intelligence_IS_V2::CursorState::IN_PROGRESS,
intelligence_query.getResponseCursorVal()
);
}
return res;
}
// LCOV_EXCL_START Reason: one templated instance is tested in intelligence ut. the rest are tested in system tests
template<typename Data>
bool
sendQueryObjectToLocalServer(
IntelligenceQuery<Data> &intelligence_query,
const std::string &query_uri,
const std::string &ip,
bool is_primary_port,
int assets_limit,
I_Messaging *i_message,
Flags<MessageConnectionConfig> conn_flags
) {
static const std::string primary_port_setting = "local intelligence server primary port";
static const std::string secondary_port_setting = "local intelligence server secondary port";
auto server_port = getSetting<uint>(
"intelligence",
is_primary_port ? primary_port_setting : secondary_port_setting
);
if (!server_port.ok()) return false;
conn_flags.reset();
if (intelligence_query.getPagingStatus().ok()) {
return sendPagingQueryMessage(
intelligence_query,
query_uri,
assets_limit,
i_message,
conn_flags,
ip,
*server_port
);
}
return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags, ip, *server_port);
}
// LCOV_EXCL_STOP
template<typename Data>
bool
sendQueryObject(IntelligenceQuery<Data> &intelligence_query, const std::string &query_uri, int assets_limit)
{
auto i_message = getMessaging();
Flags<MessageConnectionConfig> conn_flags;
bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
false;
crowdsec_enabled = getProfileAgentSettingWithDefault<bool>(
crowdsec_enabled,
"layer7AccessControl.crowdsec.enabled"
);
bool use_local_intelligence = getProfileAgentSettingWithDefault<bool>(
false,
"agent.config.useLocalIntelligence"
);
auto server_ip = getSetting<std::string>("intelligence", "local intelligence server ip");
if (server_ip.ok() && (use_local_intelligence || crowdsec_enabled)) {
if (sendQueryObjectToLocalServer(
intelligence_query,
query_uri,
*server_ip,
true,
assets_limit,
i_message,
conn_flags
)
) {
return true;
}
if (sendQueryObjectToLocalServer(
intelligence_query,
query_uri,
*server_ip,
false,
assets_limit,
i_message,
conn_flags
)
) {
return true;
protected:
virtual ~I_Intelligence_IS_V2() {}
};
}
if (intelligence_query.getPagingStatus().ok()) {
return sendPagingQueryMessage(intelligence_query, query_uri, assets_limit, i_message, conn_flags);
}
#include "intelligence_is_v2/intelligence_interface_impl.h"
return sendQueryMessage(intelligence_query, query_uri, i_message, conn_flags);
}
virtual I_Messaging * getMessaging() const = 0;
virtual I_TimeGet * getTimer() const = 0;
virtual I_MainLoop * getMainloop() const = 0;
virtual Maybe<std::string> getOfflineInfoString(const SerializableQueryFilter &query) const = 0;
virtual bool getIsOfflineOnly() const = 0;
};
#endif // __I_INTELLIGENCE_IS_V2_H__

99
core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2_impl.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_replay_impl.h
View File

@ -11,18 +11,16 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __QUERY_RESPONSE_V2_IMPL_H_
#define __QUERY_RESPONSE_V2_IMPL_H_
#ifndef __ASSET_REPLAY_IMPL_H__
#define __ASSET_REPLAY_IMPL_H__
#ifndef __QUERY_RESPONSE_V2_H__
#error intelligence_query_response_v2_impl.h should not be included directly!
#endif // __QUERY_RESPONSE_V2_H__
#ifndef __ASSET_REPLY_H__
#error asset_replay_impl.h should not be included directly!
#endif // __ASSET_REPLY_H__
#include "debug.h"
#include "customized_cereal_multimap.h"
#include "intelligence_types_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
void
AssetReply<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
@ -84,87 +82,72 @@ AssetReply<UserSerializableReplyAttr>::getData() const
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::loadFromJson(cereal::JSONInputArchive &ar)
template <typename Values>
bool
AssetReply<UserSerializableReplyAttr>::matchValues(const Values &values) const
{
std::string raw_data;
ar(
cereal::make_nvp("status", raw_data),
cereal::make_nvp("totalNumAssets", total_num_assets),
cereal::make_nvp("assetCollections", asset_collections)
);
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
if (source.template matchValues<Values>(values)) return true;
}
return false;
}
try {
ar(cereal::make_nvp("cursor", cursor));
} catch(...) {}
template <typename UserSerializableReplyAttr>
UserSerializableReplyAttr
AssetReply<UserSerializableReplyAttr>::mergeReplyData() const
{
UserSerializableReplyAttr reply_data;
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
UserSerializableReplyAttr data_by_source = source.mergeReplyData();
reply_data.insert(data_by_source);
}
return reply_data;
}
template<typename UserSerializableReplyAttr>
void
IntelligenceQueryResponseT<UserSerializableReplyAttr>::loadFromJson(const std::string &json_response)
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
serialize(in_ar);
}
template<typename UserSerializableReplyAttr>
template<class Archive>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::serialize(Archive &ar)
IntelligenceQueryResponseT<UserSerializableReplyAttr>::serialize(Archive &ar)
{
std::string raw_data;
ar(
cereal::make_nvp("status", raw_data),
cereal::make_nvp("totalNumAssets", total_num_assets),
cereal::make_nvp("assetCollections", asset_collections)
);
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
try {
ar(cereal::make_nvp("cursor", cursor));
IntelligenceQueryResponse::serialize(ar);
} catch(...) {}
}
template <typename UserSerializableReplyAttr>
Intelligence_IS_V2::ResponseStatus
IntelligenceQueryResponse<UserSerializableReplyAttr>::getResponseStatus() const
{
return status;
}
template <typename UserSerializableReplyAttr>
uint
IntelligenceQueryResponse<UserSerializableReplyAttr>::getAmountOfAssets() const
{
return total_num_assets;
}
template <typename UserSerializableReplyAttr>
const std::string &
IntelligenceQueryResponse<UserSerializableReplyAttr>::getCursor() const
{
return cursor;
}
template <typename UserSerializableReplyAttr>
int
IntelligenceQueryResponse<UserSerializableReplyAttr>::getAssetCollectionsSize() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::getAssetCollectionsSize() const
{
return asset_collections.size();
}
template <typename UserSerializableReplyAttr>
const std::vector<AssetReply<UserSerializableReplyAttr>> &
IntelligenceQueryResponse<UserSerializableReplyAttr>::getData() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::getData() const
{
return asset_collections;
}
template <typename UserSerializableReplyAttr>
bool
IntelligenceQueryResponse<UserSerializableReplyAttr>::isValidInBulk() const
IntelligenceQueryResponseT<UserSerializableReplyAttr>::isLast(uint asset_limit)
{
return !partial_fail_in_bulk;
return getResponseStatus() == ResponseStatus::DONE && getAssetCollectionsSize() < asset_limit;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQueryResponse<UserSerializableReplyAttr>::setFailInBulk()
{
partial_fail_in_bulk = true;
}
#endif // __QUERY_RESPONSE_V2_IMPL_H_
#endif // __ASSET_REPLAY_IMPL_H__

81
core/include/services_sdk/interfaces/intelligence_is_v2/query_response_v2.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_reply.h
View File

@ -11,25 +11,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __QUERY_RESPONSE_V2_H__
#define __QUERY_RESPONSE_V2_H__
#ifndef __ASSET_REPLY_H__
#define __ASSET_REPLY_H__
#include <sstream>
#include <vector>
#include <map>
#include <string>
#include <vector>
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "cereal/types/map.hpp"
#include "debug.h"
#include "maybe_res.h"
#include "customized_cereal_map.h"
#include "customized_cereal_multimap.h"
#include "asset_source.h"
#include "query_request_v2.h"
#include "intelligence_types_v2.h"
#include "asset_source_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
#include "maybe_res.h"
template <typename UserSerializableReplyAttr>
class AssetReply
@ -55,26 +47,10 @@ public:
const std::string & getAssetOrder() const { return asset_order; }
const std::string & getAssetKind() const { return asset_kind; }
UserSerializableReplyAttr
mergeReplyData() const
{
UserSerializableReplyAttr reply_data;
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
UserSerializableReplyAttr data_by_source = source.mergeReplyData();
reply_data.insert(data_by_source);
}
return reply_data;
}
UserSerializableReplyAttr mergeReplyData() const;
template <typename Values>
bool
matchValues(const Values &values) const
{
for (const SerializableAssetSource<UserSerializableReplyAttr> &source : sources) {
if (source.template matchValues<Values>(values)) return true;
}
return false;
}
bool matchValues(const Values &values) const;
private:
uint asset_schema_version = 0;
@ -94,33 +70,48 @@ private:
std::vector<SerializableAssetSource<UserSerializableReplyAttr>> sources;
};
template <typename UserSerializableReplyAttr>
class IntelligenceQueryResponse
{
public:
IntelligenceQueryResponse() {}
void loadFromJson(cereal::JSONInputArchive &ar);
void loadFromJson(const std::string &json_response);
template<class Archive>
void serialize(Archive &ar);
Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
uint getAmountOfAssets() const;
const std::string & getCursor() const;
int getAssetCollectionsSize() const;
const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
bool isValidInBulk() const;
void setFailInBulk();
Intelligence_IS_V2::ResponseStatus getResponseStatus() const { return status; }
const std::string & getCursor() const { return cursor; }
uint getAmountOfAssets() const { return total_num_assets; }
bool isValidInBulk() const { return !partial_fail_in_bulk; }
void setFailInBulk() { partial_fail_in_bulk = true; }
private:
Intelligence_IS_V2::ResponseStatus status = Intelligence_IS_V2::ResponseStatus::IN_PROGRESS;
uint total_num_assets = 0;
std::string cursor = "";
std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
bool partial_fail_in_bulk = false;
};
#include "query_response_v2_impl.h"
template <typename UserSerializableReplyAttr>
class IntelligenceQueryResponseT : public IntelligenceQueryResponse
{
public:
void loadFromJson(const std::string &json_response);
#endif // __QUERY_RESPONSE_V2_H__
template<class Archive>
void serialize(Archive &ar);
uint getAssetCollectionsSize() const;
bool isLast(uint asset_limit);
const std::vector<AssetReply<UserSerializableReplyAttr>> & getData() const;
private:
std::vector<AssetReply<UserSerializableReplyAttr>> asset_collections;
};
#include "asset_replay_impl.h"
#endif // __ASSET_REPLY_H__

38
core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2.h → core/include/services_sdk/interfaces/intelligence_is_v2/asset_source.h
View File

@ -11,14 +11,15 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ASSET_SOURCE_V2_H__
#define __ASSET_SOURCE_V2_H__
#ifndef __ASSET_SOURCE_H__
#define __ASSET_SOURCE_H__
#include <chrono>
#include <string>
#include <vector>
#include "debug.h"
#include "intelligence_types_v2.h"
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "customized_cereal_map.h"
#include "intelligence_types_v2.h"
template <typename UserSerializableReplyAttr>
class SerializableAssetSource
@ -26,7 +27,26 @@ class SerializableAssetSource
public:
SerializableAssetSource() {}
void load(cereal::JSONInputArchive &ar);
void load(cereal::JSONInputArchive &ar)
{
uint raw_seconds;
ar(
cereal::make_nvp("tenantId", tenant_id),
cereal::make_nvp("sourceId", source_id),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("ttl", raw_seconds),
cereal::make_nvp("expirationTime", expiration_time),
cereal::make_nvp("confidence", confidence)
);
ttl = std::chrono::seconds(raw_seconds);
UserSerializableReplyAttr raw_attribute;
try {
ar(cereal::make_nvp("attributes", raw_attribute));
attributes.clear();
attributes.push_back(raw_attribute);
} catch(const std::exception &e) {}
}
const std::string & getTenantId() const { return tenant_id; }
const std::string & getSourceId() const { return source_id; }
@ -66,6 +86,4 @@ private:
std::vector<UserSerializableReplyAttr> attributes;
};
#include "asset_source_v2_impl.h"
#endif //__ASSET_SOURCE_V2_H__
#endif // __ASSET_SOURCE_H__

47
core/include/services_sdk/interfaces/intelligence_is_v2/asset_source_v2_impl.h
View File

@ -1,47 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __ASSET_SOURCE_V2_IMPL_H__
#define __ASSET_SOURCE_V2_IMPL_H__
#ifndef __ASSET_SOURCE_V2_H__
#error intelligence_query_impl_8_0.h should not be included directly!
#endif //__ASSET_V2_SOURCE_H__
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
void
SerializableAssetSource<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
{
uint raw_seconds;
ar(
cereal::make_nvp("tenantId", tenant_id),
cereal::make_nvp("sourceId", source_id),
cereal::make_nvp("assetId", asset_id),
cereal::make_nvp("ttl", raw_seconds),
cereal::make_nvp("expirationTime", expiration_time),
cereal::make_nvp("confidence", confidence)
);
ttl = std::chrono::seconds(raw_seconds);
UserSerializableReplyAttr raw_attribute;
try {
ar(cereal::make_nvp("attributes", raw_attribute));
attributes.clear();
attributes.push_back(raw_attribute);
} catch(const std::exception &e) {}
}
#endif //__ASSET_SOURCE_V2_IMPL_H__

63
core/include/services_sdk/interfaces/intelligence_is_v2/bulk_query_response_v2.h
View File

@ -15,12 +15,12 @@
#define __BULK_QUERY_RESPONSE_V2_H__
#include <sstream>
#include <vector>
#include <string>
#include <vector>
#include "asset_reply.h"
#include "cereal/archives/json.hpp"
#include "cereal/types/vector.hpp"
#include "debug.h"
#include "intelligence_types_v2.h"
@ -29,15 +29,7 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
class BulkResponseError
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("statusCode", status_code),
cereal::make_nvp("message", message)
);
}
void serialize(cereal::JSONInputArchive &ar);
unsigned int getIndex() const { return index; }
int getStatusCode() const { return status_code; }
@ -49,46 +41,69 @@ private:
std::string message;
};
template <typename UserSerializableReplyAttr>
class ValidBulkQueryResponse
{
public:
void serialize(cereal::JSONInputArchive &ar);
unsigned int getIndex() const { return index; }
const IntelligenceQueryResponse & getResponse() const { return response; }
private:
unsigned int index;
IntelligenceQueryResponse response;
};
template <typename UserSerializableReplyAttr>
class ValidBulkQueryResponseT : public ValidBulkQueryResponse
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
try {
ValidBulkQueryResponse::serialize(ar);
} catch (...) {}
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("response", response)
);
}
unsigned int getIndex() const { return index; }
const IntelligenceQueryResponse<UserSerializableReplyAttr> & getResponse() const { return response; }
const IntelligenceQueryResponseT<UserSerializableReplyAttr> & getResponse() const { return response; }
private:
unsigned int index;
IntelligenceQueryResponse<UserSerializableReplyAttr> response;
IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
};
class IntelligenceQueryBulkResponse
{
public:
void serialize(cereal::JSONInputArchive &ar);
const std::vector<ValidBulkQueryResponse> & getValid() { return valid_responses; }
const std::vector<BulkResponseError> & getErrors() { return errors; }
private:
std::vector<ValidBulkQueryResponse> valid_responses;
std::vector<BulkResponseError> errors;
};
template <typename UserSerializableReplyAttr>
class IntelligenceQueryBulkResponse
class IntelligenceQueryBulkResponseT : public IntelligenceQueryBulkResponse
{
public:
void
serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("queriesResponse", valid_responses));
try {
ar(cereal::make_nvp("errors", errors));
IntelligenceQueryBulkResponse::serialize(ar);
} catch(...) {}
ar(cereal::make_nvp("queriesResponse", valid_responses));
}
const std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
const std::vector<BulkResponseError> & getErrors() { return errors; }
const std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> & getValid() { return valid_responses; }
private:
std::vector<ValidBulkQueryResponse<UserSerializableReplyAttr>> valid_responses;
std::vector<BulkResponseError> errors;
std::vector<ValidBulkQueryResponseT<UserSerializableReplyAttr>> valid_responses;
};
#endif // __BULK_QUERY_RESPONSE_V2_H__

50
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_interface_impl.h Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_INTERFACE_IMPL_H__
#define __INTELLIGENCE_INTERFACE_IMPL_H__
#ifndef __I_INTELLIGENCE_IS_V2_H__
#error intelligence_interface_impl.h should not be included directly!
#endif // __I_INTELLIGENCE_IS_V2_H__
template <typename Data>
Maybe<std::vector<AssetReply<Data>>>
I_Intelligence_IS_V2::queryIntelligence(QueryRequest &query_request, bool ignore_in_progress, bool is_pretty)
{
auto response = getResponse(query_request, is_pretty);
if (!response.ok()) return response.passErr();
auto serializable_response = response->getSerializableResponse<Data>();
if (!query_request.isPagingActivated()) return serializable_response.getData();
if (serializable_response.isLast(query_request.getAssetsLimit())) {
query_request.setCursor(Intelligence_IS_V2::CursorState::DONE, "");
} else {
query_request.setCursor(Intelligence_IS_V2::CursorState::IN_PROGRESS, response->getCursor());
if (ignore_in_progress) return genError("Query intelligence response with InProgress status");
}
return serializable_response.getData();
}
template<typename Data>
Maybe<std::vector<Maybe<std::vector<AssetReply<Data>>>>>
I_Intelligence_IS_V2::queryIntelligence(std::vector<QueryRequest> &query_requests, bool is_pretty)
{
auto res = getResponse(query_requests, is_pretty, true);
if (!res.ok()) return res.passErr();
return res->getBulkData<Data>();
}
#endif // __INTELLIGENCE_INTERFACE_IMPL_H__

77
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2.h
View File

@ -1,77 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_QUERY_V2_H__
#define __INTELLIGENCE_QUERY_V2_H__
#include<vector>
#include "cereal/archives/json.hpp"
#include "intelligence_types_v2.h"
#include "query_request_v2.h"
#include "query_response_v2.h"
#include "bulk_query_response_v2.h"
#include "rest.h"
template <typename UserSerializableReplyAttr>
class IntelligenceQuery
{
public:
IntelligenceQuery(QueryRequest &filter, bool is_pretty)
:
request(filter),
response(),
responses(),
is_bulk(false),
is_pretty(is_pretty)
{}
IntelligenceQuery(std::vector<QueryRequest> &filters, bool is_pretty)
:
requests(filters),
response(),
responses(),
is_bulk(true),
is_pretty(is_pretty)
{}
Maybe<std::string> genJson() const;
bool loadJson(const std::string &json);
void load(cereal::JSONInputArchive &ar);
void save(cereal::JSONOutputArchive &ar) const;
std::vector<AssetReply<UserSerializableReplyAttr>> getData();
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> getBulkData();
ResponseStatus getResponseStatus();
int getResponseAssetCollectionsSize() const { return response.getAssetCollectionsSize(); }
const std::string & getResponseCursorVal() const { return response.getCursor(); }
void activatePaging();
Maybe<Intelligence_IS_V2::CursorState> getPagingStatus();
void setRequestCursor(CursorState state, const std::string &value);
private:
static QueryRequest dummy_query_request;
static std::vector<QueryRequest> dummy_query_requests;
std::vector<QueryRequest> &requests = dummy_query_requests;
QueryRequest &request = dummy_query_request;
IntelligenceQueryResponse<UserSerializableReplyAttr> response;
std::vector<IntelligenceQueryResponse<UserSerializableReplyAttr>> responses;
bool is_bulk;
bool is_pretty;
};
#include "intelligence_query_v2_impl.h"
#endif // __INTELLIGENCE_QUERY_V2_H__

191
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_query_v2_impl.h
View File

@ -1,191 +0,0 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_QUERY_V2_IMPL_H_
#define __INTELLIGENCE_QUERY_V2_IMPL_H_
#ifndef __INTELLIGENCE_QUERY_V2_H__
#error intelligence_query_impl_v2.h should not be included directly!
#endif // __INTELLIGENCE_QUERY_V2_H__
#include <sstream>
#include "json_stream.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
template <typename UserSerializableReplyAttr>
QueryRequest IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_request = QueryRequest();
template <typename UserSerializableReplyAttr>
std::vector<QueryRequest> IntelligenceQuery<UserSerializableReplyAttr>::dummy_query_requests =
std::vector<QueryRequest>();
template <typename UserSerializableReplyAttr>
Maybe<std::string>
IntelligenceQuery<UserSerializableReplyAttr>::genJson() const
{
{
std::stringstream str_stream;
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(json_stream);
if (is_bulk) {
std::vector<BulkQueryRequest> bulk_requests;
int index = 0;
for (QueryRequest &request : requests) {
bulk_requests.push_back(BulkQueryRequest(request, index++));
}
out_ar(cereal::make_nvp("queries", bulk_requests));
} else {
request.saveToJson(out_ar);
}
}
return str_stream.str();
}
}
template <typename UserSerializableReplyAttr>
bool
IntelligenceQuery<UserSerializableReplyAttr>::loadJson(const std::string &json)
{
try {
std::stringstream in;
in.str(json);
try {
cereal::JSONInputArchive in_ar(in);
load(in_ar);
} catch (const Intelligence_IS_V2::IntelligenceException &e) {
dbgWarning(D_INTELLIGENCE) << "Failed to load query response. Error: " << e.what();
return false;
}
return true;
} catch (const std::exception &e) {
return false;
}
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::load(cereal::JSONInputArchive &ar)
{
if (is_bulk) {
IntelligenceQueryBulkResponse<UserSerializableReplyAttr> bulk_response;
bulk_response.serialize(ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
responses.clear();
responses.reserve(requests.size());
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << requests.size() << " items";
for (unsigned int query_idx = 0; query_idx < requests.size(); query_idx++) {
if (valid_response[valid_idx].getIndex() == query_idx) {
responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
responses.emplace_back();
responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
responses[query_idx].setFailInBulk();
}
}
} else {
response.loadFromJson(ar);
}
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::save(cereal::JSONOutputArchive &ar) const
{
if (!is_bulk) {
request.saveToJson(ar);
} else {
ar(cereal::make_nvp("queries", requests));
}
}
template <typename UserSerializableReplyAttr>
std::vector<AssetReply<UserSerializableReplyAttr>>
IntelligenceQuery<UserSerializableReplyAttr>::getData()
{
return response.getData();
}
template <typename UserSerializableReplyAttr>
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
IntelligenceQuery<UserSerializableReplyAttr>::getBulkData()
{
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
bulk_data.reserve(responses.size());
int index = 0;
for (const auto &res: responses) {
if (!res.isValidInBulk()) {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
bulk_data.push_back(genError("Received error for request in bulk"));
index++;
} else {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
bulk_data.push_back(res.getData());
index++;
}
}
return bulk_data;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::activatePaging()
{
request.setCursor(Intelligence_IS_V2::CursorState::START, "start");
}
template <typename UserSerializableReplyAttr>
Maybe<Intelligence_IS_V2::CursorState>
IntelligenceQuery<UserSerializableReplyAttr>::getPagingStatus()
{
if (is_bulk) return genError("Paging not activated in bulk mode");
if (!request.isPagingActivated()) return genError("Paging not activated");
return request.getCursorState();
}
template <typename UserSerializableReplyAttr>
ResponseStatus
IntelligenceQuery<UserSerializableReplyAttr>::getResponseStatus()
{
if (!is_bulk) return response.getResponseStatus();
if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
for (const auto &response_itr : responses) {
if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return ResponseStatus::IN_PROGRESS;
}
}
return ResponseStatus::DONE;
}
template <typename UserSerializableReplyAttr>
void
IntelligenceQuery<UserSerializableReplyAttr>::setRequestCursor(CursorState state, const std::string &value)
{
request.setCursor(state, value);
}
#endif //__INTELLIGENCE_QUERY_V2_IMPL_H_

111
core/include/services_sdk/interfaces/intelligence_is_v2/intelligence_response.h Normal file
View File

@ -0,0 +1,111 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_RESPONSE_H__
#define __INTELLIGENCE_RESPONSE_H__
#include <chrono>
#include <string>
#include <vector>
#include <maybe_res.h>
#include "asset_reply.h"
#include "bulk_query_response_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
namespace Intelligence
{
class Response
{
public:
Response() = default;
Response(const std::string &json_body, size_t size, bool is_bulk)
:
json_response(json_body), size(size), is_bulk(is_bulk)
{}
Maybe<void> load();
Intelligence_IS_V2::ResponseStatus getResponseStatus() const;
const std::string getCursor() const { return single_response.getCursor(); }
void setJsonResponse(const std::string &jsonResponse) { json_response = jsonResponse; }
template <typename UserSerializableReplyAttr>
IntelligenceQueryResponseT<UserSerializableReplyAttr> getSerializableResponse() const
{
IntelligenceQueryResponseT<UserSerializableReplyAttr> response;
response.loadFromJson(json_response);
return response;
}
template <typename UserSerializableReplyAttr>
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>>
getBulkData() const
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
IntelligenceQueryBulkResponseT<UserSerializableReplyAttr> bulk_response;
bulk_response.serialize(in_ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
std::vector<IntelligenceQueryResponseT<UserSerializableReplyAttr>> serializable_responses;
serializable_responses.reserve(size);
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
serializable_responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
serializable_responses.emplace_back();
serializable_responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
serializable_responses[query_idx].setFailInBulk();
}
}
std::vector<Maybe<std::vector<AssetReply<UserSerializableReplyAttr>>>> bulk_data;
bulk_data.reserve(serializable_responses.size());
int index = 0;
for (const auto &res: serializable_responses) {
if (!res.isValidInBulk()) {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk failed";
bulk_data.push_back(genError("Received error for request in bulk"));
index++;
} else {
dbgTrace(D_INTELLIGENCE) << "Request #" << index << " in bulk received valid response";
bulk_data.push_back(res.getData());
index++;
}
}
return bulk_data;
}
private:
std::string json_response;
std::vector<IntelligenceQueryResponse> responses;
IntelligenceQueryResponse single_response;
size_t size = 0;
bool is_bulk = false;
};
}
#endif // __INTELLIGENCE_RESPONSE_H__

3
core/include/services_sdk/interfaces/intelligence_is_v2/query_filter_v2.h
View File

@ -59,7 +59,6 @@ public:
Condition getConditionType() const { return condition_type; }
const std::string & getKey() const { return key; }
const ValueVariant & getValue() const { return value; }
private:
Condition condition_type = Condition::EQUALS;
@ -85,8 +84,6 @@ public:
const std::vector<SerializableQueryCondition> & getConditionOperands() const { return condition_operands; }
const std::vector<SerializableQueryFilter> & getQueriesOperands() const { return queries_operands; }
Maybe<SerializableQueryCondition::ValueVariant> getConditionValueByKey(const std::string &key) const;
bool empty() const { return condition_operands.empty() && queries_operands.empty(); }
SerializableQueryFilter operator &&(const SerializableQueryFilter &other_query);

21
core/include/services_sdk/interfaces/intelligence_is_v2/query_request_v2.h
View File

@ -97,11 +97,11 @@ public:
void setObjectType(const ObjectType &obj_type);
void setAssetsLimit(uint _assets_limit);
bool checkMinConfidence(uint upper_confidence_limit);
bool checkMinConfidence(uint upper_confidence_limit) const;
void activatePaging();
bool isPagingActivated();
Maybe<CursorState> getCursorState();
Maybe<CursorState> getCursorState() const;
bool isPagingFinished();
void setCursor(CursorState state, const std::string &value);
bool empty() const { return query.empty(); }
@ -124,21 +124,4 @@ private:
Maybe<std::string> convertObjectTypeToString() const;
};
class BulkQueryRequest
{
public:
BulkQueryRequest() {}
BulkQueryRequest(QueryRequest &request, int index);
void saveToJson(cereal::JSONOutputArchive &ar) const;
void save(cereal::JSONOutputArchive &ar) const;
QueryRequest getQueryRequest() const;
private:
QueryRequest request;
int index;
};
#endif // __QUERY_REQUEST_V2_H__

2
core/include/services_sdk/interfaces/intelligence_is_v2/requested_attributes_v2.h
View File

@ -51,7 +51,7 @@ public:
uint getSize() const { return requested_attributes.size(); }
bool isRequestedAttributesMapEmpty() const { return requested_attributes.empty(); }
bool checkMinConfidence(uint upper_confidence_limit);
bool checkMinConfidence(uint upper_confidence_limit) const;
private:
std::unordered_map<std::string, uint> requested_attributes;

38
core/include/services_sdk/interfaces/mock/mock_intelligence.h Normal file
View File

@ -0,0 +1,38 @@
#ifndef __MOCK_INTELLIGENCE_H__
#define __MOCK_INTELLIGENCE_H__
#include "i_intelligence_is_v2.h"
#include "cptest.h"
std::ostream &
operator<<(std::ostream &os, const Intelligence::Response &)
{
return os;
}
std::ostream &
operator<<(std::ostream &os, const Intelligence::Invalidation &)
{
return os;
}
class MockIntelligence : public Singleton::Provide<I_Intelligence_IS_V2>::From<MockProvider<I_Intelligence_IS_V2>>
{
public:
using InvalidationCb = std::function<void(const Intelligence::Invalidation &)>;
using Invalidation = Intelligence::Invalidation;
using Response = Intelligence::Response;
MOCK_CONST_METHOD1(sendInvalidation, bool(const Invalidation &invalidation));
MOCK_METHOD2(registerInvalidation, Maybe<uint>(const Invalidation &invalidation, const InvalidationCb &callback));
MOCK_METHOD1(unregisterInvalidation, void(uint id));
MOCK_CONST_METHOD3(
getResponse,
Maybe<Response>(const std::vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk)
);
MOCK_CONST_METHOD2(getResponse, Maybe<Response>(const QueryRequest &query_request, bool is_pretty));
MOCK_CONST_METHOD0(getIsOfflineOnly, bool(void));
MOCK_CONST_METHOD1(getOfflineInfoString, Maybe<std::string>(const SerializableQueryFilter &query));
};
#endif // __MOCK_INTELLIGENCE_H__

3
core/include/services_sdk/resources/intelligence_is_v2/data_string_v2.h
View File

@ -14,6 +14,9 @@
#ifndef __DATA_STRING_V2_H__
#define __DATA_STRING_V2_H__
#include <string>
#include "cereal/archives/json.hpp"
class DataString
{
public:

1
core/include/services_sdk/utilities/agent_core_utilities.h
View File

@ -32,6 +32,7 @@ Maybe<std::vector<std::string>> getDirectoryFiles(const std::string &path);
bool makeDir(const std::string &path, mode_t permission = S_IRWXU);
bool makeDirRecursive(const std::string &path, mode_t permission = S_IRWXU);
bool deleteDirectory(const std::string &path, bool delete_content = false);
bool touchFile(const std::string &path);
bool
copyFile(

6
core/intelligence_is_v2/CMakeLists.txt
View File

@ -1,7 +1,7 @@
include_directories(include)
add_library(
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc
intelligence_is_v2 intelligence_comp_v2.cc query_request_v2.cc intelligence_server.cc intelligence_response.cc asset_replay.cc bulk_query_response_v2.cc
intelligence_types_v2.cc query_filter_v2.cc requested_attributes_v2.cc query_types_v2.cc json_stream.cc invalidation.cc intelligence_request.cc
)
add_subdirectory(intelligence_is_v2_ut)
#add_subdirectory(intelligence_is_v2_ut)

38
core/intelligence_is_v2/asset_replay.cc Normal file
View File

@ -0,0 +1,38 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/asset_reply.h"
using namespace std;
void
IntelligenceQueryResponse::loadFromJson(const std::string &json_response)
{
std::stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
serialize(in_ar);
}
template <class Archive>
void
IntelligenceQueryResponse::serialize(Archive &ar)
{
std::string raw_data;
ar(cereal::make_nvp("status", raw_data), cereal::make_nvp("totalNumAssets", total_num_assets));
status = Intelligence_IS_V2::convertStringToResponseStatus(raw_data);
try {
ar(cereal::make_nvp("cursor", cursor));
} catch (...) {}
}

39
core/intelligence_is_v2/bulk_query_response_v2.cc Normal file
View File

@ -0,0 +1,39 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/bulk_query_response_v2.h"
using namespace std;
void
BulkResponseError::serialize(cereal::JSONInputArchive &ar)
{
ar(
cereal::make_nvp("index", index),
cereal::make_nvp("statusCode", status_code),
cereal::make_nvp("message", message)
);
}
void
ValidBulkQueryResponse::serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("index", index), cereal::make_nvp("response", response));
}
void
IntelligenceQueryBulkResponse::serialize(cereal::JSONInputArchive &ar)
{
ar(cereal::make_nvp("errors", errors));
ar(cereal::make_nvp("queriesResponse", valid_responses));
}

53
core/intelligence_is_v2/include/intelligence_request.h Normal file
View File

@ -0,0 +1,53 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_REQUEST_H__
#define __INTELLIGENCE_REQUEST_H__
#include "intelligence_is_v2/query_request_v2.h"
#include <vector>
#include "maybe_res.h"
namespace Intelligence {
class IntelligenceRequest : ClientRest
{
public:
IntelligenceRequest(const std::vector<QueryRequest> &queries, bool is_pretty, bool is_bulk)
:
queries(queries), is_pretty(is_pretty), is_bulk(is_bulk)
{}
Maybe<void> checkAssetsLimit() const;
Maybe<void> checkMinConfidence() const;
bool isPagingAllowed() const;
bool isPagingActivated() const;
Maybe<bool> isPagingFinished() const;
Maybe<Intelligence_IS_V2::CursorState> getPagingStatus() const;
bool loadJson(const std::string &json);
Maybe<std::string> genJson() const;
Maybe<std::string> getResponseFromFog() const;
size_t getSize() const { return queries.size(); }
bool isBulk() const { return is_bulk; }
private:
const std::vector<QueryRequest> &queries;
bool is_pretty = true;
bool is_bulk = false;
Maybe<std::string> response_from_fog = genError("Uninitialized");
};
}
#endif // __INTELLIGENCE_REQUEST_H__

50
core/intelligence_is_v2/include/intelligence_server.h Normal file
View File

@ -0,0 +1,50 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#ifndef __INTELLIGENCE_SERVER_H__
#define __INTELLIGENCE_SERVER_H__
#include <vector>
#include "maybe_res.h"
#include "i_messaging.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_request.h"
namespace Intelligence {
class Sender
{
public:
Sender(IntelligenceRequest request);
Maybe<Response> sendIntelligenceRequest();
private:
Maybe<Response> sendQueryObjectToLocalServer(bool is_primary_port);
Maybe<Response> sendQueryMessage();
Maybe<Response> sendMessage();
Maybe<Response> createResponse();
IntelligenceRequest request;
Flags<MessageConnectionConfig> conn_flags;
bool is_local_intelligence;
Maybe<std::string> server_ip = genError("No server ip set");
Maybe<unsigned int> server_port = genError("No port is set");
I_Messaging * i_message = nullptr;
I_TimeGet * i_timer = nullptr;
I_MainLoop * i_mainloop = nullptr;
};
}
#endif // __INTELLIGENCE_SERVER_H__

114
core/intelligence_is_v2/intelligence_comp_v2.cc
View File

@ -17,9 +17,10 @@
#include "cache.h"
#include "config.h"
#include "table.h"
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_invalidation.h"
#include "intelligence_is_v2/intelligence_response.h"
#include "intelligence_request.h"
#include "intelligence_server.h"
using namespace std;
using namespace chrono;
@ -221,50 +222,10 @@ class IntelligenceComponentV2::Impl
Singleton::Provide<I_Intelligence_IS_V2>::From<IntelligenceComponentV2>
{
public:
class OfflineIntelligeceHandler
{
public:
void
init()
{
filesystem_prefix = getFilesystemPathConfig();
dbgTrace(D_INTELLIGENCE) << "OfflineIntelligeceHandler init. file systen prefix: " << filesystem_prefix;
offline_intelligence_path = getConfigurationWithDefault<string>(
filesystem_prefix + "/conf/offline/intelligence",
"intelligence",
"offline intelligence path"
);
}
Maybe<string>
getValueByIdentifier(const string &identifier) const
{
string asset_file_path = offline_intelligence_path + "/" + identifier;
ifstream asset_info(asset_file_path);
if (!asset_info.is_open()) {
return genError("Could not open file: " + asset_file_path);
}
stringstream info_txt;
info_txt << asset_info.rdbuf();
asset_info.close();
return info_txt.str();
}
private:
string filesystem_prefix = "";
string offline_intelligence_path = "";
};
void
init()
{
offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
registerConfigLoadCb([&]() {
offline_mode_only = getConfigurationWithDefault<bool>(false, "intelligence", "offline intelligence only");
});
offline_intelligence.init();
message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
@ -283,7 +244,6 @@ public:
bool
sendInvalidation(const Invalidation &invalidation) const override
{
if (offline_mode_only) return false;
return hasLocalIntelligence() ? sendLocalInvalidation(invalidation) : sendGlobalInvalidation(invalidation);
}
@ -301,48 +261,28 @@ public:
invalidations.erase(id);
}
I_Messaging *
getMessaging() const override
Maybe<Response>
getResponse(const vector<QueryRequest> &query_requests, bool is_pretty, bool is_bulk) const override
{
return message != NULL ? message : Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
IntelligenceRequest intelligence_req(query_requests, is_pretty, is_bulk);
if (!intelligence_req.checkAssetsLimit().ok()) return intelligence_req.checkAssetsLimit().passErr();
if (!intelligence_req.checkMinConfidence().ok()) return intelligence_req.checkMinConfidence().passErr();
if (intelligence_req.isPagingActivated()) {
auto is_paging_finished = intelligence_req.isPagingFinished();
if (is_paging_finished.ok() && *is_paging_finished) {
return genError("Paging is activated and already finished. No need for more queries.");
}
}
Sender intelligence_server(intelligence_req);
auto response = intelligence_server.sendIntelligenceRequest();
return response;
}
I_TimeGet *
getTimer() const override
Maybe<Intelligence::Response>
getResponse(const QueryRequest &query_request, bool is_pretty) const override
{
return timer != NULL ? timer : Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
}
I_MainLoop *
getMainloop() const override
{
return mainloop != NULL ? mainloop : Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
}
Maybe<string>
getOfflineInfoString(const SerializableQueryFilter &query) const override
{
string ip_attr_key = "mainAttributes.ip";
auto valueVariant = query.getConditionValueByKey(ip_attr_key);
if (!valueVariant.ok()) {
return genError("could not find IP main attribute in the given query.");
}
const SerializableQueryCondition::ValueVariant& value = valueVariant.unpack();
if (const string* identifier_value = boost::get<string>(&value)) {
if (*identifier_value == "") {
return genError("Could not find IP main attribute in the given query.");
}
return offline_intelligence.getValueByIdentifier(*identifier_value);
}
return genError("Value is not of type string.");
}
bool
getIsOfflineOnly() const override
{
return offline_mode_only;
vector<QueryRequest> queries = {query_request};
return getResponse(queries, is_pretty, false);
}
private:
@ -362,7 +302,7 @@ private:
bool
sendLocalInvalidationImpl(const Invalidation &invalidation) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
auto server = getSetting<string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
@ -435,8 +375,6 @@ private:
bool
sendRegistration(const Invalidation &invalidation) const
{
if (offline_mode_only) return false;
InvalidationRegistration registration;
registration.addInvalidation(invalidation);
@ -446,7 +384,7 @@ private:
bool
sendLocalRegistrationImpl(const InvalidationRegistration::RestCall &registration) const
{
auto server = getSetting<std::string>("intelligence", "local intelligence server ip");
auto server = getSetting<string>("intelligence", "local intelligence server ip");
if (!server.ok()) {
dbgWarning(D_INTELLIGENCE) << "Local intelligence server not configured";
return false;
@ -485,13 +423,11 @@ private:
void
sendReccurringInvalidationRegistration() const
{
if (offline_mode_only || !hasLocalIntelligence() || invalidations.empty()) return;
if (!hasLocalIntelligence() || invalidations.empty()) return;
sendLocalRegistrationImpl(invalidations.getRegistration());
}
OfflineIntelligeceHandler offline_intelligence;
bool offline_mode_only = false;
InvalidationCallBack invalidations;
I_Messaging *message = nullptr;
I_TimeGet *timer = nullptr;
@ -511,8 +447,6 @@ void IntelligenceComponentV2::init() { pimpl->init(); }
void
IntelligenceComponentV2::preload()
{
registerExpectedConfiguration<string>("intelligence", "offline intelligence path");
registerExpectedConfiguration<bool>("intelligence", "offline intelligence only");
registerExpectedConfiguration<uint>("intelligence", "maximum request overall time");
registerExpectedConfiguration<uint>("intelligence", "maximum request lap time");
registerExpectedSetting<string>("intelligence", "local intelligence server ip");

405
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_comp_v2_ut.cc
View File

@ -1,15 +1,15 @@
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "read_attribute_v2.h"
#include "cptest.h"
#include "singleton.h"
#include "config.h"
#include "config_component.h"
#include "mock/mock_messaging.h"
#include "cptest.h"
#include "mock/mock_intelligence.h"
#include "mock/mock_mainloop.h"
#include "mock/mock_time_get.h"
#include "mock/mock_messaging.h"
#include "mock/mock_rest_api.h"
#include "mock/mock_time_get.h"
#include "read_attribute_v2.h"
#include "singleton.h"
using namespace std;
using namespace testing;
@ -49,9 +49,7 @@ public:
mockRestCall(_, "new-invalidation/source/invalidation", _)
).WillRepeatedly(Return(true));
string offline_intel_path = cptestFnameInExeDir("offline_intelligence_files_v2");
setConfiguration<string>(offline_intel_path, string("intelligence"), string("offline intelligence path"));
conf.preload();
intelligence.preload();
intelligence.init();
}
@ -72,6 +70,31 @@ public:
I_MainLoop::Routine routine;
};
class IntelligenceComponentMockTest : public Test, Singleton::Consume<I_Intelligence_IS_V2>
{
public:
IntelligenceComponentMockTest()
{
debug_output.clear();
Debug::setNewDefaultStdout(&debug_output);
Debug::setUnitTestFlag(D_METRICS, Debug::DebugLevel::TRACE);
Debug::setUnitTestFlag(D_INTELLIGENCE, Debug::DebugLevel::TRACE);
setConfiguration<bool>(false, string("metric"), string("fogMetricSendEnable"));
conf.preload();
}
~IntelligenceComponentMockTest()
{
Debug::setNewDefaultStdout(&cout);
}
::Environment env;
ConfigComponent conf;
stringstream debug_output;
StrictMock<MockIntelligence> intelligence_mock;
};
class Profile
{
public:
@ -97,6 +120,252 @@ private:
DataString phase;
};
TEST_F(IntelligenceComponentMockTest, getResponseErrorTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
Maybe<Intelligence::Response> res_error = genError("Test error");
EXPECT_CALL(intelligence_mock, getResponse(_, _)
).WillOnce(Return(res_error));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_FALSE(maybe_ans.ok());
}
TEST_F(IntelligenceComponentMockTest, getResponseTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
string response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": \"1.1.1.1\",\n"
" \"phase\": \"testing\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"testing\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
"}\n"
);
Intelligence::Response response(response_str, 1, false);
EXPECT_CALL(intelligence_mock, getResponse(_, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
auto vec = maybe_ans.unpack();
EXPECT_EQ(vec.size(), 1u);
auto iter = vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
}
TEST_F(IntelligenceComponentMockTest, bulkOnlineIntelligenceMockTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentMockTest>();
vector<QueryRequest> requests;
requests.emplace_back(Condition::EQUALS, "category", "whatever", true);
requests.emplace_back(Condition::EQUALS, "category", "cloud", true);
requests.emplace_back(Condition::EQUALS, "category", "nothing", true);
requests.emplace_back(Condition::EQUALS, "category", "iot", true);
string response_str(
"{\n"
" \"errors\": [\n"
" {\n"
" \"index\": 0,\n"
" \"statusCode\": 400,\n"
" \"message\": \"Bad request. Error: Invalid cursor\"\n"
" },"
" {\n"
" \"index\": 2,\n"
" \"statusCode\": 405,\n"
" \"message\": \"Bad request. Error: Something else\"\n"
" }"
" ],\n" // errors
" \"queriesResponse\": [\n"
" {\n"
" \"index\": 1,\n"
" \"response\": {\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-ip\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"some-group-id\",\n"
" \"name\": \"[1.1.1.1]\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"ip\",\n"
" \"group\": \"\",\n"
" \"order\": \"\",\n"
" \"kind\": \"\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": [\n"
" \"1.1.1.1\",\n"
" \"2.2.2.2\"\n"
" ],\n"
" \"phase\": \"testing\"\n"
" },\n" // mainAttributes
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"color\": \"red\",\n"
" \"user\": \"Omry\",\n"
" \"phase\": \"testing\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" },\n" // source 1
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
" \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 600,\n"
" \"attributes\": {\n"
" \"color\": \"white\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n" // source 2
" ]\n" // sources
" }\n" // asset 1
" ],\n" // asset collection
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
" }\n" // response
" },\n" // queryresponse 1
" {\n"
" \"index\": 3,\n"
" \"response\": {\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-ip\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"some-group-id\",\n"
" \"name\": \"[2.2.2.2]\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"iot\",\n"
" \"family\": \"ip\",\n"
" \"group\": \"\",\n"
" \"order\": \"\",\n"
" \"kind\": \"\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": [\n"
" \"1.1.1.1\",\n"
" \"2.2.2.2\"\n"
" ],\n"
" \"phase\": \"testing\"\n"
" },\n" // mainAttributes
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"color\": \"red\",\n"
" \"user\": \"Omry2\",\n"
" \"phase\": \"testing2\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" },\n" // source 1
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229bb11\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7bbb11\",\n"
" \"assetId\": \"cb068860528cb6bfb000cc35e79f11aeefed2\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 600,\n"
" \"attributes\": {\n"
" \"color\": \"white\",\n"
" \"user\": \"Max\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n" // source 2
" ]\n" // sources
" }\n" // asset 1
" ],\n" // asset collection
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
" }\n" // response
" }\n" // queryresponse 1
" ]\n" // queryresponses
"}\n"
);
Intelligence::Response response(response_str, 4, true);
EXPECT_CALL(intelligence_mock, getResponse(_, _, _)
).WillOnce(Return(response));
auto maybe_ans = intell->queryIntelligence<Profile>(requests);
EXPECT_TRUE(maybe_ans.ok());
auto vec = maybe_ans.unpack();
EXPECT_EQ(vec.size(), 4u);
EXPECT_FALSE(vec[0].ok());
EXPECT_TRUE(vec[1].ok());
EXPECT_FALSE(vec[2].ok());
EXPECT_TRUE(vec[3].ok());
auto assets1_vec = vec[1].unpack();
EXPECT_EQ(assets1_vec.size(), 1u);
auto iter = assets1_vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
auto assets3_vec = vec[3].unpack();
EXPECT_EQ(assets1_vec.size(), 1u);
iter = assets3_vec.begin();
EXPECT_EQ(iter->getData().begin()->getUser().toString(), "Omry2");
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing2");
}
TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@ -154,6 +423,75 @@ TEST_F(IntelligenceComponentTestV2, fakeOnlineIntelligenceTest)
EXPECT_EQ(iter->getData().begin()->getPhase().toString(), "testing");
}
TEST_F(IntelligenceComponentTestV2, fakeLocalIntelligenceTest)
{
stringstream configuration;
configuration << "{";
configuration << " \"agentSettings\":[";
configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}";
configuration << " ],";
configuration << " \"intelligence\":{";
configuration << " \"local intelligence server ip\":\"127.0.0.1\",";
configuration << " \"local intelligence server primary port\":9090";
configuration << " }";
configuration << "}";
Singleton::Consume<Config::I_Config>::from(conf)->loadConfiguration(configuration);
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "category", "cloud", true);
string response_str(
"{\n"
" \"assetCollections\": [\n"
" {\n"
" \"schemaVersion\": 1,\n"
" \"assetType\": \"workload-cloud-fake-online-test\",\n"
" \"assetTypeSchemaVersion\": 1,\n"
" \"permissionType\": \"tenant\",\n"
" \"permissionGroupId\": \"fake-online-test-group\",\n"
" \"name\": \"fake-online-test-asset\",\n"
" \"class\": \"workload\",\n"
" \"category\": \"cloud\",\n"
" \"family\": \"fake-online-test\",\n"
" \"mainAttributes\": {\n"
" \"ipv4Addresses\": \"1.1.1.1\",\n"
" \"phase\": \"testing\"\n"
" },\n"
" \"sources\": [\n"
" {\n"
" \"tenantId\": \"175bb55c-e36f-4ac5-a7b1-7afa1229aa00\",\n"
" \"sourceId\": \"54d7de10-7b2e-4505-955b-cc2c2c7aaa00\",\n"
" \"assetId\": \"50255c3172b4fb7fda93025f0bfaa7abefd1\",\n"
" \"ttl\": 120,\n"
" \"expirationTime\": \"2020-07-29T11:21:12.253Z\",\n"
" \"confidence\": 500,\n"
" \"attributes\": {\n"
" \"phase\": \"testing\",\n"
" \"user\": \"Omry\",\n"
" \"owners\": { \"names\": [ { \"name1\": \"Bob\", \"name2\": \"Alice\" } ] }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" ],\n"
" \"status\": \"done\",\n"
" \"totalNumAssets\": 2,\n"
" \"cursor\": \"start\"\n"
"}\n"
);
MessageMetadata md;
EXPECT_CALL(messaging_mock, sendSyncMessage(HTTPMethod::POST, _, _, MessageCategory::INTELLIGENCE, _)
).WillOnce(DoAll(SaveArg<4>(&md), Return(HTTPResponse(HTTPStatusCode::HTTP_OK, response_str))));
auto maybe_ans = intell->queryIntelligence<Profile>(request);
EXPECT_TRUE(maybe_ans.ok());
EXPECT_EQ(md.getHostName(), "127.0.0.1");
}
TEST_F(IntelligenceComponentTestV2, multiAssetsIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
@ -626,6 +964,7 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
).WillOnce(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, paging_done_response_str)));
auto maybe_ans3 = intell->queryIntelligence<Profile>(request);
if (!maybe_ans3.ok()) cout << maybe_ans3.getErr() + "\n";
EXPECT_TRUE(maybe_ans3.ok());
auto vec3 = maybe_ans3.unpack();
EXPECT_EQ(vec3.size(), 1);
@ -636,54 +975,6 @@ TEST_F(IntelligenceComponentTestV2, pagingQueryTest)
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
}
TEST_F(IntelligenceComponentTestV2, offlineIntelligenceTest)
{
setConfiguration<bool>(true, string("intelligence"), string("offline intelligence only"));
intelligence.init();
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();
QueryRequest request(Condition::EQUALS, "ip", "1.2.3.4", true);
auto maybe_ans = intell->queryIntelligence<Profile>(request);
ASSERT_TRUE(maybe_ans.ok());
vector<AssetReply<Profile>> vec = maybe_ans.unpack();
vector<AssetReply<Profile>>::iterator assets_iter = vec.begin();
EXPECT_EQ(assets_iter->getAssetSchemaVersion(), 1);
EXPECT_EQ(assets_iter->getAssetType(), "workload-cloud-ip");
EXPECT_EQ(assets_iter->getAssetTypeSchemaVersion(), 1);
EXPECT_EQ(assets_iter->getAssetPermissionGroupId(), "offline-group-id");
EXPECT_EQ(assets_iter->getAssetName(), "offline-asset");
EXPECT_EQ(assets_iter->getAssetClass(), "workload");
EXPECT_EQ(assets_iter->getAssetCategory(), "cloud");
EXPECT_EQ(assets_iter->getAssetFamily(), "offline family");
EXPECT_EQ(assets_iter->getAssetGroup(), "offline testing");
EXPECT_EQ(assets_iter->getAssetOrder(), "");
EXPECT_EQ(assets_iter->getAssetKind(), "");
map<string, vector<string>> attributes_map = assets_iter->getMainAttributes();
EXPECT_EQ(attributes_map["ip"].front(), "1.2.3.4");
vector<SerializableAssetSource<Profile>>::const_iterator sources_iter = assets_iter->getSources().begin();
EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229aa00");
EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7aaa00");
EXPECT_EQ(sources_iter->getAssetId(), "50255c3172b4fb7fda93025f0bfaa7abefd1");
EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
EXPECT_EQ(sources_iter->getConfidence(), 700);
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Omry");
EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
sources_iter++;
EXPECT_EQ(sources_iter->getTenantId(), "175bb55c-e36f-4ac5-a7b1-7afa1229bb11");
EXPECT_EQ(sources_iter->getSourceId(), "54d7de10-7b2e-4505-955b-cc2c2c7bbb11");
EXPECT_EQ(sources_iter->getAssetId(), "cb068860528cb6bfb000cc35e79f11aeefed2");
EXPECT_EQ(sources_iter->getTTL(), chrono::seconds(120));
EXPECT_EQ(sources_iter->getExpirationTime(), "2010-05-15T21:50:12.253Z");
EXPECT_EQ(sources_iter->getConfidence(), 600);
EXPECT_EQ(sources_iter->getAttributes().begin()->getUser().toString(), "Max");
EXPECT_EQ(sources_iter->getAttributes().begin()->getPhase().toString(), "offline test");
}
TEST_F(IntelligenceComponentTestV2, bulkOnlineIntelligenceTest)
{
I_Intelligence_IS_V2 *intell = Singleton::Consume<I_Intelligence_IS_V2>::by<IntelligenceComponentTestV2>();

17
core/intelligence_is_v2/intelligence_is_v2_ut/intelligence_query_v2_ut.cc
View File

@ -11,7 +11,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/intelligence_query_v2.h"
#include "intelligence_request.h"
#include "intelligence_is_v2/query_request_v2.h"
#include "cptest.h"
@ -22,7 +23,8 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
IntelligenceQuery<int> query(request, true);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, true, false);
std::string expected = "{\n"
" \"limit\": 20,\n"
@ -39,7 +41,9 @@ TEST(IntelligenceQueryTestV2, genJsonPrettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
QueryRequest request(Condition::EQUALS, "phase", "testing", true);
IntelligenceQuery<int> query(request, false);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false);
std::string expected = "{"
"\"limit\":20,"
"\"fullResponse\":true,"
@ -54,7 +58,8 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequest) {
TEST(IntelligenceQueryTestV2, genJsonUnprettySingleRequestSpaces) {
QueryRequest request(Condition::EQUALS, "ph ase", "te sti\" n g\\", true);
IntelligenceQuery<int> query(request, false);
vector<QueryRequest> requests = {request};
Intelligence::IntelligenceRequest query(requests, false, false);
std::string expected = "{"
"\"limit\":20,"
"\"fullResponse\":true,"
@ -71,7 +76,7 @@ TEST(IntelligenceQueryTestV2, genJsonPrettyBulkRequests) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
IntelligenceQuery<int> query(requests, true);
Intelligence::IntelligenceRequest query(requests, true, true);
std::string expected = "{\n"
" \"queries\": [\n"
@ -109,7 +114,7 @@ TEST(IntelligenceQueryTestV2, genJsonUnprettyBulkRequest) {
QueryRequest request1(Condition::EQUALS, "phase", "testing", true);
QueryRequest request2(Condition::EQUALS, "height", "testing", 25);
std::vector<QueryRequest> requests = {request1, request2};
IntelligenceQuery<int> query(requests, false);
Intelligence::IntelligenceRequest query(requests, false, true);
std::string expected = "{"
"\"queries\":[{"

9
core/intelligence_is_v2/intelligence_is_v2_ut/query_request_v2_ut.cc
View File

@ -172,6 +172,15 @@ TEST(QueryRequestTestV2, AttributesTest)
EXPECT_EQ(out.str(), output_json);
}
TEST(QueryRequestTestV2, AttributeConditionExceptionTest)
{
try {
Intelligence_IS_V2::createAttributeString("jey", (Intelligence_IS_V2::AttributeKeyType)5);
} catch (const IntelligenceException &e) {
EXPECT_THAT(string(e.what()), HasSubstr("Received illegal Attribute Type."));
}
}
TEST(QueryRequestTestV2, AndQueryTest)
{
QueryRequest request1(Condition::EQUALS, "phase", "testing1", true);

42
core/intelligence_is_v2/intelligence_is_v2_ut/query_response_v2_ut.cc
View File

@ -1,5 +1,3 @@
#include "intelligence_is_v2/query_response_v2.h"
#include "intelligence_is_v2/asset_source_v2.h"
#include "intelligence_is_v2/intelligence_types_v2.h"
#include "intelligence_is_v2/data_string_v2.h"
@ -31,12 +29,12 @@ public:
try{
ReadAttribute<DataString>("color", data).serialize(ar);
} catch (exception &e) {
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color";
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: color. Error:" + string(e.what());
}
try {
ReadAttribute<DataString>("user", data1).serialize(ar);
} catch (const exception &e) {
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user";
dbgError(D_INTELLIGENCE) << "Requested attribute was not found: user. Error:" + string(e.what());
}
}
@ -47,9 +45,6 @@ private:
TEST(QueryResponseTestV2, ReadAttributeTest)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
ReadAttribute<DataString> obj("user", data);
@ -70,9 +65,6 @@ TEST(QueryResponseTestV2, ReadAttributeTest)
TEST(QueryResponseTestV2, stringData1Test)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
stringData1 obj;
string data_str(
@ -95,11 +87,9 @@ TEST(QueryResponseTestV2, stringData1Test)
TEST(QueryResponseTestV2, QueryResponseTestV2)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
IntelligenceQueryResponse obj2;
string data_str(
"{\n"
" \"assetCollections\": [\n"
@ -161,12 +151,14 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
stringstream ss(data_str);
{
cereal::JSONInputArchive ar(ss);
obj.loadFromJson(ar);
obj.loadFromJson(ss.str());
obj2.loadFromJson(ss.str());
}
EXPECT_EQ(obj.getAmountOfAssets(), 2);
EXPECT_EQ(obj2.getAmountOfAssets(), 2);
EXPECT_EQ(obj.getResponseStatus(), ResponseStatus::DONE);
EXPECT_EQ(obj2.getResponseStatus(), ResponseStatus::DONE);
EXPECT_EQ(obj.getData().begin()->getAssetSchemaVersion(), 1);
EXPECT_EQ(obj.getData().begin()->getAssetType(), "workload-cloud-ip");
EXPECT_EQ(obj.getData().begin()->getAssetTypeSchemaVersion(), 1);
@ -227,11 +219,8 @@ TEST(QueryResponseTestV2, QueryResponseTestV2)
TEST(QueryResponseTestV2, MainAttributesTestV2)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
string string_attribute(
"{\n"
" \"assetCollections\": [\n"
@ -277,7 +266,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss(string_attribute);
{
cereal::JSONInputArchive ar(ss);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map = obj.getData().begin()->getMainAttributes();
@ -331,7 +320,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss2(many_strings_attribute);
{
cereal::JSONInputArchive ar(ss2);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map2 = obj.getData().begin()->getMainAttributes();
@ -390,7 +379,7 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
stringstream ss3(strings_vector_attribute);
{
cereal::JSONInputArchive ar(ss3);
obj.loadFromJson(ar);
obj.serialize(ar);
}
map<string, vector<string>> attributes_map3 = obj.getData().begin()->getMainAttributes();
@ -403,11 +392,8 @@ TEST(QueryResponseTestV2, MainAttributesTestV2)
TEST(QueryResponseTestV2, IntelligenceFailTest)
{
StrictMock<MockMainLoop> mock_ml;
NiceMock<MockTimeGet> time_get;
IntelligenceComponentV2 new_intelligence;
DataString data;
IntelligenceQueryResponse<stringData1> obj;
IntelligenceQueryResponseT<stringData1> obj;
string status_fail_data_str(
"{\n"
" \"assetCollections\": [\n"
@ -456,7 +442,7 @@ TEST(QueryResponseTestV2, IntelligenceFailTest)
{
cereal::JSONInputArchive ar(ss);
try {
obj.loadFromJson(ar);
obj.serialize(ar);
} catch (exception &e) {
EXPECT_EQ(e.what(), error_str);
}

128
core/intelligence_is_v2/intelligence_request.cc Normal file
View File

@ -0,0 +1,128 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_request.h"
#include "debug.h"
#include "intelligence_comp_v2.h"
#include "intelligence_is_v2/json_stream.h"
using namespace Intelligence;
using namespace std;
USE_DEBUG_FLAG(D_INTELLIGENCE);
static const unsigned int upper_assets_limit = 50;
static const unsigned int upper_confidence_limit = 1000;
Maybe<void>
IntelligenceRequest::checkAssetsLimit() const
{
for (const QueryRequest &query_request : queries) {
uint assets_limit = query_request.getAssetsLimit();
if (0 < assets_limit && assets_limit <= upper_assets_limit) continue;
dbgTrace(D_INTELLIGENCE)
<< "Assets limit for request is "
<< upper_assets_limit
<< ", requests assets: "
<< assets_limit;
return genError("Assets limit valid range is of [1, " + to_string(upper_assets_limit) + "]");
}
return Maybe<void>();
}
Maybe<void>
IntelligenceRequest::checkMinConfidence() const
{
for (const QueryRequest &query_request : queries) {
if (query_request.checkMinConfidence(upper_confidence_limit)) continue;
dbgTrace(D_INTELLIGENCE) << "Illegal confidence value";
return genError(
"Minimum confidence value valid range is of [1, " + std::to_string(upper_confidence_limit) + "]"
);
}
return Maybe<void>();
}
bool
IntelligenceRequest::isPagingActivated() const
{
if (!isPagingAllowed()) return false;
return queries.begin()->getCursorState().ok();
}
Maybe<bool>
IntelligenceRequest::isPagingFinished() const
{
if (!isPagingActivated()) return genError("Paging is not activated");
return queries.begin()->getCursorState().unpack() == CursorState::DONE;
}
Maybe<Intelligence_IS_V2::CursorState>
IntelligenceRequest::getPagingStatus() const
{
if (!isPagingAllowed()) return genError("Paging is not allowed");
return queries.begin()->getCursorState();
}
bool
IntelligenceRequest::isPagingAllowed() const
{
if (isBulk()) return false;
return true;
}
Maybe<std::string>
IntelligenceRequest::genJson() const
{
std::stringstream str_stream;
JsonStream json_stream(&str_stream, is_pretty);
{
cereal::JSONOutputArchive out_ar(json_stream);
if (isBulk()) {
out_ar.setNextName("queries");
out_ar.startNode();
out_ar.makeArray();
uint index = 0;
for (const auto &query : queries) {
out_ar.setNextName(nullptr);
out_ar.startNode();
out_ar.setNextName("query");
out_ar.startNode();
query.saveToJson(out_ar);
out_ar.finishNode();
out_ar(cereal::make_nvp("index", index));
out_ar.finishNode();
index++;
}
out_ar.finishNode();
} else {
queries.begin()->saveToJson(out_ar);
}
}
return str_stream.str();
}
bool
IntelligenceRequest::loadJson(const string &json)
{
response_from_fog = json;
return true;
}
Maybe<string>
IntelligenceRequest::getResponseFromFog() const
{
return response_from_fog;
}

74
core/intelligence_is_v2/intelligence_response.cc Normal file
View File

@ -0,0 +1,74 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_is_v2/intelligence_response.h"
using namespace std;
using namespace Intelligence;
Maybe<void>
Response::load()
{
try {
stringstream in;
in.str(json_response);
cereal::JSONInputArchive in_ar(in);
if (is_bulk) {
IntelligenceQueryBulkResponse bulk_response;
bulk_response.serialize(in_ar);
unsigned int error_idx = 0;
unsigned int valid_idx = 0;
const auto &valid_response = bulk_response.getValid();
const auto &errors = bulk_response.getErrors();
responses.clear();
responses.reserve(size);
dbgTrace(D_INTELLIGENCE) << "Received response for bulk request with " << size << " items";
for (unsigned int query_idx = 0; query_idx < size; query_idx++) {
if (valid_idx < valid_response.size() && valid_response[valid_idx].getIndex() == query_idx) {
responses.push_back(valid_response[valid_idx].getResponse());
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is valid";
valid_idx++;
} else if (error_idx < errors.size() && errors[error_idx].getIndex() == query_idx) {
responses.emplace_back();
responses[query_idx].setFailInBulk();
dbgTrace(D_INTELLIGENCE) << "Item #" << query_idx << " is invalid";
error_idx++;
} else {
dbgWarning(D_INTELLIGENCE)
<< "Query index was not found neither in valid nor error responses, assuming error";
responses[query_idx].setFailInBulk();
}
}
} else {
single_response.serialize(in_ar);
}
} catch(const std::exception &e) {
return genError("Load common data failed. Error: " + string(e.what()));
}
return {};
}
Intelligence_IS_V2::ResponseStatus
Response::getResponseStatus() const
{
if (!is_bulk) return single_response.getResponseStatus();
if (responses.size() == 0) return ResponseStatus::IN_PROGRESS;
for (const auto &response_itr : responses) {
if (response_itr.isValidInBulk() && response_itr.getResponseStatus() == ResponseStatus::IN_PROGRESS) {
return ResponseStatus::IN_PROGRESS;
}
}
return ResponseStatus::DONE;
}

203
core/intelligence_is_v2/intelligence_server.cc Normal file
View File

@ -0,0 +1,203 @@
// Copyright (C) 2022 Check Point Software Technologies Ltd. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "intelligence_server.h"
#include "config.h"
#include "debug.h"
#include "enum_array.h"
#include "intelligence_comp_v2.h"
USE_DEBUG_FLAG(D_INTELLIGENCE);
using namespace Intelligence;
using namespace std;
static const string query_uri = "/api/v2/intelligence/assets/query";
static const string queries_uri = "/api/v2/intelligence/assets/queries";
static const string primary_port_setting = "local intelligence server primary port";
static const string secondary_port_setting = "local intelligence server secondary port";
Sender::Sender(IntelligenceRequest request) : request(request)
{
i_message = Singleton::Consume<I_Messaging>::by<IntelligenceComponentV2>();
i_timer = Singleton::Consume<I_TimeGet>::by<IntelligenceComponentV2>();
i_mainloop = Singleton::Consume<I_MainLoop>::by<IntelligenceComponentV2>();
bool crowdsec_enabled = std::getenv("CROWDSEC_ENABLED") ?
std::string(std::getenv("CROWDSEC_ENABLED")) == "true" :
false;
if (getProfileAgentSettingWithDefault<bool>(crowdsec_enabled, "layer7AccessControl.crowdsec.enabled")) {
is_local_intelligence = true;
}
if (getProfileAgentSettingWithDefault<bool>(false, "agent.config.useLocalIntelligence")) {
is_local_intelligence = true;
}
auto setting_server_ip = getSetting<string>("intelligence", "local intelligence server ip");
if (setting_server_ip.ok()) server_ip = *setting_server_ip;
}
Maybe<Response>
Sender::sendIntelligenceRequest()
{
if (server_ip.ok() && is_local_intelligence) {
auto response = sendQueryObjectToLocalServer(true);
if (response.ok()) return response;
dbgWarning(D_INTELLIGENCE) << "Failed to send query to primary port. Error" << response.getErr();
response = sendQueryObjectToLocalServer(false);
if (response.ok()) return response;
dbgWarning(D_INTELLIGENCE) << "Failed to send query to secondary port. Error" << response.getErr();
}
if (request.getPagingStatus().ok()) {
return sendMessage();
}
return sendQueryMessage();
}
Maybe<Response>
Sender::sendQueryObjectToLocalServer(bool is_primary_port)
{
auto local_port = getSetting<uint>(
"intelligence",
is_primary_port ? primary_port_setting : secondary_port_setting
);
if (!local_port.ok()) return genError(
"Failed to send intelligence query to local server. Config Error number:"
+ to_string(static_cast<uint>(local_port.getErr()))
);
server_port = *local_port;
conn_flags.reset();
auto res = sendQueryMessage();
server_port = genError("port unset after use");
return res;
}
Maybe<Response>
Sender::sendQueryMessage()
{
uint request_overall_timeout_conf = getConfigurationWithDefault<uint>(
20,
"intelligence",
"request overall timeout"
);
uint request_lap_timeout_conf = getConfigurationWithDefault<uint>(
5,
"intelligence",
"request lap timeout"
);
chrono::seconds request_overall_timeout = chrono::seconds(request_overall_timeout_conf);
chrono::seconds request_lap_timeout = chrono::seconds(request_lap_timeout_conf);
chrono::microseconds send_request_start_time = i_timer->getMonotonicTime();
chrono::microseconds last_lap_time = i_timer->getMonotonicTime();
chrono::seconds seconds_since_start = chrono::seconds(0);
chrono::seconds seconds_since_last_lap = chrono::seconds(0);
Maybe<Response> res = genError("Uninitialized");
do {
res = sendMessage();
if (res.ok() && res->getResponseStatus() == ResponseStatus::IN_PROGRESS) {
i_mainloop->yield(true);
}
seconds_since_start = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - send_request_start_time
);
seconds_since_last_lap = std::chrono::duration_cast<std::chrono::seconds>(
i_timer->getMonotonicTime() - last_lap_time
);
last_lap_time = i_timer->getMonotonicTime();
} while (res.ok() &&
res->getResponseStatus() == ResponseStatus::IN_PROGRESS &&
seconds_since_start < request_overall_timeout &&
seconds_since_last_lap < request_lap_timeout
);
return res;
}
Maybe<Response>
Sender::sendMessage()
{
if (server_ip.ok() || server_port.ok()) {
if (!server_ip.ok()) return genError("Can't send intelligence request. Server ip wasn't set");
if (!server_port.ok()) return genError("Can't send intelligence request. Server port wasn't set");
} else if (!server_ip.ok() && !server_port.ok()) {
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
request.isBulk() ? queries_uri : query_uri,
request,
MessageCategory::INTELLIGENCE
);
if (req_status.ok()) {
return createResponse();
};
auto response_error = req_status.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ " "
+ req_status.getErr().toString()
);
}
dbgTrace(D_INTELLIGENCE)
<< "Sending intelligence request with IP: "
<< *server_ip
<< " port: "
<< *server_port
<< " query_uri: "
<< (request.isBulk() ? queries_uri : query_uri);
MessageMetadata req_md(*server_ip, *server_port, conn_flags);
auto req_status = i_message->sendSyncMessage(
HTTPMethod::POST,
query_uri,
request,
MessageCategory::INTELLIGENCE,
req_md
);
if (!req_status.ok()) {
auto response_error = req_status.getErr().toString();
dbgWarning(D_INTELLIGENCE) << "Failed to send intelligence request. Error:" << response_error;
return genError(
"Failed to send intelligence request. "
+ req_status.getErr().getBody()
+ " "
+ req_status.getErr().toString()
);
};
return createResponse();
}
Maybe<Response>
Sender::createResponse()
{
auto mb_json_body = request.getResponseFromFog();
if (!mb_json_body.ok()) return mb_json_body.passErr();
Response response(*mb_json_body, request.getSize(), request.isBulk());
auto load_status = response.load();
if (!load_status.ok()) return load_status.passErr();
return response;
}

12
core/intelligence_is_v2/query_filter_v2.cc
View File

@ -146,18 +146,6 @@ SerializableQueryFilter::saveOperation(cereal::JSONOutputArchive &ar) const
}
}
Maybe<SerializableQueryCondition::ValueVariant>
SerializableQueryFilter::getConditionValueByKey(const string &key) const
{
for (const SerializableQueryCondition &condition : condition_operands) {
if (condition.getConditionType() == Condition::EQUALS && condition.getKey() == key) {
return condition.getValue();
}
}
return genError("Key not found.");
}
bool
SerializableQueryFilter::isOperatorComp(const Operator &oper) const
{

42
core/intelligence_is_v2/query_request_v2.cc
View File

@ -25,27 +25,6 @@ USE_DEBUG_FLAG(D_INTELLIGENCE);
static const EnumArray<ObjectType, string> object_type_to_string_array{"asset", "zone", "configuration", "shortLived"};
BulkQueryRequest::BulkQueryRequest(QueryRequest &_request, int _index)
:
request(_request),
index(_index)
{}
QueryRequest
BulkQueryRequest::getQueryRequest() const
{
return request;
}
void
BulkQueryRequest::save(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("query", getQueryRequest()),
cereal::make_nvp("index", index)
);
}
QueryRequest::QueryRequest(
Condition condition_type,
const string &key,
@ -96,22 +75,7 @@ QueryRequest::convertObjectTypeToString() const
void
QueryRequest::saveToJson(cereal::JSONOutputArchive &ar) const
{
ar(
cereal::make_nvp("limit", assets_limit),
cereal::make_nvp("fullResponse", full_response),
cereal::make_nvp("query", query)
);
auto objTypeString = convertObjectTypeToString();
if (objTypeString.ok()) {
ar(cereal::make_nvp("objectType", *objTypeString));
} else {
dbgTrace(D_INTELLIGENCE) << objTypeString.getErr();
}
if (cursor.ok()) ar(cereal::make_nvp("cursor", cursor.unpack().second));
requested_attributes.save(ar);
query_types.save(ar);
save(ar);
}
void
@ -214,7 +178,7 @@ QueryRequest::setAssetsLimit(uint _assets_limit)
}
bool
QueryRequest::checkMinConfidence(uint upper_confidence_limit)
QueryRequest::checkMinConfidence(uint upper_confidence_limit) const
{
return requested_attributes.checkMinConfidence(upper_confidence_limit);
}
@ -232,7 +196,7 @@ QueryRequest::isPagingActivated()
}
Maybe<CursorState>
QueryRequest::getCursorState()
QueryRequest::getCursorState() const
{
if (!cursor.ok()) return genError("Paging not activated");
return cursor.unpack().first;

2
core/intelligence_is_v2/requested_attributes_v2.cc
View File

@ -54,7 +54,7 @@ SerializableAttributesMap::getAttributeByKey(const string &key) const
}
bool
SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit)
SerializableAttributesMap::checkMinConfidence(uint upper_confidence_limit) const
{
for (auto const &attribute : requested_attributes) {
if (attribute.second == 0 || attribute.second > upper_confidence_limit) return false;

3
nodes/http_transaction_handler/CMakeLists.txt
View File

@ -15,6 +15,7 @@ target_link_libraries(cp-nano-http-transaction-handler
pcre2-posix
yajl_s
hiredis
maxminddb
-lshmem_ipc
-lnginx_attachment_util
@ -38,6 +39,8 @@ target_link_libraries(cp-nano-http-transaction-handler
ips
keywords
l7_access_control
geo_location
http_geo_filter
-Wl,--end-group
)

6
nodes/http_transaction_handler/main.cc
View File

@ -21,6 +21,8 @@
#include "waap.h"
#include "ips_comp.h"
#include "keyword_comp.h"
#include "http_geo_filter.h"
#include "geo_location.h"
int
main(int argc, char **argv)
@ -34,7 +36,9 @@ main(int argc, char **argv)
RateLimit,
WaapComponent,
IPSComp,
KeywordComp
KeywordComp,
GeoLocation,
HttpGeoFilter
> comps;
comps.registerGlobalValue<bool>("Is Rest primary routine", true);

4
nodes/orchestration/package/open-appsec-ctl.sh
View File

@ -1027,6 +1027,10 @@ run_status() # Initials - rs
for service in $all_services; do
print_single_service_status "$service"
done
if command -v getenforce &>/dev/null && [ "$(getenforce)" != "Disabled" ]; then
echo "SELinux has been detected, which could potentially disrupt the agent's normal operation."
fi
}
run_load_settings() # Initials - rls