diff --git a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc index fa07cfc..df6e9a1 100755 --- a/components/attachment-intakers/nginx_attachment/nginx_attachment.cc +++ b/components/attachment-intakers/nginx_attachment/nginx_attachment.cc @@ -1135,7 +1135,11 @@ private: "webUserResponse" ); + bool remove_event_id_param = + getProfileAgentSettingWithDefault("false", "nginxAttachment.removeRedirectEventId") == "true"; + string uuid; + string redirectUrl; if (i_transaction_table->hasState()) { NginxAttachmentOpaque &opaque = i_transaction_table->getState(); uuid = opaque.getSessionUUID(); @@ -1145,7 +1149,12 @@ private: if (web_trigger_conf.getDetailsLevel() == "Redirect") { web_response_data.response_data.redirect_data.redirect_location_size = web_trigger_conf.getRedirectURL().size(); - web_response_data.response_data.redirect_data.add_event_id = web_trigger_conf.getAddEventId() ? 1 : 0; + bool add_event = web_trigger_conf.getAddEventId(); + if (add_event && !remove_event_id_param) { + web_response_data.response_data.redirect_data.redirect_location_size += + strlen("?event_id=") + uuid.size(); + } + web_response_data.response_data.redirect_data.add_event_id = add_event ? 1 : 0; web_response_data.web_repsonse_type = static_cast(ngx_web_response_type_e::REDIRECT_WEB_RESPONSE); } else { web_response_data.response_data.custom_response_data.title_size = @@ -1159,8 +1168,13 @@ private: verdict_data_sizes.push_back(sizeof(ngx_http_cp_web_response_data_t)); if (web_trigger_conf.getDetailsLevel() == "Redirect") { - verdict_data.push_back(reinterpret_cast(web_trigger_conf.getRedirectURL().data())); - verdict_data_sizes.push_back(web_trigger_conf.getRedirectURL().size()); + redirectUrl = web_trigger_conf.getRedirectURL(); + if (!remove_event_id_param && web_trigger_conf.getAddEventId()) { + redirectUrl += "?event-id=" + uuid; + } + + verdict_data.push_back(reinterpret_cast(redirectUrl.data())); + verdict_data_sizes.push_back(redirectUrl.size()); } else { verdict_data.push_back(reinterpret_cast(web_trigger_conf.getResponseTitle().data())); verdict_data_sizes.push_back(web_trigger_conf.getResponseTitle().size()); diff --git a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc index 793505e..b996342 100755 --- a/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc +++ b/components/attachment-intakers/nginx_attachment/user_identifiers_config.cc @@ -282,7 +282,7 @@ isIpTrusted(const string &value, const vector &cidr_values) } Maybe -UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const +UsersAllIdentifiersConfig::parseXForwardedFor(const string &str, ExtractType type) const { vector header_values = split(str); @@ -291,12 +291,23 @@ UsersAllIdentifiersConfig::parseXForwardedFor(const string &str) const vector xff_values = getHeaderValuesFromConfig("x-forwarded-for"); vector cidr_values(xff_values.begin(), xff_values.end()); - for (const string &value : header_values) { - if (!IPAddr::createIPAddr(value).ok()) { - dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << value; + for (auto it = header_values.rbegin(); it != header_values.rend() - 1; ++it) { + if (!IPAddr::createIPAddr(*it).ok()) { + dbgWarning(D_NGINX_ATTACHMENT_PARSER) << "Invalid IP address found in the xff header IPs list: " << *it; return genError("Invalid IP address"); } - if (!isIpTrusted(value, cidr_values)) return genError("Untrusted Ip found"); + if (type == ExtractType::PROXYIP) continue; + if (!isIpTrusted(*it, cidr_values)) { + dbgDebug(D_NGINX_ATTACHMENT_PARSER) << "Found untrusted IP in the xff header IPs list: " << *it; + return *it; + } + } + + if (!IPAddr::createIPAddr(header_values[0]).ok()) { + dbgWarning(D_NGINX_ATTACHMENT_PARSER) + << "Invalid IP address found in the xff header IPs list: " + << header_values[0]; + return genError("Invalid IP address"); } return header_values[0]; @@ -312,7 +323,7 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext return; } NginxAttachmentOpaque &opaque = i_transaction_table->getState(); - auto value = parseXForwardedFor(header.getValue()); + auto value = parseXForwardedFor(header.getValue(), type); if (!value.ok()) { dbgTrace(D_NGINX_ATTACHMENT_PARSER) << "Could not extract source identifier from X-Forwarded-For header"; return; @@ -321,12 +332,13 @@ UsersAllIdentifiersConfig::setXFFValuesToOpaqueCtx(const HttpHeader &header, Ext if (type == ExtractType::SOURCEIDENTIFIER) { opaque.setSourceIdentifier(header.getKey(), value.unpack()); dbgDebug(D_NGINX_ATTACHMENT_PARSER) - << "Added source identifir to XFF " + << "Added source identifier from XFF header" << value.unpack(); opaque.setSavedData(HttpTransactionData::xff_vals_ctx, header.getValue()); + opaque.setSavedData(HttpTransactionData::source_identifier, value.unpack()); dbgTrace(D_NGINX_ATTACHMENT_PARSER) - << "XFF found, set ctx with value from header: " - << static_cast(header.getValue()); + << "XFF found, set ctx with value from header: " + << static_cast(header.getValue()); } else { opaque.setSavedData(HttpTransactionData::proxy_ip_ctx, value.unpack()); } diff --git a/components/include/reverse_proxy_defaults.h b/components/include/reverse_proxy_defaults.h index 0be2c3c..6c07ad2 100644 --- a/components/include/reverse_proxy_defaults.h +++ b/components/include/reverse_proxy_defaults.h @@ -28,7 +28,7 @@ static const std::string default_nginx_config_file = "/etc/cp/conf/rpmanager/ngi static const std::string default_prepare_nginx_config_file = "/etc/cp/conf/rpmanager/nginx_prepare.conf"; static const std::string default_global_conf_template = "/etc/cp/conf/rpmanager/nginx-conf-template"; static const std::string default_nginx_config_include_file = - "/etc/cp/conf/rpmanager/servers/nginx_conf_include.conf"; + "/etc/cp/conf/rpmanager/servers/00_nginx_conf_include.conf"; static const std::string default_global_conf_include_template = "/etc/cp/conf/rpmanager/nginx-conf-include-template"; static const std::string default_global_conf_include_template_no_responses = diff --git a/components/include/user_identifiers_config.h b/components/include/user_identifiers_config.h index 61274f1..d5921d6 100755 --- a/components/include/user_identifiers_config.h +++ b/components/include/user_identifiers_config.h @@ -58,7 +58,7 @@ private: const std::string::const_iterator &end, const std::string &key) const; Buffer extractKeyValueFromCookie(const std::string &cookie_value, const std::string &key) const; - Maybe parseXForwardedFor(const std::string &str) const; + Maybe parseXForwardedFor(const std::string &str, ExtractType type) const; std::vector user_identifiers; }; diff --git a/components/include/waap.h b/components/include/waap.h index 02f3983..4df31ba 100755 --- a/components/include/waap.h +++ b/components/include/waap.h @@ -33,7 +33,6 @@ class I_WaapAssetStatesManager; class I_Messaging; class I_AgentDetails; class I_Encryptor; -class I_WaapModelResultLogger; const std::string WAAP_APPLICATION_NAME = "waap application"; @@ -51,8 +50,7 @@ class WaapComponent Singleton::Consume, Singleton::Consume, Singleton::Consume, - Singleton::Consume, - Singleton::Consume + Singleton::Consume { public: WaapComponent(); diff --git a/components/security_apps/orchestration/details_resolver/details_resolver.cc b/components/security_apps/orchestration/details_resolver/details_resolver.cc index 8a4d8bf..b605505 100644 --- a/components/security_apps/orchestration/details_resolver/details_resolver.cc +++ b/components/security_apps/orchestration/details_resolver/details_resolver.cc @@ -350,7 +350,7 @@ DetailsResolver::Impl::readCloudMetadata() } if (!cloud_metadata.ok()) { - dbgWarning(D_ORCHESTRATOR) << cloud_metadata.getErr(); + dbgDebug(D_ORCHESTRATOR) << cloud_metadata.getErr(); return genError("Failed to fetch cloud metadata"); } diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h index cf8a80d..fe98ccf 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/checkpoint_product_handlers.h @@ -18,6 +18,8 @@ #include #include #include +#include +#include #if defined(gaia) @@ -100,6 +102,14 @@ checkIsInstallHorizonTelemetrySucceeded(const string &command_output) return command_output; } +Maybe +getOtlpAgentGaiaOsRole(const string &command_output) +{ + if (command_output == "" ) return string("-1"); + + return command_output; +} + Maybe getQUID(const string &command_output) { @@ -111,6 +121,13 @@ getQUID(const string &command_output) return command_output; } +Maybe +getIsAiopsRunning(const string &command_output) +{ + if (command_output == "" ) return string("false"); + + return command_output; +} Maybe checkHasSDWan(const string &command_output) @@ -186,6 +203,24 @@ getMgmtObjAttr(shared_ptr file_stream, const string &attr) return genError("Object attribute was not found. Attr: " + attr); } +Maybe +getAttrFromCpsdwanGetDataJson(const string &attr) +{ + static const std::string get_data_json_path = "/tmp/cpsdwan_getdata_orch.json"; + std::ifstream ifs(get_data_json_path); + if (ifs.is_open()) { + rapidjson::IStreamWrapper isw(ifs); + rapidjson::Document document; + document.ParseStream(isw); + + if (!document.HasParseError() && document.HasMember(attr.c_str()) && document[attr.c_str()].IsString()) { + return string(document[attr.c_str()].GetString()); + } + } + + return genError("Attribute " + attr + " was not found in " + get_data_json_path); +} + Maybe getMgmtObjUid(const string &command_output) { @@ -193,6 +228,11 @@ getMgmtObjUid(const string &command_output) return command_output; } + Maybe obj_uuid = getAttrFromCpsdwanGetDataJson("uuid"); + if (obj_uuid.ok()) { + return obj_uuid.unpack(); + } + static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C"; auto file_stream = std::make_shared(obj_path); if (!file_stream->is_open()) { @@ -310,7 +350,12 @@ getSmbObjectName(const string &command_output) if (command_output.empty() || command_output[0] != centrally_managed_comd_output) { return genError("Object name was not found"); } - + + Maybe obj_name = getAttrFromCpsdwanGetDataJson("name"); + if (obj_name.ok()) { + return obj_name.unpack(); + } + static const string obj_path = (getenv("FWDIR") ? string(getenv("FWDIR")) : "") + "/database/myown.C"; auto ifs = std::make_shared(obj_path); if (!ifs->is_open()) { diff --git a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h index 1c3a411..78a8170 100755 --- a/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h +++ b/components/security_apps/orchestration/details_resolver/details_resolver_handlers/details_resolver_impl.h @@ -73,6 +73,15 @@ SHELL_CMD_HANDLER("MGMT_QUID", "[ -d /opt/CPquid ] " "&& python3 /opt/CPquid/Quid_Api.py -i " "/opt/CPotelcol/quid_api/get_mgmt_quid.json | jq -r .message[0].MGMT_QUID || echo ''", getQUID) +SHELL_CMD_HANDLER("AIOPS_AGENT_ROLE", "[ -d /opt/CPOtlpAgent/custom_scripts ] " + "&& ENV_NO_FORMAT=1 /opt/CPOtlpAgent/custom_scripts/agent_role.sh", + getOtlpAgentGaiaOsRole) +SHELL_CMD_HANDLER( + "IS_AIOPS_RUNNING", + "FS_PATH=; " + "PID=$(ps auxf | grep -v grep | grep -E ${FS_PATH}.*cp-nano-horizon-telemetry | awk -F' ' '{printf $2}'); " + "[ -z \"{PID}\" ] && echo 'false' || echo 'true'", + getIsAiopsRunning) SHELL_CMD_HANDLER("hasSDWan", "[ -f $FWDIR/bin/sdwan_steering ] && echo '1' || echo '0'", checkHasSDWan) SHELL_CMD_HANDLER( "canUpdateSDWanData", @@ -180,8 +189,7 @@ SHELL_CMD_HANDLER( ) SHELL_CMD_HANDLER( "managements", - "sed -n '/:masters (/,$p' $FWDIR/database/myself_objects.C |" - " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'", + "echo 1", extractManagements ) #endif //gaia @@ -237,8 +245,7 @@ SHELL_CMD_HANDLER( SHELL_CMD_HANDLER( "managements", - "sed -n '/:masters (/,$p' /tmp/local.cfg |" - " sed -e ':a' -e 'N' -e '$!ba' -e 's/\\n//g' -e 's/\t//g' -e 's/ //g' | sed 's/))):.*/)))):/'", + "echo 1", extractManagements ) #endif//smb diff --git a/components/security_apps/orchestration/hybrid_mode_telemetry.cc b/components/security_apps/orchestration/hybrid_mode_telemetry.cc index 800f253..1a599a8 100755 --- a/components/security_apps/orchestration/hybrid_mode_telemetry.cc +++ b/components/security_apps/orchestration/hybrid_mode_telemetry.cc @@ -34,7 +34,9 @@ HybridModeMetric::upon(const HybridModeMetricEvent &) { auto shell_cmd = Singleton::Consume::by(); auto maybe_cmd_output = shell_cmd->getExecOutput( - getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count" + getFilesystemPathConfig() + "/watchdog/cp-nano-watchdog --restart_count", + 1000, + false ); // get wd process restart count diff --git a/components/security_apps/orchestration/include/declarative_policy_utils.h b/components/security_apps/orchestration/include/declarative_policy_utils.h index 73861c7..10056d0 100644 --- a/components/security_apps/orchestration/include/declarative_policy_utils.h +++ b/components/security_apps/orchestration/include/declarative_policy_utils.h @@ -79,8 +79,8 @@ public: ) override; std::string getUpdate(CheckUpdateRequest &request) override; bool shouldApplyPolicy() override; - void turnOffApplyPolicyFlag() override; - void turnOnApplyPolicyFlag() override; + void turnOffApplyLocalPolicyFlag() override; + void turnOnApplyLocalPolicyFlag() override; std::string getCurrPolicy() override { return curr_policy; } @@ -94,7 +94,7 @@ private: std::string curr_version; std::string curr_policy; std::string curr_checksum; - bool should_apply_policy; + bool should_apply_local_policy; }; #endif // __DECLARATIVE_POLICY_UTILS_H__ diff --git a/components/security_apps/orchestration/include/i_declarative_policy.h b/components/security_apps/orchestration/include/i_declarative_policy.h index 0401463..548da68 100644 --- a/components/security_apps/orchestration/include/i_declarative_policy.h +++ b/components/security_apps/orchestration/include/i_declarative_policy.h @@ -22,8 +22,8 @@ public: virtual std::string getCurrPolicy() = 0; - virtual void turnOffApplyPolicyFlag() = 0; - virtual void turnOnApplyPolicyFlag() = 0; + virtual void turnOffApplyLocalPolicyFlag() = 0; + virtual void turnOnApplyLocalPolicyFlag() = 0; protected: virtual ~I_DeclarativePolicy() {} diff --git a/components/security_apps/orchestration/orchestration_comp.cc b/components/security_apps/orchestration/orchestration_comp.cc index 2290d6e..95f5fa9 100755 --- a/components/security_apps/orchestration/orchestration_comp.cc +++ b/components/security_apps/orchestration/orchestration_comp.cc @@ -2033,7 +2033,7 @@ private: } auto policy_mgmt_mode = getSettingWithDefault("management", "profileManagedMode"); if (getOrchestrationMode() == OrchestrationMode::HYBRID || policy_mgmt_mode == "declarative") { - Singleton::Consume::from()->turnOnApplyPolicyFlag(); + Singleton::Consume::from()->turnOnApplyLocalPolicyFlag(); } auto policy_version = i_service_controller->getPolicyVersion(); diff --git a/components/security_apps/orchestration/service_controller/service_controller.cc b/components/security_apps/orchestration/service_controller/service_controller.cc index 200af8e..a64b103 100755 --- a/components/security_apps/orchestration/service_controller/service_controller.cc +++ b/components/security_apps/orchestration/service_controller/service_controller.cc @@ -793,7 +793,7 @@ ServiceController::Impl::updateServiceConfiguration( << "Policy file was not updated. Sending reload command regarding settings and data"; auto signal_services = sendSignalForServices(nano_services_to_update, ""); if (!signal_services.ok()) return signal_services.passErr(); - Singleton::Consume::from()->turnOffApplyPolicyFlag(); + Singleton::Consume::from()->turnOffApplyLocalPolicyFlag(); return Maybe(); } @@ -940,7 +940,7 @@ ServiceController::Impl::updateServiceConfiguration( if (new_policy_path.compare(config_file_path) == 0) { dbgDebug(D_SERVICE_CONTROLLER) << "Enforcing the default policy file"; policy_version = version_value; - Singleton::Consume::from()->turnOffApplyPolicyFlag(); + Singleton::Consume::from()->turnOffApplyLocalPolicyFlag(); return Maybe(); } @@ -959,7 +959,7 @@ ServiceController::Impl::updateServiceConfiguration( } if (!was_policy_updated && !send_signal_for_services_err.empty()) return genError(send_signal_for_services_err); - Singleton::Consume::from()->turnOffApplyPolicyFlag(); + Singleton::Consume::from()->turnOffApplyLocalPolicyFlag(); return Maybe(); } diff --git a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc index 21e3238..6ab7524 100644 --- a/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc +++ b/components/security_apps/orchestration/update_communication/declarative_policy_utils.cc @@ -17,7 +17,7 @@ void DeclarativePolicyUtils::init() { local_policy_path = getFilesystemPathConfig() + "/conf/local_policy.yaml"; - should_apply_policy = true; + should_apply_local_policy = true; Singleton::Consume::by()->addRestCall( RestAction::SET, "apply-policy" ); @@ -40,7 +40,7 @@ DeclarativePolicyUtils::upon(const ApplyPolicyEvent &event) { dbgTrace(D_ORCHESTRATOR) << "Apply policy event"; local_policy_path = event.getPolicyPath(); - should_apply_policy = true; + should_apply_local_policy = true; } // LCOV_EXCL_STOP @@ -48,19 +48,24 @@ bool DeclarativePolicyUtils::shouldApplyPolicy() { auto env_type = Singleton::Consume::by()->getEnvType(); - return env_type == EnvType::K8S ? true : should_apply_policy; + if (env_type == EnvType::K8S) { + I_OrchestrationTools *orch_tools = Singleton::Consume::by(); + auto maybe_new_version = orch_tools->readFile("/etc/cp/conf/k8s-policy-check.trigger"); + return maybe_new_version != curr_version; + } + return should_apply_local_policy; } void -DeclarativePolicyUtils::turnOffApplyPolicyFlag() +DeclarativePolicyUtils::turnOffApplyLocalPolicyFlag() { - should_apply_policy = false; + should_apply_local_policy = false; } void -DeclarativePolicyUtils::turnOnApplyPolicyFlag() +DeclarativePolicyUtils::turnOnApplyLocalPolicyFlag() { - should_apply_policy = true; + should_apply_local_policy = true; } Maybe @@ -211,6 +216,6 @@ DeclarativePolicyUtils::periodicPolicyLoad() if (*new_checksum == curr_checksum) return; - should_apply_policy = true; + should_apply_local_policy = true; curr_checksum = *new_checksum; } diff --git a/components/security_apps/orchestration/update_communication/fog_communication.cc b/components/security_apps/orchestration/update_communication/fog_communication.cc index 7a5567a..2928637 100755 --- a/components/security_apps/orchestration/update_communication/fog_communication.cc +++ b/components/security_apps/orchestration/update_communication/fog_communication.cc @@ -74,7 +74,7 @@ FogCommunication::getUpdate(CheckUpdateRequest &request) << " to: " << policy_mgmt_mode; profile_mode = policy_mgmt_mode; - i_declarative_policy->turnOnApplyPolicyFlag(); + i_declarative_policy->turnOnApplyLocalPolicyFlag(); } if (i_declarative_policy->shouldApplyPolicy()) { diff --git a/components/security_apps/rate_limit/rate_limit.cc b/components/security_apps/rate_limit/rate_limit.cc index 8d5db5b..c0549b5 100755 --- a/components/security_apps/rate_limit/rate_limit.cc +++ b/components/security_apps/rate_limit/rate_limit.cc @@ -293,8 +293,13 @@ public: return ACCEPT; } - burst = rule.getRateLimit(); - limit = calcRuleLimit(rule); + auto replicas = getenv("REPLICA_COUNT") ? std::stoi(getenv("REPLICA_COUNT")) : 1; + if (replicas == 0) { + dbgWarning(D_RATE_LIMIT) << "REPLICA_COUNT environment variable is set to 0, setting REPLICA_COUNT to 1"; + replicas = 1; + } + burst = static_cast(rule.getRateLimit()) / replicas; + limit = static_cast(calcRuleLimit(rule)) / replicas; dbgTrace(D_RATE_LIMIT) << "found rate limit rule with: " diff --git a/components/security_apps/waap/include/i_waap_model_result_logger.h b/components/security_apps/waap/include/i_waap_model_result_logger.h deleted file mode 100644 index 9d74a34..0000000 --- a/components/security_apps/waap/include/i_waap_model_result_logger.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -class IWaf2Transaction; -struct Waf2ScanResult; -namespace Waap { -namespace Scores { -struct ModelLoggingSettings; -} -} - -class I_WaapModelResultLogger { -public: - virtual ~I_WaapModelResultLogger() {} - - virtual void - logModelResult( - Waap::Scores::ModelLoggingSettings &settings, - IWaf2Transaction* transaction, - Waf2ScanResult &res, - std::string modelName, - std::string otherModelName, - double newScore, - double baseScore) = 0; -}; diff --git a/components/security_apps/waap/waap_clib/CMakeLists.txt b/components/security_apps/waap/waap_clib/CMakeLists.txt index 5310631..af71189 100755 --- a/components/security_apps/waap/waap_clib/CMakeLists.txt +++ b/components/security_apps/waap/waap_clib/CMakeLists.txt @@ -87,9 +87,10 @@ add_library(waap_clib ParserPairs.cc Waf2Util2.cc ParserPDF.cc + ParserKnownBenignSkipper.cc + ParserScreenedJson.cc ParserBinaryFile.cc RegexComparator.cc - WaapModelResultLogger.cc ) add_definitions("-Wno-unused-function") diff --git a/components/security_apps/waap/waap_clib/DeepParser.cc b/components/security_apps/waap/waap_clib/DeepParser.cc index f13c820..7bbe64a 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.cc +++ b/components/security_apps/waap/waap_clib/DeepParser.cc @@ -28,6 +28,8 @@ #include "ParserDelimiter.h" #include "ParserPDF.h" #include "ParserBinaryFile.h" +#include "ParserKnownBenignSkipper.h" +#include "ParserScreenedJson.h" #include "WaapAssetState.h" #include "Waf2Regex.h" #include "Waf2Util.h" @@ -359,6 +361,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f isRefererParamPayload, isUrlPayload, isUrlParamPayload, + isCookiePayload, flags, parser_depth, base64BinaryFileType @@ -410,6 +413,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f isRefererParamPayload, isUrlPayload, isUrlParamPayload, + isCookiePayload, flags, parser_depth, base64BinaryFileType @@ -461,6 +465,7 @@ DeepParser::onKv(const char *k, size_t k_len, const char *v, size_t v_len, int f isRefererParamPayload, isUrlPayload, isUrlParamPayload, + isCookiePayload, flags, parser_depth, base64ParamFound, @@ -835,6 +840,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, + bool isCookiePayload, int flags, size_t parser_depth, bool base64ParamFound, @@ -854,6 +860,7 @@ DeepParser::parseAfterMisleadingMultipartBoundaryCleaned( isRefererParamPayload, isUrlPayload, isUrlParamPayload, + isCookiePayload, flags, parser_depth, b64FileType @@ -918,6 +925,7 @@ bool isRefererPayload, bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, +bool isCookiePayload, int flags, size_t parser_depth ) { @@ -959,6 +967,7 @@ DeepParser::createInternalParser( bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, + bool isCookiePayload, int flags, size_t parser_depth, Waap::Util::BinaryFileType b64FileType @@ -978,7 +987,19 @@ DeepParser::createInternalParser( << "\n\tflags: " << flags << "\n\tparser_depth: " - << parser_depth; + << parser_depth + << "\n\tisBodyPayload: " + << isBodyPayload + << "\n\tisRefererPayload: " + << isRefererPayload + << "\n\tisRefererParamPayload: " + << isRefererParamPayload + << "\n\tisUrlPayload: " + << isUrlPayload + << "\n\tisUrlParamPayload: " + << isUrlParamPayload + << "\n\tisCookiePayload: " + << isCookiePayload; bool isPipesType = false, isSemicolonType = false, isAsteriskType = false, isCommaType = false, isAmperType = false; bool isKeyValDelimited = false; @@ -1045,6 +1066,53 @@ DeepParser::createInternalParser( } } + if (Waap::Util::isScreenedJson(cur_val)) { + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse screened JSON"; + m_parsersDeque.push_back(std::make_shared>(*this, parser_depth + 1)); + offset = 0; + return offset; + } + + dbgTrace(D_WAAP_DEEP_PARSER) + << "Offset = " + << offset + << " depth = " + << m_depth + << " isBodyPayload = " + << isBodyPayload; + //Detect sensor_data format in body and just use dedicated filter for it + if (m_depth == 1 + && isBodyPayload + && Waap::Util::detectKnownSource(cur_val) == Waap::Util::SOURCE_TYPE_SENSOR_DATA) { + m_parsersDeque.push_back( + std::make_shared>( + *this, + parser_depth + 1, + Waap::Util::SOURCE_TYPE_SENSOR_DATA + ) + ); + offset = 0; + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse data_sensor data - skipping it"; + return offset; + } + // Detect cookie parameter sensorsdata2015jssdkcross + // and causes false positives due to malformed JSON. Make preprocessing to parse it correctly + if (m_depth == 2 + && isCookiePayload) { + offset = Waap::Util::definePrefixedJson(cur_val); + if (offset >= 0) { + m_parsersDeque.push_back( + std::make_shared>( + *this, + parser_depth + 1, + m_pTransaction + ) + ); + dbgTrace(D_WAAP_DEEP_PARSER) << "Starting to parse JSON data"; + return offset; + } + } + // Detect wbxml (binary XML) data type if (m_depth == 1 && isBodyPayload && !valueStats.isUTF16 && m_pWaapAssetState->isWBXMLSampleType(cur_val)) { m_is_wbxml = true; @@ -1374,6 +1442,7 @@ DeepParser::createInternalParser( isRefererParamPayload, isUrlPayload, isUrlParamPayload, + isCookiePayload, flags, parser_depth ); diff --git a/components/security_apps/waap/waap_clib/DeepParser.h b/components/security_apps/waap/waap_clib/DeepParser.h index ab04a43..6290a4f 100755 --- a/components/security_apps/waap/waap_clib/DeepParser.h +++ b/components/security_apps/waap/waap_clib/DeepParser.h @@ -129,6 +129,7 @@ private: bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, + bool isCookiePayload, int flags, size_t parser_depth, Waap::Util::BinaryFileType b64FileType @@ -144,6 +145,7 @@ private: bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, + bool isCookiePayload, int flags, size_t parser_depth ); @@ -160,6 +162,7 @@ private: bool isRefererParamPayload, bool isUrlPayload, bool isUrlParamPayload, + bool isCookiePayload, int flags, size_t parser_depth, bool base64ParamFound, diff --git a/components/security_apps/waap/waap_clib/WaapAssetState.cc b/components/security_apps/waap/waap_clib/WaapAssetState.cc index 493b308..e592a3a 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetState.cc +++ b/components/security_apps/waap/waap_clib/WaapAssetState.cc @@ -783,6 +783,55 @@ WaapAssetState::filterKeywordsDueToLongText(Waf2ScanResult &res) const #endif } +// std::string nicePrint() - is a function used to create std::string that will represent all data that is +// collected inside Waf2ScanResult object. This function is used for debugging purposes. it should make deep-dive +// into the object easier. + +std::string +WaapAssetState::nicePrint(Waf2ScanResult &res) const +{ + std::string result = "Waf2ScanResult:\n"; + result += "keyword_matches:\n"; + for (const auto &keyword : res.keyword_matches) { + result += keyword + "\n"; + } + result += "regex_matches:\n"; + for (const auto ®ex : res.regex_matches) { + result += regex + "\n"; + } + result += "filtered_keywords:\n"; + for (const auto &filtered : res.filtered_keywords) { + result += filtered + "\n"; + } + result += "found_patterns:\n"; + for (const auto &pattern : res.found_patterns) { + result += pattern.first + ":\n"; + for (const auto &value : pattern.second) { + result += value + "\n"; + } + } + result += "unescaped_line: " + res.unescaped_line + "\n"; + result += "param_name: " + res.param_name + "\n"; + result += "location: " + res.location + "\n"; + result += "score: " + std::to_string(res.score) + "\n"; + result += "scoreNoFilter: " + std::to_string(res.scoreNoFilter) + "\n"; + result += "scoreArray:\n"; + for (const auto &score : res.scoreArray) { + result += std::to_string(score) + "\n"; + } + result += "keywordCombinations:\n"; + for (const auto &combination : res.keywordCombinations) { + result += combination + "\n"; + } + result += "attack_types:\n"; + for (const auto &attack : res.attack_types) { + result += attack + "\n"; + } + result += "m_isAttackInParam: " + std::to_string(res.m_isAttackInParam) + "\n"; + return result; +} + + bool checkBinaryData(const std::string &line, bool binaryDataFound) { @@ -1033,7 +1082,7 @@ WaapAssetState::apply( // Scan unescaped_line with aho-corasick once, and reuse it in multiple calls to checkRegex below // This is done to improve performance of regex matching. SampleValue unescapedLineSample(res.unescaped_line, m_Signatures->m_regexPreconditions); - + dbgTrace(D_WAAP_SAMPLE_SCAN) << "after doing second set of checkRegex calls..." << nicePrint(res); checkRegex( unescapedLineSample, m_Signatures->specific_acuracy_keywords_regex, @@ -1111,7 +1160,7 @@ WaapAssetState::apply( } bool os_cmd_ev = Waap::Util::find_in_map_of_stringlists_keys("os_cmd_ev", res.found_patterns); - + dbgTrace(D_WAAP_SAMPLE_SCAN) << "before evasion checking " << nicePrint(res); if (os_cmd_ev) { dbgTrace(D_WAAP_EVASIONS) << "os command evasion found"; @@ -1295,6 +1344,47 @@ WaapAssetState::apply( } } + bool path_traversal_ev = Waap::Util::find_in_map_of_stringlists_keys("path_traversal", res.found_patterns); + dbgTrace(D_WAAP_EVASIONS) + << "path_traversal_ev = " << path_traversal_ev + << " sample = " << res.unescaped_line + << " res.unescaped_line.find(2f) = " << res.unescaped_line.find("2f"); + if ((path_traversal_ev) && (res.unescaped_line.find("2f") != std::string::npos)) { + // Possible path traversal evasion .2f. detected: - clean up and scan with regexes again. + dbgTrace(D_WAAP_EVASIONS) << "comment evasion .2f. found" << res.unescaped_line + << "Status beroe evasion checking " << nicePrint(res); + + std::string unescaped = line; + replaceAll(unescaped, "2f", "/"); + size_t kwCount = res.keyword_matches.size(); + + if (res.unescaped_line != unescaped) { + SampleValue unescapedSample(unescaped, m_Signatures->m_regexPreconditions); + checkRegex(unescapedSample, m_Signatures->specific_acuracy_keywords_regex, res.keyword_matches, + res.found_patterns, longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->words_regex, res.keyword_matches, res.found_patterns, + longTextFound, binaryDataFound); + checkRegex(unescapedSample, m_Signatures->pattern_regex, res.regex_matches, res.found_patterns, + longTextFound, binaryDataFound); + } + + if (kwCount == res.keyword_matches.size()) { + // Remove the evasion keyword if no real evasion found + keywordsToRemove.push_back("path_traversal"); + path_traversal_ev = false; + } + else if (!binaryDataFound) { + // Recalculate repetition and/or probing indicators + unsigned int newWordsCount = 0; + calcRepetitionAndProbing(res, ignored_keywords, unescaped, detectedRepetition, detectedProbing, + newWordsCount); + // Take minimal words count because empirically it means evasion was probably succesfully decoded + wordsCount = std::min(wordsCount, newWordsCount); + } + dbgTrace(D_WAAP_EVASIONS) << "status after evasion checking " << nicePrint(res); + } + + bool quoutes_space_evasion = Waap::Util::find_in_map_of_stringlists_keys( "quotes_space_ev_fast_reg", res.found_patterns @@ -1726,7 +1816,7 @@ WaapAssetState::apply( wordsCount = std::min(wordsCount, newWordsCount); } } - + dbgTrace(D_WAAP_SAMPLE_SCAN) << "after evasions..." << nicePrint(res); // Remove evasion keywords that should not be reported because there's no real evasion found if (!keywordsToRemove.empty()) { dbgTrace(D_WAAP_SAMPLE_SCAN) diff --git a/components/security_apps/waap/waap_clib/WaapAssetState.h b/components/security_apps/waap/waap_clib/WaapAssetState.h index d760dab..22d730d 100755 --- a/components/security_apps/waap/waap_clib/WaapAssetState.h +++ b/components/security_apps/waap/waap_clib/WaapAssetState.h @@ -49,6 +49,7 @@ private: //ugly but needed for build Waap::Util::map_of_stringlists_t & found_patterns, bool longTextFound, bool binaryDataFound) const; void filterKeywordsDueToLongText(Waf2ScanResult &res) const; + std::string nicePrint(Waf2ScanResult &res) const; public: // Load and compile signatures from file diff --git a/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc b/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc deleted file mode 100644 index 2a92f43..0000000 --- a/components/security_apps/waap/waap_clib/WaapModelResultLogger.cc +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "WaapModelResultLogger.h" -#include "Waf2Engine.h" -#include "i_time_get.h" -#include "i_messaging.h" -#include "i_instance_awareness.h" -#include "http_manager.h" -#include "LogGenWrapper.h" -#include "rest.h" -#include "debug.h" - -USE_DEBUG_FLAG(D_WAAP_MODEL_LOGGER); - -using namespace std; - -static const unsigned int MAX_FILES_PER_WINDOW = 5; -static const unsigned int MAX_LOGS_PER_WINDOW = 1800; -static constexpr std::chrono::minutes RATE_LIMIT_WINDOW_MINUTES = std::chrono::minutes(30); - -class WaapModelReport : public RestGetFile -{ -public: - WaapModelReport(const vector &_data) : data(_data) {} - -private: - C2S_PARAM(vector, data); -}; - -class WaapModelResultLogger::Impl - : - Singleton::Provide::From -{ -public: - Impl(size_t maxLogs) : max_logs(maxLogs), sent_files_count(0), sent_logs_count(0), - last_sent_s3(std::chrono::minutes::zero()), - last_kusto_log_window(std::chrono::minutes::zero()) {} - virtual ~Impl(); - void - logModelResult( - Waap::Scores::ModelLoggingSettings &settings, - IWaf2Transaction* transaction, - Waf2ScanResult &res, - string modelName, - string otherModelName, - double score, - double otherScore) override; - -private: - void logToStream(WaapModelResult &result, chrono::minutes now); - void logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now); - bool shouldSendLogsToS3(chrono::minutes now); - void sendLogsToS3(); - size_t max_logs; - unsigned int sent_files_count; - unsigned int sent_logs_count; - std::chrono::minutes last_sent_s3; - std::chrono::minutes last_kusto_log_window; - std::map> logs; -}; - -WaapModelResultLogger::WaapModelResultLogger(size_t maxLogs) : pimpl(make_unique(maxLogs)) -{ -} - -WaapModelResultLogger::~WaapModelResultLogger() -{ -} - -void -WaapModelResultLogger::logModelResult( - Waap::Scores::ModelLoggingSettings &settings, - IWaf2Transaction* transaction, - Waf2ScanResult &res, - std::string modelName, - std::string otherModelName, - double score, - double otherScore -) -{ - pimpl->logModelResult(settings, transaction, res, modelName, otherModelName, score, otherScore); -} - -void -WaapModelResultLogger::Impl::logModelResult( - Waap::Scores::ModelLoggingSettings &settings, - IWaf2Transaction* transaction, - Waf2ScanResult &res, - string modelName, - string otherModelName, - double score, - double otherScore) -{ - if (transaction == NULL) return; - if (!Singleton::exists()) { - dbgError(D_WAAP_MODEL_LOGGER) << "Messaging service is not available, will not log"; - return; - } - - double score_diff = score - otherScore; - if (settings.logLevel == Waap::Scores::ModelLogLevel::DIFF && - ! ((score_diff > 0 && score >= 1.5f && otherScore < 4.0f) || - (score_diff < 0 && score < 4.0f && otherScore >= 1.5f))) { - return; - } - - auto current_time = Singleton::Consume::by()->getWalltime(); - auto now = chrono::duration_cast(current_time); - - WaapModelResult result = WaapModelResult( - *transaction, - res, - modelName, - otherModelName, - score, - otherScore, - now.count() - ); - - if (settings.logToStream) logToStream(result, now); - if (settings.logToS3) logToS3(result, transaction, now); -} - -void WaapModelResultLogger::Impl::logToS3(WaapModelResult &result, IWaf2Transaction* transaction, chrono::minutes now) -{ - auto asset_state = transaction->getAssetState(); - string asset_id = (asset_state != nullptr) ? asset_state->m_assetId : ""; - auto asset_logs = logs.find(asset_id); - if (asset_logs == logs.end()) { - logs.emplace(asset_id, vector()); - } - logs.at(asset_id).push_back(result); - if (shouldSendLogsToS3(now)) { - sendLogsToS3(); - } -} - -void WaapModelResultLogger::Impl::logToStream(WaapModelResult &result, chrono::minutes now) -{ - if (now - last_kusto_log_window > RATE_LIMIT_WINDOW_MINUTES) { - last_kusto_log_window = now; - sent_logs_count = 0; - } - else if (sent_logs_count > MAX_LOGS_PER_WINDOW) { - return; - } - sent_logs_count++; - dbgTrace(D_WAAP_MODEL_LOGGER) << "Logging WAAP model telemetry"; - - auto maybeLogTriggerConf = getConfiguration("rulebase", "log"); - LogGenWrapper logGenWrapper( - maybeLogTriggerConf, - "WAAP Model Telemetry", - ReportIS::Audience::SECURITY, - LogTriggerConf::SecurityType::ThreatPrevention, - ReportIS::Severity::CRITICAL, - ReportIS::Priority::HIGH, - false); - - LogGen& waap_log = logGenWrapper.getLogGen(); - waap_log.addMarkerSuffix(result.location); - waap_log << LogField("httpuripath", result.uri); - waap_log << LogField("matchedlocation", result.location); - waap_log << LogField("matchedparameter", result.param); - waap_log << LogField("matchedindicators", Waap::Util::vecToString(result.keywords), LogFieldOption::XORANDB64); - waap_log << LogField("matchedsample", result.sample, LogFieldOption::XORANDB64); - waap_log << LogField("waapkeywordsscore", (int)(result.otherScore * 100)); - waap_log << LogField("waapfinalscore", (int)(result.score * 100)); - waap_log << LogField("indicatorssource", result.modelName); - waap_log << LogField("indicatorsversion", result.otherModelName); -} - -bool WaapModelResultLogger::Impl::shouldSendLogsToS3(chrono::minutes now) -{ - if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) return true; - for (const auto &asset_logs : logs) { - if (asset_logs.second.size() >= max_logs) return true; - } - return false; -} - -void WaapModelResultLogger::Impl::sendLogsToS3() -{ - dbgFlow(D_WAAP_MODEL_LOGGER) << "Sending logs to fog"; - - I_Messaging *msg = Singleton::Consume::by(); - - for (auto &asset_logs : logs) { - if (asset_logs.second.empty()) { - continue; - } - if (sent_files_count >= MAX_FILES_PER_WINDOW) { - dbgInfo(D_WAAP_MODEL_LOGGER) << "Reached max files per window, will wait for next window"; - asset_logs.second.clear(); - continue; - } - I_AgentDetails *agentDetails = Singleton::Consume::by(); - string tenant_id = agentDetails->getTenantId(); - string agent_id = agentDetails->getAgentId(); - string asset_id = asset_logs.first; - if (Singleton::exists()) { - I_InstanceAwareness* instance = Singleton::Consume::by(); - Maybe uniqueId = instance->getUniqueID(); - if (uniqueId.ok()) - { - agent_id += "/" + uniqueId.unpack(); - } - } - string uri = "/storage/waap/" + - tenant_id + "/" + asset_id + "/waap_model_results/window_" + - to_string(last_sent_s3.count()) + "-" + to_string(sent_files_count) + - "/" + agent_id + "/data.data"; - WaapModelReport report = WaapModelReport(asset_logs.second); - - dbgInfo(D_WAAP_MODEL_LOGGER) << "Sending logs for asset " << asset_logs.first << - ", length " << asset_logs.second.size() << - ", uri " << uri; - msg->sendAsyncMessage( - HTTPMethod::PUT, - uri, - report, - MessageCategory::LOG - ); - - asset_logs.second.clear(); - } - - auto current_time = Singleton::Consume::by()->getWalltime(); - auto now = chrono::duration_cast(current_time); - if (now - last_sent_s3 > RATE_LIMIT_WINDOW_MINUTES) { - last_sent_s3 = now; - sent_files_count = 0; - } else { - sent_files_count++; - } -} - -WaapModelResultLogger::Impl::~Impl() -{} diff --git a/components/security_apps/waap/waap_clib/WaapModelResultLogger.h b/components/security_apps/waap/waap_clib/WaapModelResultLogger.h deleted file mode 100644 index 71a03e6..0000000 --- a/components/security_apps/waap/waap_clib/WaapModelResultLogger.h +++ /dev/null @@ -1,109 +0,0 @@ -// Copyright (C) 2024 Check Point Software Technologies Ltd. All rights reserved. - -// Licensed under the Apache License, Version 2.0 (the "License"); -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include "cereal/archives/json.hpp" - -#include "i_waap_model_result_logger.h" -#include "DeepAnalyzer.h" -#include "i_transaction.h" -#include "ScanResult.h" -#include "WaapAssetState.h" -#include "WaapScores.h" - -class WaapModelResultLogger - : - Singleton::Provide -{ -public: - WaapModelResultLogger(size_t maxLogs = MAX_WAAP_MODEL_LOGS); - virtual ~WaapModelResultLogger(); - virtual void logModelResult( - Waap::Scores::ModelLoggingSettings &settings, - IWaf2Transaction* transaction, - Waf2ScanResult &res, - std::string modelName, - std::string otherModelName, - double score, - double otherScore - ); - class Impl; - -protected: - std::unique_ptr pimpl; - static const size_t MAX_WAAP_MODEL_LOGS = 20000; -}; - -class WaapModelResult -{ -public: - WaapModelResult( - IWaf2Transaction &transaction, - Waf2ScanResult &res, - const std::string &modelName, - const std::string &otherModelName, - double score, - double otherScore, - uint64_t time - ) : uri(transaction.getUri()), location(res.location), param(res.param_name), - modelName(modelName), otherModelName(otherModelName), - score(score), otherScore(otherScore), keywords(res.keywordsAfterFilter), - sample(res.unescaped_line.substr(0, 100)), id(transaction.getIndex()), time(time) - { - } - - template - void serialize(Archive &ar) const - { - ar(cereal::make_nvp("uri", uri)); - ar(cereal::make_nvp("location", location)); - ar(cereal::make_nvp("param", param)); - ar(cereal::make_nvp("modelName", modelName)); - ar(cereal::make_nvp("otherModelName", otherModelName)); - ar(cereal::make_nvp("score", score)); - ar(cereal::make_nvp("otherScore", otherScore)); - ar(cereal::make_nvp("keywords", keywords)); - ar(cereal::make_nvp("sample", sample)); - ar(cereal::make_nvp("id", id)); - ar(cereal::make_nvp("time", time)); - } - - std::string toString() const - { - std::stringstream message_stream; - { - cereal::JSONOutputArchive ar(message_stream); - serialize(ar); - } - return message_stream.str(); - } - - std::string uri; - std::string location; - std::string param; - std::string modelName; - std::string otherModelName; - double score; - double otherScore; - std::vector keywords; - std::string sample; - uint64_t id; - uint64_t time; -}; diff --git a/components/security_apps/waap/waap_clib/WaapOverride.h b/components/security_apps/waap/waap_clib/WaapOverride.h index 024c656..77c76ae 100755 --- a/components/security_apps/waap/waap_clib/WaapOverride.h +++ b/components/security_apps/waap/waap_clib/WaapOverride.h @@ -48,8 +48,9 @@ public: m_tag = to_lower_copy(m_tag); if (m_tag != "sourceip" && m_tag != "sourceidentifier" && m_tag != "url" && m_tag != "hostname" && - m_tag != "keyword" && m_tag != "paramname" && m_tag != "paramvalue" && m_tag != "paramlocation" && - m_tag != "responsebody" && m_tag != "headername" && m_tag != "headervalue" && m_tag != "method") { + m_tag != "keyword" && m_tag != "indicator" && m_tag != "paramname" && m_tag != "paramvalue" && + m_tag != "paramlocation" && m_tag != "responsebody" && m_tag != "headername" && + m_tag != "headervalue" && m_tag != "method") { m_isValid = false; dbgDebug(D_WAAP_OVERRIDE) << "Invalid override tag: " << m_tag; } diff --git a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc index e0c686d..98f6dd7 100755 --- a/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc +++ b/components/security_apps/waap/waap_clib/WaapOverrideFunctor.cc @@ -105,7 +105,7 @@ bool WaapOverrideFunctor::operator()( } return false; } - else if (tagLower == "keyword") { + else if (tagLower == "keyword" || tagLower == "indicator") { for (const auto &rx : rxes) { for (const std::string& keywordStr : waf2Transaction.getKeywordMatches()) { if (REGX_MATCH(keywordStr)) { diff --git a/components/security_apps/waap/waap_clib/WaapScanner.cc b/components/security_apps/waap/waap_clib/WaapScanner.cc index 6f62499..9f81568 100755 --- a/components/security_apps/waap/waap_clib/WaapScanner.cc +++ b/components/security_apps/waap/waap_clib/WaapScanner.cc @@ -15,7 +15,6 @@ #include "WaapScores.h" #include "Waf2Engine.h" #include "i_transaction.h" -#include "WaapModelResultLogger.h" #include #include "debug.h" #include "reputation_features_events.h" @@ -111,10 +110,6 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN for (auto keyword : newKeywords) { res.keywordsAfterFilter.push_back(keyword); } - res.scoreArray.clear(); - res.coefArray.clear(); - res.keywordCombinations.clear(); - double res_score = getScoreFromPool(res, newKeywords, poolName); std::string other_pool_name = Waap::Scores::getOtherScorePoolName(); @@ -123,14 +118,10 @@ double Waap::Scanner::getScoreData(Waf2ScanResult& res, const std::string &poolN if (applyLearning && poolName != other_pool_name && modelLoggingSettings.logLevel != Waap::Scores::ModelLogLevel::OFF) { double other_score = getScoreFromPool(res, newKeywords, other_pool_name); - dbgDebug(D_WAAP_SCANNER) << "Comparing score from pool " << poolName << ": " << res_score << ", vs. pool " << other_pool_name << ": " << other_score << ", score difference: " << res_score - other_score << ", sample: " << res.unescaped_line; - Singleton::Consume::by()->logModelResult( - modelLoggingSettings, m_transaction, res, poolName, other_pool_name, res_score, other_score - ); res.other_model_score = other_score; } else { res.other_model_score = res_score; @@ -142,6 +133,9 @@ double Waap::Scanner::getScoreFromPool( Waf2ScanResult &res, const std::vector &newKeywords, const std::string &poolName ) { + res.scoreArray.clear(); + res.coefArray.clear(); + res.keywordCombinations.clear(); KeywordsStats stats = m_transaction->getAssetState()->scoreBuilder.getSnapshotStats(poolName); if (!newKeywords.empty()) { diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.cc b/components/security_apps/waap/waap_clib/Waf2Engine.cc index cadae7b..ea05e19 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.cc +++ b/components/security_apps/waap/waap_clib/Waf2Engine.cc @@ -1358,7 +1358,7 @@ Waf2Transaction::isHtmlType(const char* data, int data_len){ dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false"; return false; } - std::string body(data); + std::string body(data, data_len); if(!m_pWaapAssetState->getSignatures()->html_regex.hasMatch(body)) { dbgTrace(D_WAAP) << "Waf2Transaction::isHtmlType: false"; @@ -1661,6 +1661,9 @@ void Waf2Transaction::appendCommonLogFields(LogGen& waapLog, waapLog << LogField("sourcePort", m_remote_port); waapLog << LogField("httpHostName", m_hostStr); waapLog << LogField("httpMethod", m_methodStr); + if (!m_siteConfig->get_AssetId().empty()) waapLog << LogField("assetId", m_siteConfig->get_AssetId()); + if (!m_siteConfig->get_AssetName().empty()) waapLog << LogField("assetName", m_siteConfig->get_AssetName()); + const auto& autonomousSecurityDecision = std::dynamic_pointer_cast( m_waapDecision.getDecision(AUTONOMOUS_SECURITY_DECISION)); bool send_extended_log = shouldSendExtendedLog(triggerLog); @@ -2343,6 +2346,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { exceptions_dict["sourceIdentifier"].insert(m_source_identifier); exceptions_dict["url"].insert(getUriStr()); exceptions_dict["hostName"].insert(m_hostStr); + exceptions_dict["method"].insert(m_methodStr); for (auto &keyword : res.keyword_matches) { exceptions_dict["indicator"].insert(keyword); @@ -2355,8 +2359,9 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { auto behaviors = exceptions.unpack().getBehavior(exceptions_dict, getAssetState()->m_filtersMngr->getMatchedOverrideKeywords()); for (const auto &behavior : behaviors) { + dbgTrace(D_WAAP_OVERRIDE) << "got behavior: " << behavior.getId(); if (!res.filtered_keywords.empty() || res.score > 0) { - dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " with filtered indicators"; + dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "' with filtered indicators"; std::string overrideId = behavior.getId(); if (m_overrideOriginalMaxScore.find(overrideId) == m_overrideOriginalMaxScore.end()){ m_overrideOriginalMaxScore[overrideId] = res.scoreNoFilter; @@ -2375,7 +2380,7 @@ Waf2Transaction::shouldIgnoreOverride(const Waf2ScanResult &res) { } if (behavior == action_ignore) { - dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for " << res.param_name << " should ignore."; + dbgTrace(D_WAAP_OVERRIDE) << "matched exceptions for param '" << res.param_name << "': should ignore."; std::string overrideId = behavior.getId(); if (!overrideId.empty()) { m_matchedOverrideIds.insert(overrideId); diff --git a/components/security_apps/waap/waap_clib/Waf2Engine.h b/components/security_apps/waap/waap_clib/Waf2Engine.h index 6e1caa6..27f2b73 100755 --- a/components/security_apps/waap/waap_clib/Waf2Engine.h +++ b/components/security_apps/waap/waap_clib/Waf2Engine.h @@ -41,7 +41,6 @@ #include "i_waap_telemetry.h" #include "i_deepAnalyzer.h" #include "i_time_get.h" -#include "i_waap_model_result_logger.h" #include "table_opaque.h" #include "WaapResponseInspectReasons.h" #include "WaapResponseInjectReasons.h" diff --git a/components/security_apps/waap/waap_clib/Waf2Util.cc b/components/security_apps/waap/waap_clib/Waf2Util.cc index d360bfa..e22fdc3 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util.cc @@ -35,6 +35,7 @@ #include "user_identifiers_config.h" #include "Waf2Regex.h" #include "ParserBinaryFile.h" +#include "ParserKnownBenignSkipper.h" using boost::algorithm::to_lower_copy; using namespace std; @@ -1218,21 +1219,21 @@ static const SingleRegex csp_report_policy_re( "csp_report_policy" ); static const SingleRegex base64_key_value_detector_re( - "^[^<>{};,&\\?|=\\s]+={1}\\s*.+", - err, - "base64_key_value"); + "^[^<>{};,&\\?|=\\s]+={1}\\s*.+", + err, + "base64_key_value"); static const SingleRegex json_key_value_detector_re( "\\A[^<>{};,&\\?|=\\s]+=[{\\[][^;\",}\\]]*[,:\"].+[\\s\\S]", - err, - "json_key_value"); + err, + "json_key_value"); static const SingleRegex base64_key_detector_re( - "^[^<>{};,&\\?|=\\s]+={1}", - err, - "base64_key"); + "^[^<>{};,&\\?|=\\s]+={1}", + err, + "base64_key"); static const SingleRegex base64_prefix_detector_re( - "data:\\S*;base64,\\S+|base64,\\S+", - err, - "base64_prefix"); + "data:\\S*;base64,\\S+|base64,\\S+", + err, + "base64_prefix"); // looks for combination ={*:*} //used to allow parsing param=JSON to reduce false positives diff --git a/components/security_apps/waap/waap_clib/Waf2Util.h b/components/security_apps/waap/waap_clib/Waf2Util.h index 48542dd..d2d8cdf 100755 --- a/components/security_apps/waap/waap_clib/Waf2Util.h +++ b/components/security_apps/waap/waap_clib/Waf2Util.h @@ -894,6 +894,16 @@ namespace Util { bool isValidJson(const std::string &input); + enum KnownSourceType { + SOURCE_TYPE_UNKNOWN = 0, + SOURCE_TYPE_SENSOR_DATA = 1 + }; + + KnownSourceType detectKnownSource(const std::string &input); + bool isScreenedJson(const std::string &input); + + int definePrefixedJson(const std::string &input); + bool detectJSONasParameter(const std::string &s, std::string &key, std::string &value); diff --git a/components/security_apps/waap/waap_clib/Waf2Util2.cc b/components/security_apps/waap/waap_clib/Waf2Util2.cc index 63b50cc..b4860cf 100644 --- a/components/security_apps/waap/waap_clib/Waf2Util2.cc +++ b/components/security_apps/waap/waap_clib/Waf2Util2.cc @@ -1,5 +1,7 @@ #include "Waf2Util.h" +#include "Waf2Regex.h" #include +#include "debug.h" namespace Waap { namespace Util { @@ -628,5 +630,52 @@ isValidJson(const std::string &input) return false; } +KnownSourceType +detectKnownSource(const std::string &input) +{ + static bool err = false; + static const SingleRegex known_source_sensor_data_re( + "^\\{\\\"sensor_data\\\":\\\"", + err, + "known_source_sensor_data" + ); + if (known_source_sensor_data_re.hasMatch(input)) { + return SOURCE_TYPE_SENSOR_DATA; + } + return SOURCE_TYPE_UNKNOWN; +} + +int +definePrefixedJson(const std::string &input) +{ + static const size_t MAX_JSON_PREFIX_LEN = 32; + static const size_t MIN_PARAMETER_LEN = 4; + if (input.size() < MIN_PARAMETER_LEN) { + return -1; + } + + for (size_t i = 0; i < std::min(input.size(), MAX_JSON_PREFIX_LEN) - 2 ; ++i) { + if (input[i] == '-' && input[i+1] == '{') return i + 1; + } + + return -1; +} + +bool +isScreenedJson(const std::string &input) +{ + static bool err = false; + static const SingleRegex screened_json_re( + R"(^"{\s*\\"\w+\\"\s*:\s*\\"["\w])", + err, + "screened_json" + ); + + if (screened_json_re.hasMatch(input)) { + return true; + } + return false; +} + } // namespace Util } // namespace Waap diff --git a/components/security_apps/waap/waap_component_impl.cc b/components/security_apps/waap/waap_component_impl.cc index 72db6ca..0fa0f26 100755 --- a/components/security_apps/waap/waap_component_impl.cc +++ b/components/security_apps/waap/waap_component_impl.cc @@ -50,8 +50,7 @@ WaapComponent::Impl::Impl() : drop_response(ngx_http_cp_verdict_e::TRAFFIC_VERDICT_DROP), waapStateTable(NULL), transactionsCount(0), - deepAnalyzer(), - waapModelResultLogger() + deepAnalyzer() { } @@ -536,9 +535,15 @@ WaapComponent::Impl::respond(const HttpResponseBodyEvent &event) verdict = drop_response.getVerdict(); } + bool sould_inject_response = waf2Transaction.shouldInjectResponse(); + // in Chunked transfer encoding the last chunk is always empty - and we leave it empty + bool should_stay_empty_chunk = event.isLastChunk() && dataBufLen == 0; + dbgTrace(D_WAAP) + << (sould_inject_response ? "should Inject Response" : "should not Inject Response") + << (should_stay_empty_chunk ? " empty last chunk will stay empty" : ""); if (verdict == pending_response.getVerdict() && - waf2Transaction.shouldInjectResponse() && - !event.isLastChunk() + sould_inject_response && + !should_stay_empty_chunk ) { // Inject if needed. Note that this is only reasonable to do if there was no DROP decision above diff --git a/components/security_apps/waap/waap_component_impl.h b/components/security_apps/waap/waap_component_impl.h index 33c8551..91ab2c3 100755 --- a/components/security_apps/waap/waap_component_impl.h +++ b/components/security_apps/waap/waap_component_impl.h @@ -19,7 +19,6 @@ #include "table_opaque.h" #include "i_transaction.h" #include "waap_clib/DeepAnalyzer.h" -#include "waap_clib/WaapModelResultLogger.h" #include "waap_clib/WaapAssetState.h" #include "waap_clib/WaapAssetStatesManager.h" #include "reputation_features_agg.h" @@ -81,7 +80,6 @@ private: uint64_t transactionsCount; // instance of singleton classes DeepAnalyzer deepAnalyzer; - WaapModelResultLogger waapModelResultLogger; WaapAssetStatesManager waapAssetStatesManager; std::unordered_set m_seen_assets_id; }; diff --git a/components/utils/generic_rulebase/match_query.cc b/components/utils/generic_rulebase/match_query.cc index ab37c09..53b350a 100644 --- a/components/utils/generic_rulebase/match_query.cc +++ b/components/utils/generic_rulebase/match_query.cc @@ -111,7 +111,7 @@ MatchQuery::load(cereal::JSONInputArchive &archive_in) is_specific_label = false; } } - is_ignore_keyword = (key == "indicator"); + is_ignore_keyword = (key == "indicator" || key == "keyword"); if (condition_type != Conditions::Exist) { archive_in(cereal::make_nvp("value", value)); @@ -244,9 +244,10 @@ MatchQuery::getAllKeys() const bool MatchQuery::matchAttributes( const unordered_map> &key_value_pairs, - set &matched_override_keywords ) const + set &matched_override_keywords) const { + dbgTrace(D_RULEBASE_CONFIG) << "Start matching attributes"; if (type == MatchType::Condition) { auto key_value_pair = key_value_pairs.find(key); if (key_value_pair == key_value_pairs.end()) { @@ -257,9 +258,11 @@ MatchQuery::matchAttributes( } else if (type == MatchType::Operator && operator_type == Operators::And) { for (const MatchQuery &inner_match: items) { if (!inner_match.matchAttributes(key_value_pairs, matched_override_keywords)) { + dbgTrace(D_RULEBASE_CONFIG) << "Failed to match attributes for AND operator"; return false; } } + dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched all inner matches for AND operator"; return true; } else if (type == MatchType::Operator && operator_type == Operators::Or) { // With 'or' condition, evaluate matched override keywords first and add the ones that were fully matched @@ -272,6 +275,7 @@ MatchQuery::matchAttributes( res = true; } } + dbgTrace(D_RULEBASE_CONFIG) << "Match result for OR operator is: " << res; return res; } else { dbgWarning(D_RULEBASE_CONFIG) << "Unsupported match query type"; @@ -285,6 +289,7 @@ MatchQuery::getMatch( const unordered_map> &key_value_pairs) MatchQuery::MatchResult matches; matches.matched_keywords = make_shared>(); matches.is_match = matchAttributes(key_value_pairs, *matches.matched_keywords); + dbgTrace(D_RULEBASE_CONFIG) << "Match result: " << matches.is_match; return matches; } @@ -306,10 +311,13 @@ MatchQuery::matchAttributes( if (isKeyTypeIp()) { match = matchAttributesIp(values); + dbgTrace(D_RULEBASE_CONFIG) << "Match result for IP address: " << match; } else if (isRegEx()) { match = matchAttributesRegEx(values, matched_override_keywords); + dbgTrace(D_RULEBASE_CONFIG) << "Match result for regex: " << match; } else { match = matchAttributesString(values); + dbgTrace(D_RULEBASE_CONFIG) << "Match result for string: " << match; } return negate ? !match : match; @@ -324,6 +332,8 @@ MatchQuery::matchAttributesRegEx( boost::cmatch value_matcher; for (const boost::regex &val_regex : regex_values) { for (const string &requested_match_value : values) { + dbgTrace(D_RULEBASE_CONFIG) << "Matching value: '" << requested_match_value + << "' with regex: '" << val_regex << "'"; if (NGEN::Regex::regexMatch( __FILE__, __LINE__, diff --git a/components/utils/generic_rulebase/parameters_config.cc b/components/utils/generic_rulebase/parameters_config.cc index 7677b86..e2575a7 100644 --- a/components/utils/generic_rulebase/parameters_config.cc +++ b/components/utils/generic_rulebase/parameters_config.cc @@ -119,10 +119,13 @@ ParameterException::getBehavior( for (const MatchBehaviorPair &match_behavior_pair: match_queries) { MatchQuery::MatchResult match_res = match_behavior_pair.match.getMatch(key_value_pairs); if (match_res.is_match) { - dbgTrace(D_RULEBASE_CONFIG) << "Successfully matched an exception from a list of matches."; + dbgTrace(D_RULEBASE_CONFIG) + << "Successfully matched an exception from a list of matches, behavior: " + << match_behavior_pair.behavior.getId(); // When matching indicators with action=ignore, we expect no behavior override. // Instead, a matched keywords list should be returned which will be later removed from score calculation if (match_res.matched_keywords->size() > 0 && match_behavior_pair.behavior == action_ignore) { + dbgTrace(D_RULEBASE_CONFIG) << "Got action ignore"; matched_override_keywords.insert(match_res.matched_keywords->begin(), match_res.matched_keywords->end()); } else { diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index f3a45c7..f710371 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -37,7 +37,7 @@ target_link_libraries( ngen_core -Wl,-whole-archive "table;debug_is;shell_cmd;metric;tenant_manager;messaging;encryptor;time_proxy;singleton;mainloop;environment;logging;report;rest" - "config;intelligence_is_v2;event_is;memory_consumption;connkey" + "compression_utils;-lz;config;intelligence_is_v2;event_is;memory_consumption;connkey" "instance_awareness;socket_is;agent_details;agent_details_reporter;buffers;cpu;agent_core_utilities" "report_messaging" -Wl,-no-whole-archive diff --git a/core/agent_details_reporter/agent_details_reporter.cc b/core/agent_details_reporter/agent_details_reporter.cc index 7c8725c..f954c5d 100644 --- a/core/agent_details_reporter/agent_details_reporter.cc +++ b/core/agent_details_reporter/agent_details_reporter.cc @@ -117,7 +117,7 @@ private: return; } - dbgInfo(D_AGENT_DETAILS) + dbgDebug(D_AGENT_DETAILS) << "Successfully handled attributes persistence. Operation: " << operation << ", Path " diff --git a/core/include/services_sdk/interfaces/messaging/http_response.h b/core/include/services_sdk/interfaces/messaging/http_response.h index 5b4b813..6a782ec 100644 --- a/core/include/services_sdk/interfaces/messaging/http_response.h +++ b/core/include/services_sdk/interfaces/messaging/http_response.h @@ -31,15 +31,26 @@ public: HTTPResponse() = default; // LCOV_EXCL_STOP - HTTPResponse(HTTPStatusCode _status_code, const std::string &_body) : status_code(_status_code), body(_body) {} + HTTPResponse( + HTTPStatusCode _status_code, + const std::string &_body, + std::unordered_map _headers = std::unordered_map() + ) + : + status_code(_status_code), + body(_body), + headers(_headers) + {} HTTPStatusCode getHTTPStatusCode() const; const std::string & getBody() const; std::string toString() const; + Maybe getHeaderVal(const std::string &header_key); private: HTTPStatusCode status_code; std::string body; + std::unordered_map headers; }; #endif // __HTTP_RESPONSE_H__ diff --git a/core/include/services_sdk/interfaces/messaging/messaging_enums.h b/core/include/services_sdk/interfaces/messaging/messaging_enums.h index ef7b0e7..81f9702 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_enums.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_enums.h @@ -63,6 +63,7 @@ enum class HTTPStatusCode HTTP_PROXY_AUTHENTICATION_REQUIRED = 407, HTTP_REQUEST_TIME_OUT = 408, HTTP_PAYLOAD_TOO_LARGE = 413, + HTTP_TOO_MANY_REQUESTS = 429, // 5xx - Server error responses. HTTP_INTERNAL_SERVER_ERROR = 500, HTTP_NOT_IMPLEMENTED = 501, diff --git a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h index 6cf7ed9..57d0d66 100644 --- a/core/include/services_sdk/interfaces/messaging/messaging_metadata.h +++ b/core/include/services_sdk/interfaces/messaging/messaging_metadata.h @@ -8,6 +8,7 @@ #include "config.h" #include "singleton.h" #include "i_agent_details.h" +#include "i_time_get.h" class MessageProxySettings { @@ -54,7 +55,7 @@ private: uint16_t proxy_port = 0; }; -class MessageMetadata +class MessageMetadata : Singleton::Consume { public: inline MessageMetadata(); @@ -227,6 +228,26 @@ public: return sni_host_name; } + void + setRateLimitBlock(uint block_time) + { + is_rate_limit_block = true; + auto timer = Singleton::Consume::by(); + auto current_timeout = timer->getMonotonicTime() + std::chrono::seconds(block_time); + rate_limit_block_time = current_timeout.count(); + } + + bool + isRateLimitBlock() const + { + if (is_rate_limit_block) { + auto timer = Singleton::Consume::by(); + uint current_time = timer->getMonotonicTime().count(); + if (current_time < rate_limit_block_time) return true; + } + return false; + } + template void serialize(Archive &ar) @@ -243,7 +264,9 @@ public: cereal::make_nvp("is_to_fog", is_to_fog), cereal::make_nvp("ca_path", ca_path), cereal::make_nvp("client_cert_path", client_cert_path), - cereal::make_nvp("client_key_path", client_key_path) + cereal::make_nvp("client_key_path", client_key_path), + cereal::make_nvp("is_rate_limit_block", is_rate_limit_block), + cereal::make_nvp("rate_limit_block_time", rate_limit_block_time) ); } @@ -262,6 +285,8 @@ private: std::string external_certificate = ""; bool should_buffer = false; bool is_to_fog = false; + bool is_rate_limit_block = false; + uint rate_limit_block_time = 0; }; #endif // __MESSAGING_METADATA_H__ diff --git a/core/include/services_sdk/resources/debug_flags.h b/core/include/services_sdk/resources/debug_flags.h index 0e0ca5c..4fb78fb 100644 --- a/core/include/services_sdk/resources/debug_flags.h +++ b/core/include/services_sdk/resources/debug_flags.h @@ -107,6 +107,8 @@ DEFINE_FLAG(D_COMPONENT, D_ALL) DEFINE_FLAG(D_WAAP_PARSER_PAIRS, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_PDF, D_WAAP_PARSER) DEFINE_FLAG(D_WAAP_PARSER_BINARY_FILE, D_WAAP_PARSER) + DEFINE_FLAG(D_WAAP_PARSER_KNOWN_SOURCE_SKIPPER, D_WAAP_PARSER) + DEFINE_FLAG(D_WAAP_PARSER_SCREENED_JSON, D_WAAP_PARSER) DEFINE_FLAG(D_IPS, D_COMPONENT) DEFINE_FLAG(D_FILE_UPLOAD, D_COMPONENT) diff --git a/core/include/services_sdk/resources/generic_metric.h b/core/include/services_sdk/resources/generic_metric.h index 4414de3..6765af5 100644 --- a/core/include/services_sdk/resources/generic_metric.h +++ b/core/include/services_sdk/resources/generic_metric.h @@ -24,6 +24,7 @@ #include "i_mainloop.h" #include "i_time_get.h" #include "i_agent_details.h" +#include "i_encryptor.h" #include "i_instance_awareness.h" #include "i_environment.h" #include "i_messaging.h" @@ -57,6 +58,7 @@ class GenericMetric Singleton::Consume, Singleton::Consume, Singleton::Consume, + Singleton::Consume, public Listener { public: diff --git a/core/include/services_sdk/resources/metric/metric_calc.h b/core/include/services_sdk/resources/metric/metric_calc.h index 836f6dd..81728c3 100644 --- a/core/include/services_sdk/resources/metric/metric_calc.h +++ b/core/include/services_sdk/resources/metric/metric_calc.h @@ -23,6 +23,8 @@ #include "report/report.h" #include "customized_cereal_map.h" +#include "compression_utils.h" +#include "i_encryptor.h" class GenericMetric; @@ -56,6 +58,11 @@ public: value(_value) { timestamp = Singleton::Consume::by()->getWalltimeStr(); + // convert timestamp to RFC 3339 format + std::size_t pos = timestamp.find('.'); + if (pos != std::string::npos) { + timestamp = timestamp.substr(0, pos) + "Z"; + } asset_id = Singleton::Consume::by()->getAgentId(); } @@ -118,7 +125,73 @@ public: } // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage - std::string + Maybe + toString() const + { + std::stringstream ss; + { + cereal::JSONOutputArchive ar(ss); + serialize(ar); + } + auto res = compressAndEncodeData(ss.str()); + if (!res.ok()) { + return genError("Failed to compress and encode the data"); + } + return res.unpack(); + } +// LCOV_EXCL_STOP + +private: + Maybe + compressAndEncodeData(const std::string &unhandled_data) const + { + std::string data_holder = unhandled_data; + auto compression_stream = initCompressionStream(); + CompressionResult compression_response = compressData( + compression_stream, + CompressionType::GZIP, + data_holder.size(), + reinterpret_cast(data_holder.c_str()), + true + ); + finiCompressionStream(compression_stream); + if (!compression_response.ok) { + // send log to Kibana + return genError("Failed to compress(gzip) data"); + } + + std::string compressed_data = + std::string((const char *)compression_response.output, compression_response.num_output_bytes); + + auto encryptor = Singleton::Consume::by(); + Maybe handled_data = encryptor->base64Encode(compressed_data); + + if (compression_response.output) free(compression_response.output); + compression_response.output = nullptr; + compression_response.num_output_bytes = 0; + return handled_data; + } + + std::vector metrics; +}; + +class CompressAndEncodeAIOPSMetrics +{ +public: + CompressAndEncodeAIOPSMetrics(const AiopsMetricList &_aiops_metrics) : aiops_metrics(_aiops_metrics) {} + + void + serialize(cereal::JSONOutputArchive &ar) const + { + auto metric_str = aiops_metrics.toString(); + if (!metric_str.ok()) { + return; + } + ar(cereal::make_nvp("records", metric_str.unpack())); + } + +// LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage + Maybe toString() const { std::stringstream ss; @@ -131,7 +204,7 @@ public: // LCOV_EXCL_STOP private: - std::vector metrics; + AiopsMetricList aiops_metrics; }; class MetricCalc diff --git a/core/include/services_sdk/resources/rest.h b/core/include/services_sdk/resources/rest.h index a668bc9..c9a34fb 100644 --- a/core/include/services_sdk/resources/rest.h +++ b/core/include/services_sdk/resources/rest.h @@ -33,6 +33,8 @@ #include "maybe_res.h" #include "rest/schema_printer.h" +static const std::string BULK_ARRAY_NAME = "bulkArray"; + /// @class JsonError /// @brief Class representing JSON parsing errors. /// diff --git a/core/intelligence_is_v2/intelligence_comp_v2.cc b/core/intelligence_is_v2/intelligence_comp_v2.cc index 9e9aadc..9fa13c2 100644 --- a/core/intelligence_is_v2/intelligence_comp_v2.cc +++ b/core/intelligence_is_v2/intelligence_comp_v2.cc @@ -93,10 +93,13 @@ public: res << "\"name\": \"" << details->getAgentId() << "\", "; auto rest = Singleton::Consume::by(); res << "\"url\": \"http://127.0.0.1:" << rest->getListeningPort() <<"/set-new-invalidation\", "; + res << "\"capabilities\": { \"getBulkCallback\": " << "true" << " }, "; res << "\"dataMap\": ["; res << stream.str(); res << " ] }"; + dbgTrace(D_INTELLIGENCE) << res.str(); + return res; } @@ -148,13 +151,19 @@ public: void performCallBacks(const Invalidation &invalidation, const string ®istration_id) const override { - dbgDebug(D_INTELLIGENCE) << "Looking for callbacks for invalidation " << invalidation.genObject(); + dbgTrace(D_INTELLIGENCE) + << "Looking for callbacks for invalidation " + << invalidation.genObject() + << " registration id: " + << registration_id; if (registration_id != "") { auto invalidation_cb = registration_id_to_cb.find(registration_id); if (invalidation_cb != registration_id_to_cb.end()) return invalidation_cb->second(invalidation); } + dbgDebug(D_INTELLIGENCE) << "Have not found callback per registration id"; + for (auto ®isted_invalidation : callbacks) { - dbgTrace(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject(); + dbgDebug(D_INTELLIGENCE) << "Checking against: " << registed_invalidation.second.first.genObject(); performCallBacksImpl(invalidation, registed_invalidation.second); } } @@ -169,6 +178,7 @@ private: auto ®istereed_invalidation = invalidation_and_cb.first; auto &cb = invalidation_and_cb.second; if (!registereed_invalidation.matches(actual_invalidation)) return; + dbgTrace(D_INTELLIGENCE) << "Found a matching invalidation registration, should callback"; cb(actual_invalidation); } @@ -177,7 +187,7 @@ private: uint running_id = 0; }; -class ReceiveInvalidation : public ServerRest +class SingleReceivedInvalidation : public ServerRest { public: void @@ -248,6 +258,29 @@ private: C2S_OPTIONAL_PARAM(string, invalidationType); }; + +class ReceiveInvalidation : public ServerRest +{ +public: + + void + doCall() override + { + dbgTrace(D_INTELLIGENCE) + << (bulkArray.isActive() ? + "BULK invalidations, receiving invalidations in bulks" + : "error in format, expected bulk invalidations, not single"); + + for (SingleReceivedInvalidation &r : bulkArray.get()) { + r.doCall(); + } + return; + } + +private: + C2S_LABEL_PARAM(vector, bulkArray, BULK_ARRAY_NAME); +}; + class PagingController { public: diff --git a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc index 87bc63b..3895ad1 100644 --- a/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc +++ b/core/intelligence_is_v2/intelligence_is_v2_ut/invalidation_ut.cc @@ -486,6 +486,8 @@ TEST_F(IntelligenceInvalidation, register_for_invalidation) EXPECT_THAT(body, HasSubstr("\"mainAttributes\": [ { \"attr2\": \"2\" } ]")); EXPECT_THAT(body, HasSubstr("\"attributes\": [ { \"attr3\": \"3\" } ]")); EXPECT_TRUE(md.getConnectionFlags().isSet(MessageConnectionConfig::UNSECURE_CONN)); + EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }")); + } TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation) @@ -529,6 +531,7 @@ TEST_F(IntelligenceInvalidation, register_for_multiple_assets_invalidation) )); EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); + EXPECT_THAT(body, HasSubstr("\"capabilities\": { \"getBulkCallback\": true }")); EXPECT_THAT( body, @@ -606,7 +609,7 @@ TEST_F(IntelligenceInvalidation, invalidation_callback) .setObjectType(Intelligence::ObjectType::ASSET); stringstream json; - json << invalidation2.genObject(); + json << "[" << invalidation2.genObject() << "]"; mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 1u); @@ -650,7 +653,7 @@ TEST_F(IntelligenceInvalidation, delete_invalidation_callback) .setObjectType(Intelligence::ObjectType::ASSET); stringstream json; - json << invalidation2.genObject(); + json << "[" << invalidation2.genObject() << "]"; mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 0u); @@ -694,7 +697,7 @@ TEST_F(IntelligenceInvalidation, invalidation_short_handling) .setObjectType(Intelligence::ObjectType::ASSET); stringstream json; - json << invalidation2.genObject(); + json << "[" << invalidation2.genObject() << "]"; mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 0u); @@ -789,7 +792,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets) .setObjectType(Intelligence::ObjectType::ASSET); stringstream json1; - json1 << not_matching_invalidation.genObject(); + json1 << "[" << not_matching_invalidation.genObject() << "]"; mock_invalidation->performRestCall(json1); EXPECT_EQ(recieved_invalidations.size(), 0u); @@ -805,7 +808,7 @@ TEST_F(IntelligenceInvalidation, invalidation_flow_with_multiple_assets) .setObjectType(Intelligence::ObjectType::ASSET); stringstream json2; - json2 << matching_invalidation.genObject(); + json2 << "[" << matching_invalidation.genObject() << "]"; mock_invalidation->performRestCall(json2); EXPECT_EQ(recieved_invalidations.size(), 1u); @@ -865,7 +868,7 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_2_registred_assets) auto stop_listening_2 = make_scope_exit([&] { invalidation_2_to_register.stopListening(i_intelligence); }); stringstream json; - json << matching_invalidation.genObject(); + json << "[" << matching_invalidation.genObject() << "]"; mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 2u); @@ -927,9 +930,43 @@ TEST_F(IntelligenceInvalidation, invalidation_cb_match_by_registration_id) string modifiedJsonString = matching_invalidation.genObject().substr(2); stringstream json; - json << "{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString; + json << "[{ \"invalidationRegistrationId\": \""<< *registration_id << "\", " << modifiedJsonString << "]"; cout << json.str() << endl; mock_invalidation->performRestCall(json); EXPECT_EQ(recieved_invalidations.size(), 1u); } + +TEST_F(IntelligenceInvalidation, bulk_invalidation_callback) +{ + stringstream configuration; + configuration << "{"; + configuration << " \"agentSettings\":["; + configuration << " {\"key\":\"agent.config.useLocalIntelligence\",\"id\":\"id1\",\"value\":\"true\"}"; + configuration << " ],"; + configuration << " \"intelligence\":{"; + configuration << " \"local intelligence server ip\":\"127.0.0.1\","; + configuration << " \"local intelligence server primary port\":9090"; + configuration << " }"; + configuration << "}"; + Singleton::Consume::from(conf)->loadConfiguration(configuration); + + EXPECT_CALL( + messaging_mock, + sendSyncMessage(_, "/api/v2/intelligence/invalidation/register", _, _, _) + ).WillRepeatedly(Return(HTTPResponse(HTTPStatusCode::HTTP_OK, ""))); + + auto invalidation = Invalidation("agent_y"); + EXPECT_NE(i_intelligence->registerInvalidation(invalidation, callback), 0); + auto invalidation2 = Invalidation("agent2"); + EXPECT_NE(i_intelligence->registerInvalidation(invalidation2, callback), 0); + auto invalidation3 = Invalidation("agent3"); + EXPECT_NE(i_intelligence->registerInvalidation(invalidation3, callback), 0); + + dbgTrace(D_INTELLIGENCE) << "2 callbacks: "; + + stringstream json3; + json3 << "[{\"class\":\"agent3\"},{\"class\":\"agent2\"}]"; + mock_invalidation->performRestCall(json3); + EXPECT_EQ(recieved_invalidations.size(), 2u); +} diff --git a/core/messaging/include/response_parser.h b/core/messaging/include/response_parser.h index 7e86e87..d08adbe 100644 --- a/core/messaging/include/response_parser.h +++ b/core/messaging/include/response_parser.h @@ -25,7 +25,7 @@ private: bool isLegalChunkedResponse(const std::string &res); Maybe status_code = genError("Not received"); - Maybe> headers = genError("Not received"); + Maybe> headers = genError("Not received"); std::string body; std::string raw_response; bool error = false; diff --git a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc index 6505a48..63bdbbd 100644 --- a/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc +++ b/core/messaging/messaging_buffer_comp/messaging_buffer_comp.cc @@ -109,7 +109,7 @@ MessagingBufferComponent::Impl::init() encryptor = Singleton::Consume::by(); mainloop = Singleton::Consume::by(); messaging = Singleton::Consume::from(); - + auto sub_path = getProfileAgentSettingWithDefault("nano_agent/event_buffer/", "eventBuffer.baseFolder"); buffer_root_path = getLogFilesPathConfig() + "/" + sub_path; string executable_name = @@ -276,11 +276,18 @@ MessagingBufferComponent::Impl::sendMessage() } if (res == HTTPStatusCode::HTTP_SUSPEND) { - dbgDebug(D_MESSAGING) << "Suspended connection - sleeping for a while"; + dbgDebug(D_MESSAGING) << "Message is Suspended - sleeping for a while"; mainloop->yield(chrono::seconds(1)); return true; } + if (res == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) { + dbgDebug(D_MESSAGING) << "Suspended message due to rate limit block - sleeping for a while"; + mainloop->yield(chrono::seconds(1)); + popMessage(); + return true; + } + ++curr_no_retries; if (curr_no_retries >= getProfileAgentSettingWithDefault(10, "eventBuffer.maxNumOfSendigRetries")) { dbgWarning(D_MESSAGING) << "Reached maximum number of retries - poping message"; @@ -295,6 +302,12 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons { MessageMetadata message_metadata = message.getMessageMetadata(); message_metadata.setShouldBufferMessage(false); + + if (message_metadata.isRateLimitBlock()) { + mainloop->yield(chrono::seconds(1)); + return HTTPStatusCode::HTTP_SUSPEND; + } + auto res = messaging->sendSyncMessage( message.getMethod(), message.getURI(), @@ -305,6 +318,9 @@ MessagingBufferComponent::Impl::sendMessage(const BufferedMessage &message) cons if (res.ok()) return HTTPStatusCode::HTTP_OK; if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_SUSPEND) return HTTPStatusCode::HTTP_SUSPEND; + if (res.getErr().getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) { + return HTTPStatusCode::HTTP_TOO_MANY_REQUESTS; + } return HTTPStatusCode::HTTP_UNKNOWN; } diff --git a/core/messaging/messaging_comp/http_response.cc b/core/messaging/messaging_comp/http_response.cc index 37440fc..babbc96 100644 --- a/core/messaging/messaging_comp/http_response.cc +++ b/core/messaging/messaging_comp/http_response.cc @@ -36,6 +36,7 @@ static const map status_code_to_string = { { HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED, "407 - HTTP_PROXY_AUTHENTICATION_REQUIRED" }, { HTTPStatusCode::HTTP_REQUEST_TIME_OUT, "408 - HTTP_REQUEST_TIME_OUT" }, { HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE, "413 - HTTP_PAYLOAD_TOO_LARGE" }, + { HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, "429 - HTTP_TOO_MANY_REQUESTS" }, { HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR, "500 - HTTP_INTERNAL_SERVER_ERROR" }, { HTTPStatusCode::HTTP_NOT_IMPLEMENTED, "501 - HTTP_NOT_IMPLEMENTED" }, { HTTPStatusCode::HTTP_BAD_GATEWAY, "502 - HTTP_BAD_GATEWAY" }, @@ -63,6 +64,7 @@ static const map num_to_status_code = { { 407, HTTPStatusCode::HTTP_PROXY_AUTHENTICATION_REQUIRED }, { 408, HTTPStatusCode::HTTP_REQUEST_TIME_OUT }, { 413, HTTPStatusCode::HTTP_PAYLOAD_TOO_LARGE }, + { 429, HTTPStatusCode::HTTP_TOO_MANY_REQUESTS }, { 500, HTTPStatusCode::HTTP_INTERNAL_SERVER_ERROR }, { 501, HTTPStatusCode::HTTP_NOT_IMPLEMENTED }, { 502, HTTPStatusCode::HTTP_BAD_GATEWAY }, @@ -100,6 +102,16 @@ HTTPResponse::toString() const return "[Status-code]: " + code->second + ", [Body]: " + (body.empty() ? "{}" : body); } +Maybe +HTTPResponse::getHeaderVal(const string &header_key) +{ + auto header = headers.find(header_key); + if (header == headers.end()) { + return genError("Header \'" + header_key + "\' not found."); + } + return header->second; +} + Maybe HTTPResponseParser::parseData(const string &data, bool is_connect) { @@ -116,7 +128,7 @@ HTTPResponseParser::parseData(const string &data, bool is_connect) if (!handleBody(is_connect)) return genError("Response not ready!"); - return HTTPResponse(status_code.unpack(), body); + return HTTPResponse(status_code.unpack(), body, headers.unpack()); } static string @@ -133,7 +145,7 @@ bool HTTPResponseParser::handleHeaders() { stringstream ss(raw_response); - map header_map; + unordered_map header_map; while (true) { string header; @@ -171,7 +183,7 @@ HTTPResponseParser::getHeaderVal(const string &header_key) auto headers_map = headers.unpack(); auto header = headers_map.find(header_key); if (header == headers_map.end()) { - return genError("Header\'" + header_key + "\' not found."); + return genError("Header \'" + header_key + "\' not found."); } return header->second; } diff --git a/core/messaging/messaging_comp/messaging_comp.cc b/core/messaging/messaging_comp/messaging_comp.cc index e48a8c7..c5ddb40 100644 --- a/core/messaging/messaging_comp/messaging_comp.cc +++ b/core/messaging/messaging_comp/messaging_comp.cc @@ -148,6 +148,21 @@ MessagingComp::sendMessage( auto response = i_conn->sendRequest(conn, *req); if (!response.ok()) return response.passErr(); + auto response_data = response.unpack(); + + if (response_data.getHTTPStatusCode() == HTTPStatusCode::HTTP_TOO_MANY_REQUESTS) { + dbgDebug(D_MESSAGING) << "Too many requests. Suspend the message"; + auto rate_limit_metadata = message_metadata; + uint retry_after_sec = 60; + auto retry_after_header = response_data.getHeaderVal("retry-after"); + if (retry_after_header.ok()) { + retry_after_sec = stoi(*retry_after_header); + } + rate_limit_metadata.setShouldBufferMessage(true); + rate_limit_metadata.setRateLimitBlock(retry_after_sec); + return suspendMessage(body, method, uri, category, rate_limit_metadata); + } + if (is_to_fog && method == HTTPMethod::GET) fog_get_requests_cache.emplaceEntry(uri, *response); return response; } @@ -355,6 +370,15 @@ MessagingComp::suspendMessage( const MessageMetadata &message_metadata ) const { + if (message_metadata.isRateLimitBlock()) { + dbgInfo(D_MESSAGING) << "Rate limit block is active, message is suspended, message is buffered."; + i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); + return genError( + HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, + "The connection is suspended due to rate limit block, message is buffered." + ); + } + if (message_metadata.shouldBufferMessage()) { dbgWarning(D_MESSAGING) << "Buffering message due to connection suspended"; i_messaging_buffer->pushNewBufferedMessage(body, method, uri, category, message_metadata, false); diff --git a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc index a90a4d3..dae991f 100644 --- a/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc +++ b/core/messaging/messaging_comp/messaging_comp_ut/messaging_comp_ut.cc @@ -276,3 +276,30 @@ TEST_F(TestMessagingComp, testSetFogConnection) EXPECT_CALL(mock_messaging_connection, establishConnection(metadata, category)).WillOnce(Return(conn)); EXPECT_TRUE(messaging_comp.setFogConnection(category)); } + +TEST_F(TestMessagingComp, testRateLimitBlock) +{ + setAgentDetails(); + string body = "test body"; + HTTPMethod method = HTTPMethod::POST; + string uri = "/test-uri"; + MessageCategory category = MessageCategory::GENERIC; + + MessageConnectionKey conn_key(fog_addr, fog_port, MessageCategory::GENERIC); + Flags conn_flags; + conn_flags.setFlag(MessageConnectionConfig::UNSECURE_CONN); + MessageMetadata conn_metadata(fog_addr, fog_port, conn_flags, false, true); + Connection conn(conn_key, conn_metadata); + + EXPECT_CALL(mock_messaging_connection, getFogConnectionByCategory(MessageCategory::GENERIC)) + .WillOnce(Return(conn)); + + unordered_map res_headers = {{"Retry-After", "10"}}; + HTTPResponse res(HTTPStatusCode::HTTP_TOO_MANY_REQUESTS, "response!!", res_headers); + EXPECT_CALL(mock_messaging_connection, mockSendRequest(_, _, _)).WillOnce(Return(res)); + auto sending_res = messaging_comp.sendSyncMessage(method, uri, body, category, conn_metadata); + ASSERT_FALSE(sending_res.ok()); + HTTPResponse http_res = sending_res.getErr(); + EXPECT_EQ(http_res.getBody(), "The connection is suspended due to rate limit block, message is buffered."); + EXPECT_EQ(http_res.getHTTPStatusCode(), HTTPStatusCode::HTTP_TOO_MANY_REQUESTS); +} diff --git a/core/metric/generic_metric.cc b/core/metric/generic_metric.cc index 39ed949..ef64e25 100644 --- a/core/metric/generic_metric.cc +++ b/core/metric/generic_metric.cc @@ -32,7 +32,10 @@ MetricMetadata::Units operator"" _unit(const char *str, size_t) { return MetricM MetricMetadata::Description operator"" _desc(const char *str, size_t) { return MetricMetadata::Description{str}; } // LCOV_EXCL_START Reason: Tested in unit test (testAIOPSMapMetric), but not detected by coverage -static ostream & operator<<(ostream &os, const AiopsMetricList &metrics) { return os << metrics.toString(); } +static ostream & operator<<(ostream &os, const CompressAndEncodeAIOPSMetrics &metrics) +{ + return os << metrics.toString(); +} // LCOV_EXCL_STOP vector @@ -44,8 +47,8 @@ MetricCalc::getAiopsMetrics() const string name = getMetricDotName() != "" ? getMetricDotName() : getMetricName(); string units = getMetircUnits(); string description = getMetircDescription(); - string type = getMetricType() == MetricType::GAUGE ? "gauge" : "counter"; - + string type = getMetricType() == MetricType::GAUGE ? "Gauge" : "Counter"; + return { AiopsMetricData(name, type, units, description, getBasicLabels(), value) }; } @@ -320,17 +323,17 @@ class PrometheusRest : public ClientRest public: Metric(const string &n, const string &t, const string &d, const string &l, const string &v) : - name(n), - type(t), - description(d), + metric_name(n), + metric_type(t), + metric_description(d), labels(l), value(v) {} private: - C2S_PARAM(string, name); - C2S_PARAM(string, type); - C2S_PARAM(string, description); + C2S_PARAM(string, metric_name); + C2S_PARAM(string, metric_type); + C2S_PARAM(string, metric_description); C2S_PARAM(string, labels); C2S_PARAM(string, value); }; @@ -356,6 +359,7 @@ void GenericMetric::generatePrometheus() { if (!getProfileAgentSettingWithDefault(false, "prometheus")) return; + dbgTrace(D_METRICS) << "Generate prometheus metric"; vector all_metrics; for (auto &calc : calcs) { @@ -371,7 +375,7 @@ GenericMetric::generatePrometheus() new_config_req_md.setConnectioFlag(MessageConnectionConfig::UNSECURE_CONN); Singleton::Consume::by()->sendSyncMessage( HTTPMethod::POST, - "/set-prometheus-data", + "/add-metrics", rest, MessageCategory::GENERIC, new_config_req_md @@ -382,6 +386,7 @@ void GenericMetric::generateAiopsLog() { if (!getConfigurationWithDefault(true, "metric", "aiopsMetricSendEnable")) return; + dbgTrace(D_METRICS) << "Generate AIOPS metric"; AiopsMetricList aiops_metrics; @@ -397,18 +402,17 @@ GenericMetric::generateAiopsLog() Level::LOG, LogLevel::INFO, audience, - team, + ReportIS::AudienceTeam::HORIZON_TELEMETRY, Severity::INFO, Priority::LOW, report_interval, LogField("agentId", Singleton::Consume::by()->getAgentId()), tags, Tags::INFORMATIONAL, - issuing_engine + ReportIS::IssuingEngine::HORIZON_TELEMETRY_METRICS ); - metric_to_fog << LogField("eventObject", aiops_metrics); - + metric_to_fog << LogField("eventObject", CompressAndEncodeAIOPSMetrics(aiops_metrics)); LogRest metric_client_rest(metric_to_fog); sendLog(metric_client_rest); } diff --git a/core/metric/metric_ut/metric_ut.cc b/core/metric/metric_ut/metric_ut.cc index a8d145a..13af6db 100644 --- a/core/metric/metric_ut/metric_ut.cc +++ b/core/metric/metric_ut/metric_ut.cc @@ -9,6 +9,7 @@ #include "mock/mock_time_get.h" #include "mock/mock_rest_api.h" #include "agent_details.h" +#include "mock/mock_encryptor.h" #include "mock/mock_messaging.h" #include "mock/mock_instance_awareness.h" #include "config.h" @@ -219,6 +220,7 @@ public: ::Environment env; ConfigComponent conf; AgentDetails agent_details; + StrictMock mock_encryptor; NiceMock messaging_mock; stringstream debug_output; I_MainLoop::Routine routine; @@ -556,7 +558,7 @@ TEST_F(MetricTest, printPromeathus) cpu_event.notify(); string message_body; - EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _)) + EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _)) .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse()))); routine(); @@ -564,44 +566,44 @@ TEST_F(MetricTest, printPromeathus) "{\n" " \"metrics\": [\n" " {\n" - " \"name\": \"cpuMax\",\n" - " \"type\": \"gauge\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuMax\",\n" + " \"metric_type\": \"gauge\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"89\"\n" " },\n" " {\n" - " \"name\": \"cpuMin\",\n" - " \"type\": \"gauge\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuMin\",\n" + " \"metric_type\": \"gauge\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"89\"\n" " },\n" " {\n" - " \"name\": \"cpuAvg\",\n" - " \"type\": \"gauge\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuAvg\",\n" + " \"metric_type\": \"gauge\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"89\"\n" " },\n" " {\n" - " \"name\": \"cpuCurrent\",\n" - " \"type\": \"gauge\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuCurrent\",\n" + " \"metric_type\": \"gauge\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"89\"\n" " },\n" " {\n" - " \"name\": \"cpuCounter\",\n" - " \"type\": \"gauge\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuCounter\",\n" + " \"metric_type\": \"gauge\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"1\"\n" " },\n" " {\n" - " \"name\": \"cpuTotalCounter\",\n" - " \"type\": \"counter\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"cpuTotalCounter\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"}\",\n" " \"value\": \"1\"\n" " }\n" @@ -636,7 +638,7 @@ TEST_F(MetricTest, printPromeathusMultiMap) HttpTransaction("/index.html", "POST", 40).notify(); string message_body; - EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/set-prometheus-data", _, _, _)) + EXPECT_CALL(messaging_mock, sendSyncMessage(_, "/add-metrics", _, _, _)) .WillOnce(DoAll(SaveArg<2>(&message_body), Return(HTTPResponse()))); routine(); @@ -644,25 +646,25 @@ TEST_F(MetricTest, printPromeathusMultiMap) "{\n" " \"metrics\": [\n" " {\n" - " \"name\": \"request.total\",\n" - " \"type\": \"counter\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"request.total\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," "method=\\\"GET\\\",url=\\\"/index.html\\\"}\",\n" " \"value\": \"1\"\n" " },\n" " {\n" - " \"name\": \"request.total\",\n" - " \"type\": \"counter\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"request.total\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," "method=\\\"POST\\\",url=\\\"/index.html\\\"}\",\n" " \"value\": \"1\"\n" " },\n" " {\n" - " \"name\": \"request.total\",\n" - " \"type\": \"counter\",\n" - " \"description\": \"\",\n" + " \"metric_name\": \"request.total\",\n" + " \"metric_type\": \"counter\",\n" + " \"metric_description\": \"\",\n" " \"labels\": \"{agent=\\\"Unknown\\\",id=\\\"87\\\"," "method=\\\"GET\\\",url=\\\"/index2.html\\\"}\",\n" " \"value\": \"1\"\n" @@ -1396,6 +1398,8 @@ TEST_F(MetricTest, testManyValuesOutOfOrder) TEST_F(MetricTest, basicAIOPSMetricTest) { EXPECT_CALL(timer, getWalltimeStr()).WillRepeatedly(Return(string("2016-11-13T17:31:24.087"))); + EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload")); + CPUMetric cpu_mt; cpu_mt.init( "CPU usage", @@ -1452,14 +1456,14 @@ TEST_F(MetricTest, basicAIOPSMetricTest) " \"eventLevel\": \"Log\",\n" " \"eventLogLevel\": \"info\",\n" " \"eventAudience\": \"Internal\",\n" - " \"eventAudienceTeam\": \"Agent Core\",\n" + " \"eventAudienceTeam\": \"\",\n" " \"eventFrequency\": 5,\n" " \"eventTags\": [\n" " \"Informational\"\n" " ],\n" " \"eventSource\": {\n" " \"agentId\": \"Unknown\",\n" - " \"issuingEngine\": \"Agent Core\",\n" + " \"issuingEngine\": \"horizonTelemetryMetrics\",\n" " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" @@ -1469,92 +1473,7 @@ TEST_F(MetricTest, basicAIOPSMetricTest) " },\n" " \"eventData\": {\n" " \"eventObject\": {\n" - " \"Metrics\": [\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpu.max\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"percrnt\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 89.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpuMin\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 89.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpuAvg\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 89.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpuCurrent\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 89.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpuCounter\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 1.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"cpuTotalCounter\",\n" - " \"MetricType\": \"counter\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 1.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {},\n" - " \"AssetID\": \"Unknown\"\n" - " }\n" - " ]\n" + " \"records\": \"compress and encode metric payload\"\n" " }\n" " }\n" " }\n" @@ -1595,8 +1514,68 @@ TEST_F(MetricTest, testAIOPSMapMetric) _, _ )).WillRepeatedly(SaveArg<2>(&message_body)); + EXPECT_CALL(mock_encryptor, base64Encode(_)).WillRepeatedly(Return("compress and encode metric payload")); routine(); + // aiops data example + // " \"Metrics\": [\n" + // " {\n" + // " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n" + // " \"MetricName\": \"/index.html\",\n" + // " \"MetricType\": \"Gauge\",\n" + // " \"MetricUnit\": \"\",\n" + // " \"MetricDescription\": \"\",\n" + // " \"MetricValue\": 0.0,\n" + // " \"ResourceAttributes\": {},\n" + // " \"MetricAttributes\": {\n" + // " \"key1\": \"value1\",\n" + // " \"key2\": \"value2\"\n" + // " },\n" + // " \"AssetID\": \"Unknown\"\n" + // " },\n" + // " {\n" + // " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n" + // " \"MetricName\": \"/index2.html\",\n" + // " \"MetricType\": \"Gauge\",\n" + // " \"MetricUnit\": \"\",\n" + // " \"MetricDescription\": \"\",\n" + // " \"MetricValue\": 0.0,\n" + // " \"ResourceAttributes\": {},\n" + // " \"MetricAttributes\": {\n" + // " \"key1\": \"value1\",\n" + // " \"key2\": \"value2\"\n" + // " },\n" + // " \"AssetID\": \"Unknown\"\n" + // " },\n" + // " {\n" + // " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n" + // " \"MetricName\": \"/index.html\",\n" + // " \"MetricType\": \"Counter\",\n" + // " \"MetricUnit\": \"\",\n" + // " \"MetricDescription\": \"\",\n" + // " \"MetricValue\": 0.0,\n" + // " \"ResourceAttributes\": {},\n" + // " \"MetricAttributes\": {},\n" + // " \"AssetID\": \"Unknown\"\n" + // " },\n" + // " {\n" + // " \"Timestamp\": \"2016-11-13T17:31:24Z\",\n" + // " \"MetricName\": \"/index2.html\",\n" + // " \"MetricType\": \"Counter\",\n" + // " \"MetricUnit\": \"\",\n" + // " \"MetricDescription\": \"\",\n" + // " \"MetricValue\": 0.0,\n" + // " \"ResourceAttributes\": {},\n" + // " \"MetricAttributes\": {},\n" + // " \"AssetID\": \"Unknown\"\n" + // " }\n" + // " ]\n" + // " }\n" + // " }\n" + // " }\n" + // "}"; + + string expected_message = "{\n" " \"log\": {\n" @@ -1608,14 +1587,14 @@ TEST_F(MetricTest, testAIOPSMapMetric) " \"eventLevel\": \"Log\",\n" " \"eventLogLevel\": \"info\",\n" " \"eventAudience\": \"Internal\",\n" - " \"eventAudienceTeam\": \"Agent Core\",\n" + " \"eventAudienceTeam\": \"\",\n" " \"eventFrequency\": 5,\n" " \"eventTags\": [\n" " \"Informational\"\n" " ],\n" " \"eventSource\": {\n" " \"agentId\": \"Unknown\",\n" - " \"issuingEngine\": \"Agent Core\",\n" + " \"issuingEngine\": \"horizonTelemetryMetrics\",\n" " \"eventTraceId\": \"\",\n" " \"eventSpanId\": \"\",\n" " \"issuingEngineVersion\": \"\",\n" @@ -1625,72 +1604,7 @@ TEST_F(MetricTest, testAIOPSMapMetric) " },\n" " \"eventData\": {\n" " \"eventObject\": {\n" - " \"Metrics\": [\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"/index.html\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 25.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {\n" - " \"url\": \"/index.html\"\n" - " },\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"/index2.html\",\n" - " \"MetricType\": \"gauge\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 20.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {\n" - " \"url\": \"/index2.html\"\n" - " },\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"/index.html\",\n" - " \"MetricType\": \"counter\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 2.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {\n" - " \"url\": \"/index.html\"\n" - " },\n" - " \"AssetID\": \"Unknown\"\n" - " },\n" - " {\n" - " \"Timestamp\": \"2016-11-13T17:31:24.087\",\n" - " \"MetricName\": \"/index2.html\",\n" - " \"MetricType\": \"counter\",\n" - " \"MetricUnit\": \"\",\n" - " \"MetricDescription\": \"\",\n" - " \"MetricValue\": 1.0,\n" - " \"ResourceAttributes\": {\n" - " \"agent\": \"Unknown\",\n" - " \"id\": \"87\"\n" - " },\n" - " \"MetricAttributes\": {\n" - " \"url\": \"/index2.html\"\n" - " },\n" - " \"AssetID\": \"Unknown\"\n" - " }\n" - " ]\n" + " \"records\": \"compress and encode metric payload\"\n" " }\n" " }\n" " }\n" diff --git a/core/rest/rest.cc b/core/rest/rest.cc index 8b15b45..3dcbfe8 100644 --- a/core/rest/rest.cc +++ b/core/rest/rest.cc @@ -33,6 +33,18 @@ ServerRest::performRestCall(istream &in) { try { try { + int firstChar = in.peek(); + if (firstChar != EOF) { + // array as root is not supported in JSONInputArchive format + // but this struct is in bulk invalidations requirements + // we will change the format here + if (firstChar == '[') { + std::string content((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::string modifiedContent = "{\"" + BULK_ARRAY_NAME + "\": " + content + "}"; + std::istringstream* modifiedStream = new std::istringstream(modifiedContent); + in.rdbuf(modifiedStream->rdbuf()); + } + } cereal::JSONInputArchive in_ar(in); load(in_ar); } catch (cereal::Exception &e) { diff --git a/nodes/orchestration/package/open-appsec-cloud-mgmt b/nodes/orchestration/package/open-appsec-cloud-mgmt index 67fa0ee..cabe2b0 100755 --- a/nodes/orchestration/package/open-appsec-cloud-mgmt +++ b/nodes/orchestration/package/open-appsec-cloud-mgmt @@ -58,6 +58,7 @@ send_notification_to_the_fog() "$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \ --header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \ --header "Content-Type: application/json" \ + --header "x-rate-limit-product-type: openappsec" \ --data "{\"logs\": [{\"log\": {\"eventTime\": \ \"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\",\"eventSeverity\": \ \"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\", @@ -73,7 +74,9 @@ send_notification_to_the_fog() upload_res=$(curl -s -o /dev/null -s -w "%{http_code}\n" \ --request POST "$var_fog/api/v1/agents/events/bulk" -H "X-Trace-Id:${correlation_id}" \ --header "Authorization: Bearer ${ra_token}" --header "user-agent: Infinity Next (a7030abf93a4c13)" \ - --header "Content-Type: application/json" --data "{\"logs\": \ + --header "Content-Type: application/json" \ + --header "x-rate-limit-product-type: openappsec" \ + --data "{\"logs\": \ [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \"Agent started onboarding process to cloud management\", \"eventSeverity\": \"Info\",\"eventPriority\": \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\", \"eventLevel\": \"Log\",\"eventAudience\": \"Internal\",\"eventAudienceTeam\": \"Agent Core\",\"eventFrequency\": 0, diff --git a/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s b/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s index 5e3e24a..a76f26b 100755 --- a/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s +++ b/nodes/orchestration/package/open-appsec-cloud-mgmt-k8s @@ -157,7 +157,9 @@ send_notification_to_the_fog() DATE=$(date "+%FT%T.000") upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \ -H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \ - --header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \ + --header "user-agent: Infinity Next (a7030abf93a4c13)" \ + --header "Content-Type: application/json" \ + --header "x-rate-limit-product-type: openappsec" \ --data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \ \"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \ \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\"eventAudience\": \ @@ -171,7 +173,9 @@ send_notification_to_the_fog() sleep 5 upload_res=$(curl -o /dev/null -s -w "%{http_code}\n" --request POST "$var_fog/api/v1/agents/events/bulk" \ -H "X-Trace-Id:${correlation_id}" --header "Authorization: Bearer ${ra_token}" \ - --header "user-agent: Infinity Next (a7030abf93a4c13)" --header "Content-Type: application/json" \ + --header "user-agent: Infinity Next (a7030abf93a4c13)" \ + --header "Content-Type: application/json" \ + --header "x-rate-limit-product-type: openappsec" \ --data "{\"logs\": [{\"log\": {\"eventTime\": \"$DATE\",\"eventName\": \ \"Agent started onboarding process to cloud management\",\"eventSeverity\": \"Info\",\"eventPriority\": \ \"Urgent\",\"eventLogLevel\": \"info\",\"eventType\": \"Event Driven\",\"eventLevel\": \"Log\",\ diff --git a/nodes/orchestration/package/orchestration_package.sh b/nodes/orchestration/package/orchestration_package.sh index 6813ed3..26fbc04 100755 --- a/nodes/orchestration/package/orchestration_package.sh +++ b/nodes/orchestration/package/orchestration_package.sh @@ -924,6 +924,8 @@ install_orchestration() cp_exec "ln -sf /storage/nano_agent${USR_LIB_PATH}/cpnano ${USR_LIB_PATH}/cpnano" cp_exec "mkdir -p /storage/nano_agent/${FILESYSTEM_PATH}" cp_exec "ln -sf /storage/nano_agent/${FILESYSTEM_PATH} ${FILESYSTEM_PATH}" + cp_copy smb_egg/nano-egg-internal /opt/fw1/bin/nano-egg-internal + chmod +x /opt/fw1/bin/nano-egg-internal fi ${INSTALL_COMMAND} lib/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ ${INSTALL_COMMAND} lib/boost/*.so* ${USR_LIB_PATH}/cpnano${VS_LIB_SUB_FOLDER}/ diff --git a/unit_test.cmake b/unit_test.cmake index 2986ba3..032aa8f 100644 --- a/unit_test.cmake +++ b/unit_test.cmake @@ -2,7 +2,7 @@ enable_testing() function(add_unit_test ut_name ut_sources use_libs) add_executable(${ut_name} ${ut_sources}) - target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group) + target_link_libraries(${ut_name} -Wl,--start-group ${use_libs} debug_is report cptest pthread packet singleton environment metric event_is buffers rest config compression_utils z ${GTEST_BOTH_LIBRARIES} gmock boost_regex pthread dl -Wl,--end-group) add_test(NAME ${ut_name} COMMAND ${ut_name}